diff --git a/.github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs b/.github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs index 8ff2bc58646..3ab2b2f4602 100644 --- a/.github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs +++ b/.github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs @@ -1,17 +1,6 @@ import { execFileSync } from 'node:child_process' -// DESIGN CHOICE: shell out to `gcloud auth print-access-token` instead of exchanging the -// external_account credentials file that google-github-actions/auth writes. -// -// Every workflow on the Cloud SQL rollout lease already runs google-github-actions/setup-gcloud -// right after auth (verified across all 11 mutating members), so gcloud is on PATH and already -// bound to the federated identity. Doing the exchange ourselves would mean reimplementing the STS -// token swap plus the service-account impersonation leg, in an action that must stay -// zero-dependency and is duplicated by hand into a second repo. gcloud already handles every ADC -// flavour and refreshes on its own. The metadata server is not an option: it does not exist on -// GitHub or Blacksmith runners. -// -// Consequence, documented in the README: the lease step MUST come after setup-gcloud. +// Use gcloud after setup-gcloud: it handles federated STS/impersonation and ADC refresh without dependencies or repo-local code in this duplicated action. Actions/Blacksmith runners have no metadata server. const TOKEN_REUSE_MS = 40 * 60 * 1_000 // GCP access tokens live ~60 min; re-mint well before that. diff --git a/.github/actions/cloud-sql-rollout-lease/holder-identity.mjs b/.github/actions/cloud-sql-rollout-lease/holder-identity.mjs index 105d0b3bff0..55fe994350d 100644 --- a/.github/actions/cloud-sql-rollout-lease/holder-identity.mjs +++ b/.github/actions/cloud-sql-rollout-lease/holder-identity.mjs @@ -1,5 +1,4 @@ -// Who we claim to be on the lease object. Shared by main and the detached renewer so both agree -// on the holder key without re-deriving it from a different set of environment variables. +// Main and detached renewer must derive the same holder key from one environment contract. export function holderIdentity(explicitHolderKey) { const repository = process.env.GITHUB_REPOSITORY ?? 'unknown' diff --git a/.github/actions/cloud-sql-rollout-lease/renew.mjs b/.github/actions/cloud-sql-rollout-lease/renew.mjs index cc8e543ba4f..b4c8422a2ff 100644 --- a/.github/actions/cloud-sql-rollout-lease/renew.mjs +++ b/.github/actions/cloud-sql-rollout-lease/renew.mjs @@ -3,8 +3,7 @@ import { createAccessTokenSource } from './gcloud-access-token.mjs' import { renewerLogPath } from './runner-state.mjs' import { CloudSqlRolloutLease, RENEW_INTERVAL_MS } from './storage-lease.mjs' -// Detached renewer. `main` spawns it, `post` kills it. It rewrites expires_at on the same -// generation-matched path as acquisition, and stops the moment the object stops being ours. +// Detached renewer shares acquisition’s generation-matched writes; stop when ownership changes. Main starts it and post kills it. const MAX_LIFETIME_MS = 6 * 60 * 60 * 1_000 // Backstop if post never runs (runner killed). diff --git a/.github/actions/cloud-sql-rollout-lease/runner-state.mjs b/.github/actions/cloud-sql-rollout-lease/runner-state.mjs index ec36a16e0f2..147ced144f9 100644 --- a/.github/actions/cloud-sql-rollout-lease/runner-state.mjs +++ b/.github/actions/cloud-sql-rollout-lease/runner-state.mjs @@ -1,7 +1,6 @@ import { appendFileSync } from 'node:fs' -// Action-state and output plumbing via the runner's file protocol, so the action needs no -// @actions/core dependency. Values are single-line by construction; anything else is rejected. +// Use runner files without @actions/core; reject multiline values rather than corrupting their protocol. /** The detached renewer's stdio is ignored, so it appends here instead and `post` echoes it. */ export function renewerLogPath() { diff --git a/.github/actions/cloud-sql-rollout-lease/storage-lease.mjs b/.github/actions/cloud-sql-rollout-lease/storage-lease.mjs index 44312377ee3..7d8c1f94b5d 100644 --- a/.github/actions/cloud-sql-rollout-lease/storage-lease.mjs +++ b/.github/actions/cloud-sql-rollout-lease/storage-lease.mjs @@ -1,10 +1,4 @@ -// Compare-and-swap lease over a single Cloud Storage object. -// -// Ported from apps/relay-fence-broker/src/mutation-lease.ts rather than imported: this action is -// duplicated verbatim into stablyai/orca, so it must carry no repo-local imports. Only the -// algorithm is shared (read metadata -> write with ifGenerationMatch -> 412 is a conflict -> -// generation-matched delete -> an expired record is free). The broker's token path is NOT shared; -// it reads the GCE metadata server, which does not exist on Actions runners. +// Ported CAS algorithm from relay-fence-broker/mutation-lease.ts: keep repo-local imports out of this duplicated action. Broker metadata-server tokens cannot work on Actions runners; use generation-matched writes/deletes, 412 conflicts and expiry. export const LEASE_TTL_MS = 35 * 60 * 1_000 export const RENEW_INTERVAL_MS = 5 * 60 * 1_000 diff --git a/.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs b/.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs index f13e01517be..73d1a1e6e78 100644 --- a/.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs +++ b/.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs @@ -12,10 +12,7 @@ const BUCKET = 'onorca-cloud-terraform-state' const OBJECT = 'terraform/state/cloud-sql-rollout/production.lock' const NOW = 1_756_000_000_000 -/** - * Enough of the Cloud Storage JSON API to exercise real compare-and-swap semantics: generations - * increment, ifGenerationMatch is enforced, and a mismatch is a 412. `faults` injects failures. - */ +/** Model real GCS generation increments, ifGenerationMatch and 412 conflicts; faults inject failures. */ function fakeStorage({ object = null, faults = [] } = {}) { const state = { object, requests: [] } const fetcher = async (rawUrl, init = {}) => { diff --git a/.github/actions/install-mobile-dependencies/action.yml b/.github/actions/install-mobile-dependencies/action.yml index 0ed2b45bb9a..a8219e37cf6 100644 --- a/.github/actions/install-mobile-dependencies/action.yml +++ b/.github/actions/install-mobile-dependencies/action.yml @@ -4,12 +4,7 @@ description: Frozen pnpm install for the mobile/ project, whose node_modules the runs: using: composite steps: - # Why a separate install: mobile is its own pnpm project, so the root install leaves - # mobile/node_modules empty and every mobile import resolves to nothing. - # Why no --ignore-scripts, unlike the root install: mobile's postinstall generates the - # gitignored terminal/mermaid webview engine modules that tracked source imports. - # The drift guard mirrors the root install so a stale mobile lockfile fails by name -- - # mobile's lockfile carries patchedDependencies that a silent rewrite would drop. + # Install the separate mobile project: root install leaves its imports unresolved. Keep lifecycle scripts for generated terminal/mermaid engines; reject lockfile drift that could drop patchedDependencies. - name: Install mobile dependencies shell: bash working-directory: mobile diff --git a/.github/actions/install-node-dependencies/action.yml b/.github/actions/install-node-dependencies/action.yml index d5f119879b7..3ae48e01da6 100644 --- a/.github/actions/install-node-dependencies/action.yml +++ b/.github/actions/install-node-dependencies/action.yml @@ -222,10 +222,7 @@ runs: rm -rf node_modules fi - # Why --frozen-lockfile: re-resolving pulls ~62 MB of registry packuments per job - # (measured) to recompute what the lockfile already pins, and the `git diff` guard - # below fails the run whenever that recomputation would have changed anything. The - # guard stays so a stale lockfile still fails by name rather than by resolver error. + # Frozen install avoids ~62MB/job of redundant resolution; retain diff guard to name stale lockfiles explicitly. - name: Install dependencies shell: bash run: | diff --git a/.github/actions/install-signpath-module/action.yml b/.github/actions/install-signpath-module/action.yml index 3f41daca3ab..19de4a94092 100644 --- a/.github/actions/install-signpath-module/action.yml +++ b/.github/actions/install-signpath-module/action.yml @@ -29,13 +29,7 @@ runs: # Why: force TLS 1.2 so gallery downloads work on older hosted images. [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 - # Why: on some hosted Windows images `Register-PSRepository -Default` - # fails inside the legacy nuget.exe provider with "Missing option value - # for: '-source'", so PSGallery is never registered and the install - # below dies with "No repository with the name 'PSGallery'". PSResourceGet - # (bundled with PowerShell 7.4+) has PSGallery registered by default and - # avoids that code path, so prefer it and fall back to PowerShellGet only - # when it is absent. + # Prefer PSResourceGet (PowerShell 7.4+): legacy nuget Register-PSRepository can fail with “Missing option value for: -source” and leave PSGallery absent. $useResourceGet = $null -ne (Get-Command -Name Install-PSResource -ErrorAction SilentlyContinue) try { @@ -53,9 +47,7 @@ runs: Set-PSRepository -Name PSGallery -InstallationPolicy Trusted } } catch { - # Why: repository registration also talks to the gallery, so a gallery - # outage can fail here before a single install is attempted. The CDN - # fallback below does not need a registered repository, so keep going. + # Repository registration can fail during gallery outages; continue so the independent CDN fallback remains reachable. Write-Warning "PSGallery repository registration failed: $_" } @@ -114,13 +106,8 @@ runs: } } - # Why: the gallery's package API (OData search + repository metadata) sits - # behind Azure Front Door and has returned 403/502/504 for every install - # attempt during gallery incidents, which hard-failed the whole Windows - # release job. The CDN that serves the nupkg itself is a separate origin - # and stays up through those incidents, so fall back to a pinned version - # fetched straight from it. The hash pin is mandatory: this route skips the - # gallery's package validation, so an unexpected payload must fail loudly. + # Gallery OData/metadata has returned 403/502/504 while the independent nupkg CDN stayed up. + # Fall back to a pinned version and mandatory hash because this bypasses gallery validation. if (-not $installed) { $version = $env:SIGNPATH_FALLBACK_VERSION $expectedHash = $env:SIGNPATH_FALLBACK_SHA256 @@ -131,9 +118,7 @@ runs: Remove-Item -LiteralPath $nupkg -Force } - # Why two URLs: the /api/v2/package route 302s to the CDN and can serve - # while the OData search endpoint is failing; the CDN URL is the same - # redirect target reached directly when the api host is down entirely. + # Try the API’s CDN redirect, then the CDN directly if the API host is down. $sources = @( "https://www.powershellgallery.com/api/v2/package/SignPath/$version", "https://cdn.powershellgallery.com/packages/signpath.$version.nupkg" @@ -173,9 +158,7 @@ runs: } Remove-SignPathModuleDirectory - # Why a version-named subdirectory: PowerShell only treats a nested folder - # as a side-by-side module version when the name matches the manifest's - # ModuleVersion, which is what makes `Import-Module SignPath` resolve it. + # PowerShell resolves side-by-side modules only when the directory matches the manifest’s ModuleVersion. $versionRoot = Join-Path -Path $signPathModulePath -ChildPath $version New-Item -ItemType Directory -Path $versionRoot -Force | Out-Null Expand-Archive -LiteralPath $nupkg -DestinationPath $versionRoot -Force diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 681fe94fbb8..355c440220f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,6 @@ version: 2 updates: - # Why only ripgrep: the bundled rg ships in every artifact and to SSH remotes, and a bump is a - # one-line pin change (the SSH cache keys on the binary's hash). Other dependencies stay manual. + # Automate only bundled ripgrep’s single pin; its hash invalidates SSH cache. Other dependency updates stay manual. - package-ecosystem: npm directory: / schedule: diff --git a/.github/scripts/check-root-directory-entries.mjs b/.github/scripts/check-root-directory-entries.mjs index 0087892c88b..642539d28e1 100644 --- a/.github/scripts/check-root-directory-entries.mjs +++ b/.github/scripts/check-root-directory-entries.mjs @@ -2,10 +2,7 @@ import { execFileSync } from 'node:child_process' import { randomUUID } from 'node:crypto' function readRootEntries(sha) { - // Why: a git pathname is arbitrary bytes, and 'utf8' folds every invalid - // sequence to U+FFFD — that mangles the reported name and makes two different - // entries compare equal, so a genuinely new one can slip past the Set below. - // latin1 maps each byte to one code unit, so the bytes survive the round trip. + // Git paths are arbitrary bytes; latin1 round-trips them while utf8 merges invalid sequences into U+FFFD, hiding distinct entries. const stdout = execFileSync('git', ['ls-tree', '-z', '--name-only', sha], { encoding: 'latin1', stdio: ['ignore', 'pipe', 'inherit'] @@ -42,10 +39,7 @@ function checkRootDirectoryEntries(argv) { ) console.log('Move each new entry under an existing top-level directory.') console.log('Blocked entries:') - // Why: an entry name is attacker-controlled and may start with '::' (the runner - // trims leading spaces before matching) or embed a newline, so printing it bare - // lets a PR forge annotations. Fence the untrusted list with an unguessable - // stop-commands token, and write the raw bytes rather than a re-encoded string. + // Untrusted names can forge runner commands/newlines; fence them with a random stop-commands token and preserve raw bytes. const resumeToken = randomUUID() console.log(`::stop-commands::${resumeToken}`) for (const entry of blockedEntries) { @@ -56,12 +50,10 @@ function checkRootDirectoryEntries(argv) { } try { - // Why: process.exit truncates a piped write part-way through on macOS, so set - // exitCode and let node flush the blocked-entry list before it exits. + // Let piped output flush; process.exit can truncate it on macOS. process.exitCode = checkRootDirectoryEntries(process.argv.slice(2)) } catch (error) { - // Why: git already reported the failure on the inherited stderr, so surface its - // status rather than a node stack trace. Anything else is a real bug — rethrow. + // Git already reported failures; preserve its exit status and rethrow other bugs. if (typeof error.status !== 'number') { throw error } diff --git a/.github/scripts/pr-test-loc-table.mjs b/.github/scripts/pr-test-loc-table.mjs index c5d233c4497..2ea55837de7 100644 --- a/.github/scripts/pr-test-loc-table.mjs +++ b/.github/scripts/pr-test-loc-table.mjs @@ -3,9 +3,7 @@ export const LOC_BLOCK_END = '' export const LOC_HANDS_OFF_COMMENT = '' -// Why the __-wrapped names: they are unambiguous test-data markers, so a dir -// cannot be one by accident. Bare `fixtures/` is deliberately absent — it reads -// as a plausible prod module name, and this classifier bills every PR. +// __-wrapped segments unambiguously mark test data; bare fixtures/ could be production, and this classifier bills every PR. const TEST_DIR_SEGMENT = /(?:^|\/)(?:__tests__|__fixtures__|__snapshots__|e2e|tests)(?:\/|$)/i const TEST_FILENAME = /\.(?:test|spec|e2e)\.[^/]+$/i diff --git a/.github/workflows/adhoc-mac-build.yml b/.github/workflows/adhoc-mac-build.yml index 0c88b53561e..7cabc3667bb 100644 --- a/.github/workflows/adhoc-mac-build.yml +++ b/.github/workflows/adhoc-mac-build.yml @@ -1,58 +1,12 @@ name: Adhoc macOS + Windows Dev Build -# Why: lets anyone cut installable macOS and Windows builds of a trusted repo ref -# so the team can run an experimental feature instead of reasoning from a diff. -# Hourly covers current main; this covers experiments and on-demand validation. -# -# Deliberately narrow scope: -# - macOS and Windows desktop installers. Linux keeps using RC/stable. -# - No tests, no lint, no e2e. PR CI and release-cut remain the gates. The one -# exception is the orcad template: like release-cut, it is merged from the -# node-server lanes that build and qualify each SSH target's slot. -# - macOS is signed and notarized so TCC grants survive updates. -# - Windows is unsigned; the published release notes explain the one-time -# SmartScreen/manual-install requirement. -# -# Artifacts publish to stablyai/orca-adhoc — separate from both orca and -# orca-hourly. Separate from orca because the main repo's releases atom feed -# exposes only its 10 newest entries. Separate from orca-hourly because a build -# from someone's branch must never be picked up by a developer who only meant to -# ride main; the two are different levels of "unvetted". -# -# From the Actions tab: pick this workflow, "Run workflow", leave "Use workflow -# from" on main, and type your branch in the first field. Or from the CLI: -# -# gh workflow run adhoc-mac-build.yml --ref main -f ref=my-branch -f label=wasm-terminal -# -# Leaving the ref field empty builds whatever "Use workflow from" is set to, which -# is what someone who only touched that picker means. Naming the branch explicitly -# is still better: the workflow file is always read from the dispatch ref, so a -# branch carrying a stale copy of this file would otherwise run that copy. -# -# GITHUB_TOKEN is scoped to this repo and cannot publish there, so writes use the -# same GitHub App as hourly, additionally installed on orca-adhoc with -# Contents: Read and write. The secret names below are historical — one App, one -# private key, both dev-channel repos — and rotating it stays a single operation. -# Provision with `bash config/scripts/setup-hourly-release-token.sh`. -# -# The requested ref is vetted before checkout: it must be a branch or tag of this -# repo, or a commit reachable from one. PR refs are refused outright — this -# workflow runs the checked-out code next to MAC_CERTS and the notary password, -# so "just build that community PR" must not become a way to hand fork code the -# release identity. A branch here always belongs to someone with write access, -# which is the same trust the dispatch button itself already requires. +# Build trusted repo refs on demand: signed/notarized macOS and unsigned Windows, with qualified orcad slots; PR/release pipelines own other gates. Publish to orca-adhoc so branch experiments cannot enter main’s hourly channel or crowd stable/RC release feeds. Dispatch main’s workflow with -f ref=my-branch to avoid stale YAML. Writes use the hourly GitHub App installed on orca-adhoc (setup-hourly-release-token.sh); GITHUB_TOKEN cannot publish cross-repo. Ref vetting rejects PR refs and unreachable SHAs before source runs beside signing secrets. on: workflow_dispatch: inputs: ref: - # Why optional: the Actions UI already shows its own "Use workflow from" - # branch picker directly above this field, and picking a branch there is - # what most people will read as "build this". Defaulting to that branch - # makes the obvious action correct. Fill this in only to build a ref other - # than the one the workflow file itself is read from — normally leave the - # picker on main and name your branch here, so a stale copy of this - # workflow on an old branch is not what runs. + # Default empty ref to the dispatch picker; prefer main’s workflow plus explicit branch input to build old branches with current YAML. description: 'Branch, tag, or SHA to build — must live in stablyai/orca; PR refs are refused (default: the branch selected above)' required: false default: '' @@ -67,27 +21,18 @@ permissions: contents: read concurrency: - # Why keyed on the ref rather than global: two people cutting builds from two - # different branches at the same time is the ordinary case here, and serialising - # them would make each wait out the other's notary queue. Re-dispatching the - # *same* branch still queues, so a push mid-build cannot race itself. + # Serialize per ref: queue repeat builds while independent branches avoid each other’s notary wait. group: adhoc-mac-build-${{ inputs.ref || github.ref_name }} cancel-in-progress: false env: ORCA_BACKGROUND_LAUNCH: '1' ADHOC_REPO: stablyai/orca-adhoc - # Why age and not a count like hourly: this channel is low-volume and bursty, so - # a count would either hold one week's experiments forever or evict a build - # someone is still running after a busy afternoon. A month is well past the "few - # days" these exist for, and by then the branch has landed or been abandoned. + # Retain by month rather than count so bursts cannot evict recent experiments or sparse periods keep them forever. ADHOC_RETAIN_DAYS: 30 jobs: - # Why: every job below checks out its own copy, and a branch name read per job builds whatever - # the branch points at when that job starts. A push mid-run then mixes commits, e.g. slot lanes - # from one commit merged by a template step from the next. Resolving once pins the whole run. - # Validate reachability before any job executes the requested source. + # Resolve once before source execution so concurrent pushes cannot mix slot/template/desktop commits. resolve-ref: if: github.repository == 'stablyai/orca' runs-on: blacksmith-2vcpu-ubuntu-2404 @@ -111,17 +56,12 @@ jobs: exit 1 ;; esac - # Bare: a work-tree repo refuses to fetch over its own checked-out - # branch. tree:0 keeps the fetch to the commit graph — no trees, no - # blobs — so this stays cheap next to the build it fronts. reftable - # because this repo has branches that differ only in casing, and the - # files backend cannot store both on a case-insensitive runner disk — - # it fails the entire fetch, not just the one ref. + # Bare avoids fetching over a checked-out branch; tree:0 fetches only the commit graph. + # Reftable preserves case-only branch names that make the files backend fail on case-insensitive disks. scratch="$RUNNER_TEMP/vet-requested-ref" git init -q --bare --ref-format=reftable "$scratch" git -C "$scratch" fetch -q --filter=tree:0 "$REPO_URL" '+refs/heads/*:refs/heads/*' '+refs/tags/*:refs/tags/*' - # Branch first to keep actions/checkout's old tie-break: bare - # rev-parse would prefer the tag when a branch shares its name. + # Resolve branches before tags, matching actions/checkout’s tie-break. sha="" case "$REQUESTED_REF" in refs/heads/*|refs/tags/*) candidates=("$REQUESTED_REF") ;; @@ -143,10 +83,7 @@ jobs: echo "::error::'$REQUESTED_REF' does not resolve to a branch, tag, or commit of stablyai/orca." exit 1 fi - # The object resolving locally is not proof a branch or tag reaches - # it: a partial clone can lazily fetch a bare SHA on demand, and - # GitHub serves PR-only commits by SHA. Reachability is the actual - # trust test. + # Lazy-fetchable SHAs can be PR-only; require reachability from a trusted branch/tag, not local object presence. if [[ -z "$(git -C "$scratch" for-each-ref --count=1 --format='%(refname)' --contains "$sha" refs/heads refs/tags)" ]]; then echo "::error::Commit $REQUESTED_REF is not reachable from any branch or tag of stablyai/orca; refusing to build it with release credentials." exit 1 @@ -212,20 +149,15 @@ jobs: MAIN_REPO_TOKEN: ${{ github.token }} run: | set -euo pipefail - # Why this check: the version scripts are read from the branch being built, - # not from main, so a branch cut before the adhoc channel landed has no - # copy of them. Say that plainly instead of failing with a module-not-found. + # Old branch refs may predate adhoc version scripts; report that explicitly. for script in adhoc-build-version dev-channel-base-version; do if [[ ! -f "config/scripts/$script.mjs" ]]; then echo "::error::$REF has no config/scripts/$script.mjs; rebase it onto a main that has the adhoc channel." exit 1 fi done - # Why git tags, not GitHub releases: unpublishing a buggy cut deletes the - # GitHub release and leaves the tag, which still owns that number. - # Releases-only let adhoc sit on a number already taken, so the updater - # would not install it. Empty on failure — the script then falls back - # to package.json. + # Unpublishing deletes releases but leaves version-owning tags; use tags to prevent updater regression. + # On lookup failure the script falls back to package.json. published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \ "repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \ --jq '.[].ref | sub("^refs/tags/"; "")' || true)" @@ -238,10 +170,7 @@ jobs: fi cat "$RUNNER_TEMP/adhoc-identity.txt" >>"$GITHUB_OUTPUT" - # Design D2, as release-cut does: every desktop build ships the orcad template (server JS plus - # every target's addons), merged from the node-server lanes that qualify each slot at this ref. - # Without it an adhoc build cannot deploy managed orcad to an SSH host. No secrets, and the mac - # job vets the ref before anything is signed. + # Ship managed-SSH orcad template from qualified per-target lanes at the vetted ref, as release-cut does. orcad-template: needs: [resolve-ref, orcad-template-support] if: needs.orcad-template-support.outputs.ships == 'true' @@ -293,22 +222,13 @@ jobs: NAME: ${{ needs.orcad-template-support.outputs.name }} SHA: ${{ needs.resolve-ref.outputs.sha }} REF: ${{ inputs.ref || github.ref_name }} - # Via env, not inline `${{ }}`: both land inside a shell string, and an - # expression expanded there is substituted before bash parses the line - # (zizmor: template-injection). + # Pass expressions via env so bash parses data rather than substituted script text (zizmor: template-injection). ACTOR: ${{ github.actor }} run: | set -euo pipefail short_sha="${SHA:0:12}" - # Why create it up front: electron-builder then uploads into a known tag - # rather than inferring one from package.json. - # - # Why --draft: everything between here and the manifest check is a window - # where the release exists but has no installable assets. A draft is - # absent from the releases list and from listReleaseBuilds, so a job that - # dies in that window — including a hard kill by the job timeout, which - # runs no cleanup step at all — leaves something invisible rather than a - # tag the picker offers and the download 404s on. + # Precreate a known tag as draft so builder cannot infer it from package.json. + # Keep incomplete assets invisible until manifest verification, even if timeout kills the job without cleanup. gh release create "$TAG" \ --repo "$ADHOC_REPO" \ --title "$NAME" \ @@ -337,13 +257,7 @@ jobs: needs.orcad-template-support.result == 'success' && needs.release-javascript.result == 'success' && needs.prepare-adhoc-mac-release.result == 'success' - # Why an environment: it gives the signing/notary/App secrets somewhere to - # live that a stale copy of this workflow on an old branch cannot reach. - # Referencing it is a no-op until repo settings give it teeth; the intended - # follow-up is to move MAC_CERTS, MAC_CERTS_PASSWORD, APPLE_ID, - # APPLE_APP_SPECIFIC_PASSWORD, APPLE_TEAM_ID, HOURLY_RELEASE_APP_ID, and - # HOURLY_RELEASE_APP_PRIVATE_KEY into it, then pin its deployment branch - # policy to main so only main's copy of this file can read them. + # Signing environment must restrict deployments to main after MAC_CERTS/password, APPLE_ID/app password/team and hourly App secrets move there; referencing it alone enforces nothing. environment: adhoc-mac-build permissions: contents: read @@ -353,21 +267,13 @@ jobs: matrix: arch: [x64, arm64] runs-on: blacksmith-12vcpu-macos-15 - # Why 150: it must exceed the worst case the retry budgets below can produce - # (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or - # the job is killed mid-retry and no cleanup step runs at all. + # Allow 150min for install 3×10 + publish 2×45 + ~25 setup/build/verify; outer timeout would prevent cleanup. timeout-minutes: 150 env: JOBS: max NODE_OPTIONS: --max-old-space-size=4096 steps: - # Why vet before checkout: everything after this step runs the checked-out - # code with release signing credentials in reach. Branches and tags of this - # repo are the intended audience; refs/pull/* would smuggle in fork code, - # and a raw SHA is only accepted when some branch or tag of this repo can - # actually reach it. Resolving to a pinned SHA here also means the commit - # that was vetted is the commit that gets checked out — a push to the - # branch between the two steps cannot swap it. + # Vet branch/tag reachability and reject refs/pull before checkout can access signing credentials; checkout the resolved SHA to prevent ref-swap races. - name: Vet the requested ref id: vetted shell: bash @@ -380,17 +286,11 @@ jobs: - name: Checkout the requested ref uses: actions/checkout@v6 with: - # Why an input at all rather than just github.ref: the whole point is to - # build code that has not landed, and the workflow definition itself - # always comes from the dispatch ref — naming the branch here instead - # applies main's current copy of this file to an arbitrary branch. + # Explicit source input lets main’s workflow build an unmerged branch rather than run that branch’s stale workflow. ref: ${{ steps.vetted.outputs.sha }} # Version helpers only read HEAD; published versions come from the release API. fetch-depth: 1 - # This job only reads stablyai/orca and never pushes; every write goes - # to the adhoc repo through a minted App token passed by env. Not - # persisting the checkout credential shrinks the blast radius if a build - # step is compromised (zizmor: artipacked). + # Read this repo without persisted checkout credentials; all writes use an env-passed App token for orca-adhoc (zizmor: artipacked). persist-credentials: false - name: Setup pnpm @@ -408,8 +308,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Why: packaging bundles the pinned Node into the macOS terminal host. Keyed on the pin alone - # with no restore-keys, so a pin bump starts empty instead of carrying old archives forward. + # Cache bundled host Node by exact pin, without restore keys, so bumps cannot restore old archives. - name: Locate the pinned Node runtime cache shell: bash run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV" @@ -430,8 +329,7 @@ jobs: restore-keys: | electron-builder-mac- - # Why both CPUs: the mac config packages x64 and arm64 from this arm64 - # runner, so the install must carry both variants of the native optional deps. + # Install both x64/arm64 optional natives because this arm64 runner packages both. - name: Install dependencies uses: nick-fields/retry@v4 with: @@ -440,13 +338,11 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 - # Why here: electron-builder's beforePack requires out/mobile-web, and the bundle - # build resolves React Native and Expo from mobile/node_modules. + # Build required out/mobile-web after installing its React Native/Expo dependencies. - uses: ./.github/actions/install-mobile-dependencies if: needs.release-javascript.outputs.supported != 'true' - # Why: signing is what makes an adhoc build installable over an existing - # Orca, so a missing cert must fail here rather than after a 20-minute build. + # Require signing cert before expensive build; unsigned macOS cannot update an existing install correctly. - name: Verify macOS signing environment run: node config/scripts/verify-macos-release-env.mjs env: @@ -475,9 +371,7 @@ jobs: ORCA_RELEASE_JAVASCRIPT_SOURCE_SHA: ${{ needs.release-javascript.outputs.supported == 'true' && needs.release-javascript.outputs.source_sha || '' }} ORCA_RELEASE_JAVASCRIPT_BUILD_IDENTITY: dev NODE_OPTIONS: --max-old-space-size=4096 - # Why: adhoc builds are not an official channel — telemetry's transport - # gate accepts only 'stable' or 'rc', so leaving this unset keeps them - # silent, which is correct for unvetted branch artifacts. + # Leave build identity unset: telemetry accepts only official stable/rc, so branch artifacts stay silent. ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token - name: Wait for the qualified orcad template @@ -512,10 +406,7 @@ jobs: name: orcad-template path: out/orcad-template - # Why the token is minted here and not at the top: installation tokens live - # one hour, everything before this point writes nothing, and the notary round - # trip inside the publish step can be tens of minutes. Minting after the build - # starts the clock at the first call that actually uses it. + # Mint the one-hour write token after build, just before potentially long notarization/publish. - name: Checkout workflow packaging scripts uses: actions/checkout@v6 with: @@ -528,9 +419,7 @@ jobs: - name: Package and notarize this Mac architecture uses: nick-fields/retry@v4 with: - # Why 45: an attempt is pack + notarize, and the notary queue is - # the unbounded part. Two attempts, because a failed adhoc build has a - # person waiting on it who can simply dispatch again. + # Allow two 45min pack/notarize attempts; an adhoc user can redispatch rather than pay for a third. timeout_minutes: 45 max_attempts: 2 retry_wait_seconds: 30 @@ -542,9 +431,7 @@ jobs: ORCA_REQUIRE_ORCAD_TEMPLATE: ${{ needs.orcad-template-support.outputs.ships == 'true' && '1' || '' }} CSC_LINK: ${{ secrets.MAC_CERTS }} CSC_KEY_PASSWORD: ${{ secrets.MAC_CERTS_PASSWORD }} - # Why all three: electron-builder's notarize step authenticates to the - # Apple notary service with the app-specific password, not with the - # signing cert. Omitting them fails the build rather than skipping it. + # Require Apple id/app-specific password/team: notarization authenticates separately from signing cert. APPLE_ID: ${{ secrets.APPLE_ID }} APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} @@ -677,9 +564,7 @@ jobs: } throw new Error('Timed out waiting for the qualified orcad template') - # Why this is the last mutating step: publishing the draft is what makes the - # build visible to listReleaseBuilds. Doing it only after the manifest check - # means the picker can never offer a release whose assets are incomplete. + # Publish only after manifest validation so the picker never offers incomplete assets. - name: Publish the verified release id: publish_live shell: bash @@ -689,18 +574,11 @@ jobs: NAME: ${{ needs.orcad-template-support.outputs.name }} run: | set -euo pipefail - # --title again: electron-builder resolves this draft by tag and may - # rewrite its title on upload. Re-asserting here means the name the - # picker reads is the one composed above, whatever it did in between. + # Reassert the picker title after electron-builder may rewrite the tag’s draft on upload. gh release edit "$TAG" --repo "$ADHOC_REPO" --draft=false --prerelease --title "$NAME" echo "Published $TAG as \"$NAME\"" - # Why: a draft left behind by a failed publish is invisible to users but still - # holds its tag name. Gated on publish_live not having succeeded so a later - # failure (the prune step) cannot delete a release that already went live and - # that people may already be installing. Why cancelled() too: a run stopped - # from the Actions UI is not a failure(), so without it a manual cancel - # mid-publish would strand the draft. + # Delete stranded drafts on failure/cancel, but never after publish_live: later prune failure must not remove an installable release. - name: Discard the draft release on failure if: >- (failure() || cancelled()) && needs.prepare-adhoc-mac-release.outputs.tag != '' && @@ -717,32 +595,21 @@ jobs: echo "::warning::Could not discard draft $TAG; remove it manually." - name: Prune expired adhoc releases - # Only after a live publish: $TAG is then a non-draft this step must not - # delete, and a run that failed before publishing has nothing to retire. + # Retire only after live publication; preserve this non-draft tag and leave failed runs to draft cleanup. if: steps.publish_live.outcome == 'success' shell: bash env: GH_TOKEN: ${{ steps.app_token.outputs.token }} - # Protect the tag this run just shipped, so no filter mistake can delete - # a build minutes after the person who cut it was told it exists. + # Exclude the just-published tag from pruning even if ordering/filter logic regresses. TAG: ${{ needs.prepare-adhoc-mac-release.outputs.tag }} run: | set -euo pipefail - # Why compute the cutoff in bash rather than with jq's `now`: this runs - # once per dispatch, and a fixed epoch makes the threshold visible in the - # log when someone asks where their build went. + # Fix/log the cutoff epoch once per dispatch so retention decisions remain auditable. cutoff=$(( $(date -u +%s) - ADHOC_RETAIN_DAYS * 86400 )) echo "Pruning adhoc releases published before $(date -u -d "@$cutoff" '+%Y-%m-%dT%H:%M:%SZ')" - # --cleanup-tag so pruning does not leave orphan tags with no release or - # assets attached. Drafts are excluded: a stale draft is the failure - # path's business, not the retention window's. - # - # Age comes from publishedAt, never createdAt. GitHub reports createdAt - # as the date of the *commit* a release's tag points at, and every tag - # here is cut from this repo's one seed commit — so all of them carry - # that same createdAt, and the day the window rolled past it the entire - # channel expired at once and a single run deleted it. A release with no - # publishedAt is kept rather than aged by guesswork. + # --cleanup-tag removes release-less tags; failure cleanup owns drafts, which retention excludes. + # Use publishedAt: GitHub’s createdAt is the seed commit’s date for every tag. Aging by it once + # expired the entire channel in one run; retain releases with unknown publishedAt. jq_filter='map(select(.isDraft | not))' if [[ -n "${TAG:-}" ]]; then jq_filter+=" | map(select(.tagName != \"${TAG//\"/\\\"}\"))" diff --git a/.github/workflows/agent-state-rules-publish.yml b/.github/workflows/agent-state-rules-publish.yml index 8a24adbc73f..716c8368fbc 100644 --- a/.github/workflows/agent-state-rules-publish.yml +++ b/.github/workflows/agent-state-rules-publish.yml @@ -1,13 +1,6 @@ name: Publish agent state rules -# The only publisher of agent state rules releases. Merging never publishes: a maintainer runs -# this from main, first to `next` (RC and dev builds), then promotes the identical file to -# `stable` after a soak. Every job runs on the dispatched commit, so the publish job runs exactly -# the script and rules the gate tested. -# -# Why GITHUB_TOKEN: release-policy.yml deletes an agent-state-rules-* release any other author -# publishes. Its releases start no workflow, so release-policy.yml sees them only when someone -# edits one by hand, and then accepts them as bot-authored prereleases. +# Maintainer-only publisher: dispatch main to next, soak, then promote identical bytes to stable. All jobs use the dispatched commit. Use GITHUB_TOKEN so release-policy accepts bot-authored prereleases; token-created releases do not trigger workflows until manual edits. on: workflow_dispatch: inputs: diff --git a/.github/workflows/cloud-verify.yml b/.github/workflows/cloud-verify.yml index 5c8ff5e0341..097a2118827 100644 --- a/.github/workflows/cloud-verify.yml +++ b/.github/workflows/cloud-verify.yml @@ -80,8 +80,7 @@ jobs: - wait: history-scan - # Compiles the workspace. No Postgres service: nothing here reaches a - # database, and the service container costs ~13s of startup. + # Compile without Postgres: no database access occurs here, avoiding ~13s service startup. build: if: github.event_name != 'pull_request' || github.event.pull_request.draft != true runs-on: blacksmith-4vcpu-ubuntu-2204 @@ -167,9 +166,7 @@ jobs: path: cloud/security-reports/ retention-days: 30 - # Runs in parallel with build. `pnpm test` compiles the workspace - # packages it needs through pretest hooks, so it does not depend on - # `pnpm build` having run. + # Run tests beside build; pretest hooks compile their own workspace dependencies. test: if: github.event_name != 'pull_request' || github.event.pull_request.draft != true runs-on: blacksmith-4vcpu-ubuntu-2204 diff --git a/.github/workflows/computer-e2e.yml b/.github/workflows/computer-e2e.yml index 54d386e9c54..11c877c9952 100644 --- a/.github/workflows/computer-e2e.yml +++ b/.github/workflows/computer-e2e.yml @@ -25,9 +25,7 @@ on: - 'config/scripts/daemon-endpoint-handover-smoke.mjs' - 'config/scripts/windows-daemon-workspace-close-repro.mjs' - 'config/scripts/verify-computer-native.mjs' - # Why: the native-smoke job boots the built terminal daemon under plain - # Node, so any change to the daemon bundle graph or the main build must - # re-run it (the v1.4.129-rc.1 daemon outage shipped with green CI). + # Re-smoke daemon bundle/build changes under plain Node; v1.4.129-rc.1 shipped a module-load outage despite green CI. - 'electron.vite.config.ts' - 'config/build-plugins/**' - 'src/main/daemon/**' @@ -90,10 +88,7 @@ jobs: - uses: ./.github/actions/install-node-dependencies with: native-runtime: node - # Why: without --config, bare vitest ignores config/vitest.config.ts (there - # is no root config) and falls back to the 5s default timeout with no - # Windows worker cap, so the real csc.exe launcher-compile tests time out - # on hosted Windows. Use the shared config so this job matches pnpm test. + # Use shared Vitest config: bare invocation misses config/ and its timeout/Windows worker cap, breaking real csc.exe tests. - run: >- pnpm vitest run --config config/vitest.config.ts config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs @@ -130,19 +125,13 @@ jobs: - run: pnpm verify:computer-native - run: pnpm build:cli - run: pnpm run build:electron-vite:parallel - # Why: boot the BUILT daemon-entry under plain Node the way production - # forks it. v1.4.129-rc.1 shipped a daemon that exited at module load - # (leaked electron require) while every other check passed; this fails - # the PR when the built daemon cannot start on ubuntu-22.04 / windows. + # Boot built daemon-entry under plain Node on Ubuntu/Windows, catching leaked electron requires like v1.4.129-rc.1. - name: Daemon boot smoke run: node config/scripts/daemon-boot-smoke.mjs - # Why: a daemon that lost its endpoint name used to delete the replacement's - # socket on exit, stranding a live daemon that hosts PTYs nothing can reach. - # Only two real processes racing the same endpoint reproduce it. + # Race real daemon processes on one endpoint; a loser must not unlink the replacement’s socket. - name: Daemon endpoint handover smoke run: node config/scripts/daemon-endpoint-handover-smoke.mjs - # Why: workspace removal overlaps graceful renderer teardown with the - # forced main-process sweep; exercise that exact pair against real ConPTY. + # Exercise workspace removal’s renderer teardown plus forced main sweep against real ConPTY. - name: Windows daemon workspace-close repro if: runner.os == 'Windows' run: node config/scripts/windows-daemon-workspace-close-repro.mjs diff --git a/.github/workflows/daily-mac-build.yml b/.github/workflows/daily-mac-build.yml index 3648edda1a9..38e20094ad6 100644 --- a/.github/workflows/daily-mac-build.yml +++ b/.github/workflows/daily-mac-build.yml @@ -1,49 +1,6 @@ name: Daily macOS Dev Build -# Why: gives developers a signed macOS build of main once a day that the in-app -# updater can install directly, without waiting for an RC cut — less noise than -# hourly. -# -# Schedule is a single UTC cron (GH Actions has no timezone-aware schedules). -# 18:15 UTC is late morning Pacific year-round (10:15am PST / 11:15am PDT). Minute -# 15 avoids stacking with hourly-mac-build, which fires at minute 0 every hour. -# -# Deliberately narrow scope: -# - macOS only. Other platforms keep using RC/stable. -# - No tests or lint on the build job. After a live publish we fire-and-forget -# the full E2E workflow at the cut SHA (same detached dispatch as -# release-cut). A red suite must not fail or delay the daily. -# - PR CI and release-cut remain the gates that matter. -# - Signed AND notarized, exactly like a release. The notary round trip is the -# one slow step kept: macOS anchors a notarized app's TCC grants on identifier -# + team rather than on its cdhash, so those grants survive an update. Without -# a ticket every daily reads as a new client and silently loses file access -# under Documents/Desktop/Downloads. -# -# Artifacts publish to stablyai/orca-daily, never to stablyai/orca: the main -# repo's releases atom feed exposes only its 10 newest entries, so high-volume -# dev tags would evict every stable/RC entry and break updates for real users. -# Separate from orca-hourly so someone riding main's hourlies never sees the -# sparse daily series mixed into that list. -# -# GITHUB_TOKEN is scoped to this repo and cannot publish there, so writes use the -# same GitHub App as hourly (installed on orca-daily with Contents: Read and -# write). Provision the App secrets with -# `bash config/scripts/setup-hourly-release-token.sh`, then grant the App access -# to orca-daily with `bash config/scripts/setup-daily-release-repo.sh`: -# HOURLY_RELEASE_APP_ID the App's numeric id (shared with hourly/adhoc) -# HOURLY_RELEASE_APP_PRIVATE_KEY the App's .pem private key -# -# Installation tokens live one hour, which is why this mints three times. Install -# and build need no token at all, and notarization can hold the publish step for -# tens of minutes; minting again once the build is done starts the clock at the -# first call that actually uses it rather than burning a third of it on -# `pnpm install`. The upload step's own retry budget (2x45m) can outlive that -# second token, so a third is minted after it for verify/publish/prune/cleanup — -# without it a stuck notary run would strand a draft that cleanup gets a 401 on. -# The release stays an unpublished draft until the manifest check passes, so the -# worst case is still an invisible draft — and the build-number query counts -# drafts, so it holds its number and the next run does not reuse it. +# Publish daily main builds to orca-daily, separate from hourly and stable/RC feeds. Sign/notarize for update-preserved macOS TCC grants; PR/release pipelines own blocking validation, with E2E detached after publish. Cron 18:15 UTC means 10:15 PST/11:15 PDT, offset from hourly. Use the shared hourly App with write access to orca-daily (setup-hourly-release-token.sh/setup-daily-release-repo.sh). Mint tokens again after build and upload because 2×45min retry can exceed their 1h life. Keep releases draft until manifests verify; drafts retain build numbers. on: schedule: @@ -112,10 +69,7 @@ jobs: head_sha: ${{ steps.freshness.outputs.head_sha }} published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }} runs-on: blacksmith-6vcpu-macos-15 - # Why 150: it must exceed the worst case the retry budgets below can produce - # (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or - # the job is killed mid-retry and no cleanup step runs at all. A typical run - # is far shorter — this is the notary queue's tail, not its median. + # Allow 150min for install 3×10 + publish 2×45 + ~25 setup/build/verify, so outer timeout cannot strand cleanup mid-retry. timeout-minutes: 150 env: NODE_OPTIONS: --max-old-space-size=4096 @@ -126,10 +80,7 @@ jobs: ref: ${{ needs.resolve-main.outputs.head_sha }} # Version helpers only read HEAD; published versions come from git tags. fetch-depth: 1 - # Why: this job only reads stablyai/orca and never pushes; every write - # goes to the daily repo through a minted App token passed by env. - # Not persisting the checkout credential shrinks the blast radius if a - # build step is compromised (zizmor: artipacked). + # Read this repo without persisted credentials; write orca-daily through env-passed App token (zizmor: artipacked). persist-credentials: false - name: Mint daily repo token @@ -141,8 +92,7 @@ jobs: owner: stablyai repositories: orca-daily - # Why: main is often idle. Rebuilding an unchanged commit burns a runner - # hour and adds a redundant tag to the retention window. + # Skip unchanged main to avoid duplicate builds/tags and runner cost. - name: Check whether main moved since the last daily id: freshness shell: bash @@ -158,9 +108,7 @@ jobs: echo "Forced dispatch; building $head_sha." exit 0 fi - # The previous daily records its source commit in the release body. - # Drafts are excluded: an unpublished leftover never shipped, so treating - # it as "the last build" would skip a build that never actually happened. + # Drafts never shipped; exclude them when comparing the previous daily’s source commit. last_body="$(gh release list --repo "$DAILY_REPO" --limit 20 --json tagName,isDraft \ --jq 'map(select(.isDraft | not)) | .[0].tagName // empty' 2>/dev/null || true)" if [[ -z "$last_body" ]]; then @@ -195,8 +143,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Why: packaging bundles the pinned Node into the macOS terminal host. Keyed on the pin alone - # with no restore-keys, so a pin bump starts empty instead of carrying old archives forward. + # Cache bundled host Node by exact pin without restore keys. - name: Locate the pinned Node runtime cache if: steps.freshness.outputs.should_build == 'true' shell: bash @@ -220,8 +167,7 @@ jobs: restore-keys: | electron-builder-mac- - # Why both CPUs: the mac config packages x64 and arm64 from this arm64 - # runner, so the install must carry both variants of the native optional deps. + # Install x64 and arm64 optional natives for both packaged macOS targets. - name: Install dependencies if: steps.freshness.outputs.should_build == 'true' uses: nick-fields/retry@v4 @@ -231,8 +177,7 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 - # Why here: electron-builder's beforePack requires out/mobile-web, and the bundle - # build resolves React Native and Expo from mobile/node_modules. + # Build required out/mobile-web using mobile’s React Native/Expo install. - uses: ./.github/actions/install-mobile-dependencies if: steps.freshness.outputs.should_build == 'true' @@ -257,21 +202,12 @@ jobs: MAIN_REPO_TOKEN: ${{ github.token }} run: | set -euo pipefail - # Existing titles, which carry the build number this series continues - # from. The script picks the number, because it restarts per base version - # and only the script knows which base this build resolved to. - # - # Why drafts count here but not in the freshness check: that check asks - # "did this commit ship", where a draft is a no. This one asks "is the - # number free", where a stranded draft still holds one. + # The script continues numbering for its resolved base version; stranded drafts still reserve numbers, + # although freshness checks exclude them because they never shipped. names="$(gh release list --repo "$DAILY_REPO" --limit 200 --json name \ --jq '.[].name // empty')" - # Why git tags, not GitHub releases: unpublishing a buggy cut deletes the - # GitHub release and leaves the tag. That dragged hourlies backwards so - # electron-updater stopped offering them; dailies would do the same. A - # separate token because GH_TOKEN above is the App's, scoped to the - # daily repo. Empty on failure — the script then falls back to - # package.json, which is stale but never wrong enough to fail a build. + # Tags survive release deletion, preventing updater version regression. Use a separate token because + # GH_TOKEN’s App grant is scoped to the channel repo; lookup failure falls back to package.json. main_tags="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \ "repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \ --jq '.[].ref | sub("^refs/tags/"; "")' || true)" @@ -285,11 +221,7 @@ jobs: node config/scripts/daily-build-version.mjs \ >"$RUNNER_TEMP/daily-identity.txt" grep -E '^(version|build_number)=' "$RUNNER_TEMP/daily-identity.txt" - # Why check rather than trust: the checkout above pins the resolved main commit, but a - # workflow_dispatch runs this file from whatever branch was dispatched. A - # branch that edits this step while main still has the old script yields - # an empty name and an untitled release — silent, and only visible once - # someone opens the releases page. Fail here instead. + # Dispatch YAML can be newer than the pinned source’s version script; reject missing output before publishing an untitled release. if ! grep -q '^name=' "$RUNNER_TEMP/daily-identity.txt"; then echo "::error::daily-build-version.mjs emitted no release name; this workflow and main's copy of the script are out of sync." exit 1 @@ -301,9 +233,7 @@ jobs: run: pnpm build:release env: NODE_OPTIONS: --max-old-space-size=4096 - # Why: daily builds are not an official channel — telemetry's transport - # gate accepts only 'stable' or 'rc', so leaving this unset keeps them - # silent, which is correct for unvetted dev artifacts. + # Leave dev identity unset so official-only stable/rc telemetry stays silent. ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token # After the app build so nothing that cleans out/ can drop it; electron-builder ships it. @@ -314,11 +244,7 @@ jobs: name: orcad-template path: out/orcad-template - # Why a second mint: everything from here on writes to the daily repo, and - # the notary round trip inside the publish step can be tens of minutes. The - # token minted at the top has already spent install + build of its one hour - # on steps that never touched it; restarting the clock here gives the slow - # part the full budget. + # Refresh the one-hour token after read-only install/build, before long publish/notary work. - name: Re-mint daily repo token for publish id: app_token_publish if: steps.freshness.outputs.should_build == 'true' @@ -340,12 +266,9 @@ jobs: SHA: ${{ steps.freshness.outputs.head_sha }} run: | set -euo pipefail - # Kept at 12 even though the title shows 7: the freshness check above - # parses this back out of the body to decide whether main has moved. + # Freshness parses the 12-character source SHA from the body even though titles show seven. short_sha="${SHA:0:12}" - # Why a file rather than an inline string: the publish step re-asserts - # this same body, and the freshness check only works if the two agree - # exactly. One source, written once, read twice. + # Write one body file for creation and reassertion; freshness parsing requires exact agreement. notes_file="$RUNNER_TEMP/daily-release-notes.md" cat >"$notes_file" <- (failure() || cancelled()) && steps.release.outputs.tag != '' && steps.publish_live.outcome != 'success' shell: bash env: - # Fall back to the publish token: if the re-mint itself is what failed, - # the older token is the only one left and may still have time on it. + # If token refresh fails, let cleanup try the older token while it may still be valid. GH_TOKEN: ${{ steps.app_token_final.outputs.token || steps.app_token_publish.outputs.token }} TAG: ${{ steps.release.outputs.tag }} run: | @@ -507,33 +388,18 @@ jobs: echo "::warning::Could not discard draft $TAG; remove it manually." - name: Prune old daily releases - # Only after a live publish: $TAG is then a non-draft we must not delete, - # and failed runs should not reshuffle retention around a draft that the - # failure path is about to discard. + # Prune only after publication; failed runs leave draft disposal to cleanup. if: steps.publish_live.outcome == 'success' shell: bash env: GH_TOKEN: ${{ steps.app_token_final.outputs.token }} - # Protect the tag this run just shipped; at the retain cap, a bad sort - # can otherwise mark the newest release as stale and delete it. + # Never prune this run’s just-published tag at the retention cap. TAG: ${{ steps.release.outputs.tag }} run: | set -euo pipefail - # Why: --cleanup-tag so pruning does not leave orphan tags behind that - # keep showing up in tag lists with no release or assets attached. - # Drafts are excluded so retention counts shipped builds only; a stale - # draft is handled by the failure path, not by the retention window. - # - # Sort by publishedAt (not createdAt). Many non-draft dailies can share - # one createdAt (bulk import / re-create), so createdAt ranking is - # unstable. publishedAt is real recency; tagName (...YYYYMMDDHHMM) is - # the deterministic tie-break. - # - # Why force $TAG to the front before slicing: a hard retain-window seat - # for this run's release. Dropping $TAG from the list *before* the slice - # would permanently keep retain+1 releases; skipping it only in the - # delete loop would under-prune when the sort is still wrong. Partition - # keeps relative order of every other tag. + # Delete pruned tags too; exclude drafts owned by failure cleanup. Rank by publishedAt/tagName: + # createdAt ties (bulk import/recreation) can evict the just-published release. Partition $TAG to + # the front before slicing so it owns exactly one retained seat without under-pruning or keeping retain+1. jq_filter='map(select(.isDraft | not)) | sort_by(.publishedAt // "", .tagName) | reverse' if [[ -n "${TAG:-}" ]]; then jq_filter+=" | (map(select(.tagName == \"${TAG//\"/\\\"}\")) + map(select(.tagName != \"${TAG//\"/\\\"}\")))" @@ -557,13 +423,7 @@ jobs: echo "::warning::Could not prune $tag" done <<<"$stale" - # Why detached dispatch, not a reusable workflow in this graph: the full suite - # is currently red on main. Inlining it here would fail the daily after the - # signed build already published. Same pattern as release-cut's post-release-e2e. - # - # Why --ref main plus the SHA input: daily tags live on orca-daily, not this - # repo, so there is no cut tag for `gh workflow run --ref`. The workflow file - # comes from main; checkout uses the exact commit this daily built. + # Dispatch E2E separately so main’s red full suite cannot fail an already-published daily. Use main’s workflow plus exact built SHA because daily tags live in another repo. post-daily-e2e: needs: build-daily-mac if: ${{ needs.build-daily-mac.outputs.published == 'true' && needs.build-daily-mac.outputs.head_sha != '' }} @@ -588,21 +448,10 @@ jobs: done echo "::warning::Failed to dispatch post-daily E2E for $SHA after 3 attempts." - # Why this runs after the mac leg rather than beside it: the tag and version - # are computed inside that job, so until it has run nothing else can name the - # release to upload into. The cost is small enough not to matter — the Windows - # leg measures ~7.5 min (install 2m45, build 35s, NSIS package 3m) against a - # ~9.5 min mac run, which keeps a daily run far inside its interval. - # - # Why `./` rather than a pinned `@main`: `uses:` resolves against the ref this - # file itself came from, which for an ordinary dispatch is main. Someone who - # deliberately points the Actions "Use workflow from" picker at a branch - # already gets that branch's copy of this entire file, so this follows the same - # rule instead of inventing a second one. + # Wait for mac release identity before Windows uploads; local ./ workflow follows the dispatch ref consistently. build-daily-win: needs: build-daily-mac - # Only once the mac release is actually live: there is no release to upload - # into otherwise, and the Windows workflow refuses to create one. + # Start Windows publication only after the mac release is live; its workflow refuses to create a missing release. if: needs.build-daily-mac.outputs.published == 'true' permissions: contents: read diff --git a/.github/workflows/dev-channel-win-build.yml b/.github/workflows/dev-channel-win-build.yml index 3872161f7b4..1efa6aa8adc 100644 --- a/.github/workflows/dev-channel-win-build.yml +++ b/.github/workflows/dev-channel-win-build.yml @@ -1,41 +1,9 @@ name: Dev Channel Windows Build -# Why its own file rather than steps inlined into hourly/daily/adhoc: one copy of -# the Windows leg instead of three, and it stays dispatchable on its own so a -# Windows artifact can be rebuilt for an existing tag without paying for the mac -# leg's packaging and notarization again. -# -# Adhoc can stage Windows alongside macOS, then publish only after macOS succeeds. -# Hourly, daily, and manual dispatches keep packaging and publishing together. -# -# Why unsigned: Windows release installers are signed by SignPath *after* -# packaging, and release-cut budgets 1h + 4h for those approval waits. That does -# not fit an hourly cadence and it does not fit "dispatch an adhoc build and go -# get coffee". So dev-channel Windows builds ship unsigned, which has one real -# consequence, handled in `src/shared/release-channel.ts`: -# -# electron-updater Authenticode-verifies every installer it downloads against -# the publisherName baked into the *installed* app's app-update.yml. Stable and -# RC carry 'SignPath Foundation', so they reject an unsigned dev installer and -# no future build can fix the copies already installed. Dev builds omit the -# name (config/electron-builder.config.cjs), so verification is skipped there. -# -# Net effect: the way *into* a dev channel on Windows is a one-time manual -# installer run. Every way out — to another dev build, or back to Stable — works -# through the in-app updater. The picker offers a download for exactly that jump. -# -# Mac workflows call this directly or stage artifacts for a later publish job. -# It is also dispatchable to rebuild Windows for an existing tag: -# -# gh workflow run dev-channel-win-build.yml --ref main \ -# -f channel=adhoc -f tag=v1.4.178-adhoc.20260819010203 \ -# -f ref= -f version=1.4.178-adhoc.20260819010203 +# Share one independently dispatchable Windows dev build across hourly/daily/adhoc. Adhoc may stage beside Mac and publish after its success. Dev installers are unsigned because SignPath approval waits exceed this cadence. Installed stable/RC publisherName rejects them, requiring one manual entry install; dev builds omit publisherName so later in-app channel changes work. Manual rebuild: dispatch main with channel/tag/ref/version for an existing release. on: - # Why the inputs are duplicated: workflow_call does not accept `type: choice`, - # and workflow_dispatch wants it so the Actions UI offers a menu instead of a - # free-text box. The channel allowlist below is what actually enforces the set, - # since a workflow_call caller can pass any string. + # Duplicate inputs because workflow_call lacks choice UI; validate its free-string channels with the allowlist. workflow_call: inputs: channel: @@ -104,18 +72,14 @@ permissions: contents: read concurrency: - # Keyed on the tag: re-dispatching the same tag must not race two uploads into - # one release, but two different channels (or two adhoc branches) are the - # ordinary case and must not wait on each other. + # Serialize per tag to prevent competing uploads while independent channels/branches proceed. group: dev-channel-win-build-${{ inputs.tag }} cancel-in-progress: false jobs: build-win: if: github.repository == 'stablyai/orca' - # Why the same environment as the mac workflows: this needs the App token - # that can write to the dev-channel repos, and it should be reachable from - # exactly the same place those secrets already live. + # Use mac workflows’ same environment for cross-repo App secrets. environment: adhoc-mac-build # The fast image has VS Build Tools 2022; signed release provenance stays separate. runs-on: ${{ inputs.fast_release && 'blacksmith-8vcpu-windows-2025' || 'windows-2022' }} @@ -132,10 +96,7 @@ jobs: VERSION: ${{ inputs.version }} steps: - # Why vet before checkout: everything after this runs the checked-out code - # with a token that can write to a release repo. The mac leg already vetted - # the ref it resolved, but this workflow is dispatchable on its own, so it - # re-derives the same guarantee rather than trusting its caller. + # Re-vet source before checkout because standalone dispatch also exposes a release-write token. - name: Vet the requested inputs id: vetted shell: bash @@ -144,8 +105,7 @@ jobs: REPO_URL: https://github.com/${{ github.repository }} run: | set -euo pipefail - # workflow_call takes channel as a free-text string, so the set is - # enforced here rather than by the input type. + # workflow_call inputs are free text; validate the channel set here. case "$CHANNEL" in hourly|daily|adhoc) ;; *) @@ -157,24 +117,18 @@ jobs: echo "::error::ref must be a full 40-character commit SHA, got '$REQUESTED_SHA'. The dispatching workflow passes the commit it resolved." exit 1 fi - # The tag must name the version being packaged, or the artifacts would - # land in a release describing a different build. + # Bind the tag to the packaged version so assets cannot describe a different build. if [[ "$TAG" != "v$VERSION" ]]; then echo "::error::tag '$TAG' does not match version '$VERSION'." exit 1 fi - # And the version must carry the channel's own prerelease identifier, - # so an hourly artifact can never be uploaded into an adhoc release. + # Require the channel’s prerelease identifier so hourly assets cannot enter adhoc releases. if [[ "$VERSION" != *"-$CHANNEL."* ]]; then echo "::error::version '$VERSION' is not a $CHANNEL version." exit 1 fi - # Reachability is the trust test: GitHub serves PR-only commits by SHA, - # so resolving the object is not proof a branch or tag of this repo - # reaches it. Bare + tree:0 keeps this to the commit graph; reftable - # because branches that differ only in casing cannot both be stored by - # the files backend on a case-insensitive runner disk, which fails the - # entire fetch rather than the one ref. + # Trust requires branch/tag reachability; GitHub serves PR-only SHAs too. Bare/tree:0 limits fetch to + # the commit graph; reftable avoids case-only refs failing the entire fetch on case-insensitive disks. scratch="$RUNNER_TEMP/vet-requested-ref" git init -q --bare --ref-format=reftable "$scratch" git -C "$scratch" fetch -q --filter=tree:0 "$REPO_URL" '+refs/heads/*:refs/heads/*' '+refs/tags/*:refs/tags/*' @@ -200,9 +154,7 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ inputs.ref }} - # This job only reads stablyai/orca and never pushes; every write goes - # to the dev-channel repo through a minted App token passed by env - # (zizmor: artipacked). + # Read this repo without persisted credentials; writes use the dev-repo env App token (zizmor: artipacked). persist-credentials: false # Why: a relay-era ref (before #27215) needs the Windows relay addon this build no longer compiles. @@ -214,11 +166,7 @@ jobs: exit 1 fi - # Why a guard and not just a build: the workflow file comes from the - # dispatch ref, but the packaging config comes from the *built* commit. A - # branch cut before Windows dev builds landed has a config that ignores - # ORCA_WIN_*, which would resolve publish.repo to the main repo. Say that - # in one sentence here rather than failing deep inside electron-builder. + # Reject old packaging configs that ignore ORCA_WIN_* and would publish to the main repo. - name: Resolve the dev-channel packaging identity shell: bash run: | @@ -247,8 +195,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Caches the Electron binary and electron-builder's tool downloads (nsis, - # winCodeSign). Same key shape as release-cut's Windows leg. + # Cache Electron/NSIS/winCodeSign tools with release-cut’s key shape. - name: Cache electron-builder downloads uses: actions/cache@v5 with: @@ -259,11 +206,7 @@ jobs: restore-keys: | electron-builder-win- - # Why retried: pnpm install triggers electron's postinstall, which pulls the - # Electron binary from GitHub release assets, and that CDN returns transient - # 504s often enough to lose a build to it. - # Why host-only: this job packages only for its own runner OS and - # architecture, so the default host-scoped install is deliberate. + # Retry transient Electron CDN 504s during install; keep host-scoped natives for this runner’s only target. - name: Install dependencies uses: nick-fields/retry@v4 with: @@ -272,13 +215,11 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile - # Why here: electron-builder's beforePack requires out/mobile-web, and the bundle - # build resolves React Native and Expo from mobile/node_modules. + # Build required out/mobile-web from mobile’s React Native/Expo install. - uses: ./.github/actions/install-mobile-dependencies if: inputs.javascript_source_sha == '' - # Why the packaging check runs before the 20-minute build: it only needs - # node_modules, and a stale config should cost seconds rather than a build. + # Validate packaging config after install, before the expensive build. - name: Verify dev-channel packaging identity shell: bash run: node config/scripts/verify-dev-channel-packaging.mjs --channel="$CHANNEL" --platform=win32 @@ -302,9 +243,7 @@ jobs: env: ORCA_RELEASE_JAVASCRIPT_SOURCE_SHA: ${{ inputs.javascript_source_sha }} ORCA_RELEASE_JAVASCRIPT_BUILD_IDENTITY: dev - # Why unset ORCA_BUILD_IDENTITY: telemetry's transport gate accepts only - # 'stable' or 'rc', so leaving it unset keeps dev builds silent — which - # is correct for unvetted artifacts. Same as the mac dev channels. + # Unset dev build identity so official stable/rc telemetry stays silent. ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token # Inline so refs predating the wait can still use the trusted dispatch workflow. @@ -340,8 +279,7 @@ jobs: name: orcad-template path: out/orcad-template - # Why the token is minted here and not at the top: installation tokens live - # one hour and nothing before this point writes anything. + # Mint the one-hour write token after read-only setup/build. - name: Mint dev channel repo token if: ${{ !inputs.stage_only }} id: app_token @@ -352,10 +290,7 @@ jobs: owner: stablyai repositories: orca-${{ inputs.channel }} - # Why: electron-builder's publisher creates a release when it cannot find - # the tag ("publish: always"). If the mac leg failed and discarded its draft - # while this was building, that would mint a fresh, untitled, Windows-only - # release. Check first and fail instead. + # Require existing release: publish:always would recreate a discarded mac draft as an untitled Windows-only release. - name: Confirm the target release still exists if: ${{ !inputs.stage_only }} shell: bash @@ -382,16 +317,10 @@ jobs: ORCA_BUILD_COMMIT: ${{ inputs.ref }} # beforePack and afterPack fail the package when the template is absent. ORCA_REQUIRE_ORCAD_TEMPLATE: ${{ inputs.orcad_template && '1' || '' }} - # Why: electron-publish refuses to upload into a release published more - # than two hours ago (gitHubPublisher.getOrCreateRelease). The mac leg - # publishes the draft live as soon as *it* finishes, so a slow notary - # queue plus a slow Windows build can cross that line and silently drop - # every Windows asset. This is the documented escape hatch. + # Allow upload beyond publisher’s 2h release-age guard; slow mac notarization/Windows build can otherwise silently drop every asset. EP_GH_IGNORE_TIME: 'true' - # Why: the updater resolves a tag, then fetches latest.yml from it. A - # release carrying the installer but not the manifest is one the picker - # offers and the update 404s on, so assert both. + # Require installer and latest.yml, preventing picker-visible releases with missing updater manifests. - name: Verify Windows update manifest published if: ${{ !inputs.stage_only }} shell: bash diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 161f5009e5a..8f7ae5d6df0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -105,8 +105,7 @@ jobs: tag="$INPUT_TAG" [[ "$EVENT_NAME" == "release" ]] && tag="$RELEASE_TAG" - # Match the stable desktop format used by release-policy.yml. This - # intentionally rejects mobile-* and all -rc.* tags. + # Use release-policy’s stable desktop tag shape, excluding mobile-* and RC tags. stable_tag=false [[ "$tag" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] && stable_tag=true @@ -147,9 +146,7 @@ jobs: fi fi - # Resolve the tag to an immutable commit before handing it to the - # deployment job. This prevents a force-moved tag from changing the - # source between authorization and checkout. + # Pin the immutable commit before deployment so force-moving a tag cannot swap authorized source. tag_ref_json='' for attempt in 1 2 3; do if tag_ref_json="$(gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$tag" 2>/dev/null)"; then @@ -173,9 +170,7 @@ jobs: fi echo "commit_sha=$commit_sha" >> "$GITHUB_OUTPUT" - # Stable tags cut before the docs package was merged are valid Orca - # releases, but cannot produce a docs deployment. Skip them before - # requesting the protected production environment. + # Old stable tags may predate docs; skip before requesting the protected production environment. docs_source_available=false for attempt in 1 2 3; do if gh api "repos/$GITHUB_REPOSITORY/contents/docs/site/package.json?ref=$commit_sha" >/dev/null 2>&1; then diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1ed90002865..ce9541a9c7d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -70,6 +70,8 @@ jobs: if: inputs.test_files == '' || github.event_name != 'pull_request' || inputs.needs_build runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 15 + outputs: + changed_e2e_shards: ${{ steps.changed-e2e-shards.outputs.shards }} env: ORCA_BACKGROUND_LAUNCH: '1' @@ -79,6 +81,19 @@ jobs: with: ref: ${{ inputs.ref || github.ref }} + - name: Plan changed E2E allocation + id: changed-e2e-shards + env: + TEST_FILES_JSON: ${{ inputs.test_files }} + run: | + node --input-type=module <<'NODE' + import { appendFileSync, existsSync } from 'node:fs' + const modulePath = './config/scripts/ci-e2e-job-selection.mjs' + const selection = existsSync(modulePath) ? await import(modulePath) : {} + const shards = selection.generalE2eShards?.(process.env.TEST_FILES_JSON) ?? ['1/1'] + appendFileSync(process.env.GITHUB_OUTPUT, `shards=${JSON.stringify(shards)}\n`) + NODE + - name: Install native build tools env: ORCA_E2E_APT_PACKAGES: build-essential python3 @@ -203,9 +218,7 @@ jobs: fail-fast: false matrix: include: - # Fourteen scheduled runs averaged 24.6 minutes per shard; shards 4 - # and 9 repeatedly hit the 30-minute cap. A 12-way trial still left - # one 30-minute shard, so 14 gives the suite enough failure headroom. + # Use 14 shards: scheduled average 24.6min and repeated 30min caps remained even in a 12-way trial. - shard: '1/14' shard_name: 1-of-14 - shard: '2/14' @@ -260,14 +273,7 @@ jobs: name: e2e-build-out${{ inputs.artifact_suffix }} path: out/ - # Why: the Electron suite is wall-clock constrained on OSS runners, but - # multiple Electron apps on one Xvfb VM contend on git/Chromium resources. - # Sharding keeps each VM at one Playwright worker while splitting the - # headless suite across separate runners. - # SKIP_BUILD makes Playwright globalSetup reuse the single build job's - # artifact instead of starting five concurrent electron-vite builds. - # ORCA_E2E_FORWARD_APP_LOGS keeps startup failures visible when Electron - # launches but never creates a BrowserWindow. + # Shard one worker per VM to avoid Electron/git/Xvfb contention. Reuse one SKIP_BUILD artifact and forward app logs for pre-window failures. - name: Balance E2E shard from timing evidence env: ORCA_BACKGROUND_LAUNCH: '1' @@ -308,9 +314,7 @@ jobs: if: matrix.shard == '1/14' run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm exec playwright test tests/e2e/worktree-switch-first-paint.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1 - # Why: Playwright retains traces/screenshots only on failure. Uploading - # them as an artifact makes post-mortem debugging on CI possible without - # re-running locally. + # Upload failure traces/screenshots so CI failures are diagnosable without reproduction. - name: Upload Playwright traces if: failure() || cancelled() uses: actions/upload-artifact@v7 @@ -321,11 +325,15 @@ jobs: if-no-files-found: ignore changed-e2e: - name: changed e2e specs + name: changed e2e specs (${{ matrix.shard }}) needs: [build] if: inputs.test_files != '' && (github.event_name != 'pull_request' || inputs.run_changed_e2e) runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON(needs.build.outputs.changed_e2e_shards || '["1/1"]') }} steps: - name: Checkout @@ -334,8 +342,7 @@ jobs: ref: ${{ inputs.ref || github.ref }} - name: Install native build and headless UI tools - # Why ripgrep: Quick Open's bounded host-side search requires rg instead of an - # unbounded inventory fallback; the paired fixture exercises that real boundary. + # Install rg for bounded Quick Open searches exercised by the paired fixture. env: *e2e_tool_packages run: *install_e2e_tools @@ -353,6 +360,7 @@ jobs: env: TEST_FILES_JSON: ${{ inputs.test_files }} LAYOUT_OWNER: ${{ inputs.layout_owner }} + SHARD: ${{ matrix.shard }} run: | if [ -f config/scripts/ci-e2e-job-selection.mjs ]; then printf '%s\n' "$TEST_FILES_JSON" | node config/scripts/ci-e2e-job-selection.mjs > "$RUNNER_TEMP/general-e2e-specs" @@ -385,8 +393,28 @@ jobs: if grep -l '@headful' "${TEST_FILES[@]}" >/dev/null; then E2E_PROJECT_ARGS+=(--project=electron-headful) fi + E2E_SELECTION_ARGS=(--shard="$SHARD") + if [ -f config/scripts/ci-e2e-shard-plan.mjs ]; then + mkdir -p ci-shards-changed + pnpm exec playwright test "${TEST_FILES[@]}" --config tests/playwright.config.ts --project=electron-headless "${E2E_PROJECT_ARGS[@]}" --list --reporter=json > ci-shards-changed/discovery.json + export ORCA_SHARD_SOURCE_SHA="$(git rev-parse HEAD)" + node config/scripts/ci-e2e-shard-plan.mjs ci-shards-changed/discovery.json "$SHARD" ci-shards-changed + pnpm exec playwright test "${TEST_FILES[@]}" --config tests/playwright.config.ts --project=electron-headless "${E2E_PROJECT_ARGS[@]}" --test-list=ci-shards-changed/selected.txt --list --reporter=json > ci-shards-changed/selected-discovery.json + node config/scripts/ci-e2e-shard-plan.mjs --verify ci-shards-changed/assignment.json ci-shards-changed/selected-discovery.json + E2E_SELECTION_ARGS=(--test-list=ci-shards-changed/selected.txt) + fi xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env "${E2E_ENV[@]}" \ - pnpm run test:e2e "${TEST_FILES[@]}" --workers=1 "${E2E_PROJECT_ARGS[@]}" + pnpm run test:e2e "${TEST_FILES[@]}" "${E2E_SELECTION_ARGS[@]}" --workers=1 "${E2E_PROJECT_ARGS[@]}" + + - name: Upload changed E2E shard assignment + if: always() + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: changed-e2e-shard-${{ strategy.job-index }}${{ inputs.artifact_suffix }}-attempt-${{ github.run_attempt }} + path: ci-shards-changed/ + retention-days: 14 + if-no-files-found: warn - name: Summarize layout save divergence if: always() && inputs.layout_owner == 'main' @@ -407,7 +435,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v7 with: - name: layout-divergence-changed${{ inputs.artifact_suffix }}-attempt-${{ github.run_attempt }} + name: layout-divergence-changed-${{ strategy.job-index }}${{ inputs.artifact_suffix }}-attempt-${{ github.run_attempt }} path: layout-divergence/ retention-days: 14 if-no-files-found: warn @@ -416,7 +444,7 @@ jobs: if: failure() || cancelled() uses: actions/upload-artifact@v7 with: - name: playwright-traces-changed${{ inputs.artifact_suffix }} + name: playwright-traces-changed-${{ strategy.job-index }}${{ inputs.artifact_suffix }} path: test-results/ retention-days: 7 if-no-files-found: ignore @@ -475,8 +503,7 @@ jobs: slot="$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)" pnpm build:orcad-prebuilds pnpm build:orcad-prebuilds --require-slots "$slot" - # The headless oracle's orcad host must hold the same layout owner as the Electron host, - # and so must the template `orca serve` materializes its server from. + # The orcad host and its materialized template must use the Electron host’s layout owner. if [ -n "$LAYOUT_OWNER" ]; then export ORCA_LAYOUT_OWNER="$LAYOUT_OWNER" pnpm build:orcad diff --git a/.github/workflows/homebrew-bump.yml b/.github/workflows/homebrew-bump.yml index a5f9b352f6e..717866794f8 100644 --- a/.github/workflows/homebrew-bump.yml +++ b/.github/workflows/homebrew-bump.yml @@ -1,22 +1,6 @@ name: Homebrew Cask Bump -# Why: keep the stablyai/homebrew-orca tap in sync with GitHub Releases. -# Triggers: -# - workflow_call → release-cut.yml finished publishing a stable tag -# - workflow_dispatch → manual backfill for a specific tag -# -# The job computes the sha256 of orca-macos-arm64.dmg + orca-macos-x64.dmg -# from the release, templates the channel-specific cask in this repo with the -# new version and hashes, then PRs the result into stablyai/homebrew-orca. -# Stable tags update Casks/orca.rb; RC tags update Casks/orca@rc.rb. -# -# Why no `release.published` trigger: other products shipped from this repo -# (e.g. mobile under `mobile-v*`) also publish GitHub Releases. A blanket -# `release.published` trigger would fire this workflow for any of them and -# attempt to download `orca-macos-*.dmg` from a non-desktop release, at -# best 404ing and at worst (if the filter were weaker) rewriting the cask -# to a non-desktop version. The tap is only ever bumped by the desktop -# release pipeline calling this workflow directly. +# Update desktop stable/RC casks in stablyai/homebrew-orca with release DMG hashes through release-cut workflow_call or manual backfill. Avoid release.published because mobile/non-desktop releases lack those DMGs and must never rewrite casks. on: workflow_call: @@ -132,10 +116,7 @@ jobs: print(src) PY - # Why: reuse the existing buf0-bot GitHub App instead of minting a new - # PAT. The app is installed org-wide, so it has access to - # stablyai/homebrew-orca automatically. GITHUB_TOKEN cannot push - # cross-repo; a short-lived installation token can. + # Use org-installed buf0-bot’s short-lived App token for cross-repo pushes; GITHUB_TOKEN is repo-scoped. - name: Generate buf0-bot token id: app-token uses: actions/create-github-app-token@v3 @@ -166,8 +147,7 @@ jobs: cp "$CASK_PATH" "tap/$CASK_PATH" cd tap - # Why: commit author must match the buf0-bot app so commits are - # attributed to the bot in the tap's history / PR UI. + # Match the buf0-bot App’s commit author for tap history/PR attribution. git config user.name "buf0-bot[bot]" git config user.email "252831055+buf0-bot[bot]@users.noreply.github.com" @@ -182,8 +162,7 @@ jobs: git commit -m "$TITLE" git push -u origin "$branch" --force - # Why: repeated backfills can race an existing PR branch, so edit - # the PR title if creation reports that one already exists. + # Backfills can race an existing branch; update its PR title when creation finds one. gh pr create \ --title "$TITLE" \ --body "Automated bump of $CASK_PATH from stablyai/orca release $TAG." \ @@ -191,8 +170,5 @@ jobs: --head "$branch" \ || gh pr edit "$branch" --title "$TITLE" - # Why: squash-merge immediately rather than --auto. The tap has no - # required checks or branch protection, and `gh pr merge --auto` - # only activates when there's something to wait on — on a repo - # with no gates it silently no-ops, leaving the PR open forever. + # Squash immediately: this tap has no gates, so gh pr merge --auto silently no-ops. gh pr merge "$branch" --squash --delete-branch diff --git a/.github/workflows/hourly-mac-build.yml b/.github/workflows/hourly-mac-build.yml index 0d415b83e13..ca4ee37aa76 100644 --- a/.github/workflows/hourly-mac-build.yml +++ b/.github/workflows/hourly-mac-build.yml @@ -1,39 +1,6 @@ name: Hourly macOS Dev Build -# Why: gives developers a signed macOS build of main every hour that the in-app -# updater can install directly, without waiting for an RC cut. -# -# Deliberately narrow scope: -# - macOS only. Other platforms keep using RC/stable. -# - No tests, no lint, no e2e. This channel trades safety for latency; PR CI -# and release-cut remain the gates that matter. -# - Signed AND notarized, exactly like a release. The notary round trip is the -# one slow step kept: macOS anchors a notarized app's TCC grants on identifier -# + team rather than on its cdhash, so those grants survive an update. Without -# a ticket every hourly reads as a new client and silently loses file access -# under Documents/Desktop/Downloads — 24 times a day. -# -# Artifacts publish to stablyai/orca-hourly, never to stablyai/orca: the main -# repo's releases atom feed exposes only its 10 newest entries, so 24 hourly -# tags a day would evict every stable/RC entry and break updates for real users. -# -# GITHUB_TOKEN is scoped to this repo and cannot publish there, so writes use a -# GitHub App installed on orca-hourly with Contents: Read and write. Its private -# key does not expire, unlike a PAT — nothing here needs yearly rotation, and the -# credential belongs to the org rather than to whoever created it. -# -# Provision the two secrets with `bash config/scripts/setup-hourly-release-token.sh`: -# HOURLY_RELEASE_APP_ID the App's numeric id -# HOURLY_RELEASE_APP_PRIVATE_KEY the App's .pem private key -# -# Installation tokens live one hour, so the build job mints twice. Install and -# build need no token at all, and notarization can hold the publish step for tens -# of minutes; minting again once the build is done starts the clock at the first -# call that actually uses it rather than burning a third of it on `pnpm install`. -# A pathological retry can still outrun the second token, but the release is an -# unpublished draft until the manifest check passes, so the damage is a stranded -# invisible draft — and the build-number query counts drafts, so it holds its -# number and the next run does not reuse it. +# Publish signed/notarized main builds to orca-hourly; notarization preserves TCC grants across updates. PR/release gates own validation. Separate repo avoids hourly tags crowding stable/RC Atom entries. Provision shared HOURLY_RELEASE_APP_ID/private key with setup-hourly-release-token.sh and App write access there. Refresh one-hour token after read-only build; a retry may still outlive it, leaving only an invisible draft whose build number remains reserved until manifests validate. on: schedule: @@ -96,9 +63,7 @@ jobs: echo "Forced dispatch; building $head_sha." exit 0 fi - # The previous hourly records its source commit in the release body. - # Drafts are excluded: an unpublished leftover never shipped, so treating - # it as "the last build" would skip a build that never actually happened. + # Drafts never shipped; exclude them when comparing the previous hourly’s source commit. last_body="$(gh release list --repo "$HOURLY_REPO" --limit 20 --json tagName,isDraft \ --jq 'map(select(.isDraft | not)) | .[0].tagName // empty' 2>/dev/null || true)" if [[ -z "$last_body" ]]; then @@ -140,10 +105,7 @@ jobs: head_sha: ${{ needs.preflight.outputs.head_sha }} published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }} runs-on: blacksmith-6vcpu-macos-15 - # Why 150: it must exceed the worst case the retry budgets below can produce - # (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or - # the job is killed mid-retry and no cleanup step runs at all. A typical run - # is far shorter — this is the notary queue's tail, not its median. + # Allow 150min for install 3×10 + publish 2×45 + ~25 setup/build/verify; outer timeout otherwise interrupts cleanup. timeout-minutes: 150 env: NODE_OPTIONS: --max-old-space-size=4096 @@ -154,10 +116,7 @@ jobs: ref: ${{ needs.preflight.outputs.head_sha }} # Version helpers only read HEAD; published versions come from git tags. fetch-depth: 1 - # Why: this job only reads stablyai/orca and never pushes; every write - # goes to the hourly repo through a minted App token passed by env. - # Not persisting the checkout credential shrinks the blast radius if a - # build step is compromised (zizmor: artipacked). + # Read this repo without persisted credentials; write orca-hourly via env App token (zizmor: artipacked). persist-credentials: false - name: Mint hourly repo token @@ -182,8 +141,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Why: packaging bundles the pinned Node into the macOS terminal host. Keyed on the pin alone - # with no restore-keys, so a pin bump starts empty instead of carrying old archives forward. + # Cache bundled host Node by exact pin without restore keys. - name: Locate the pinned Node runtime cache shell: bash run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV" @@ -204,8 +162,7 @@ jobs: restore-keys: | electron-builder-mac- - # Why both CPUs: the mac config packages x64 and arm64 from this arm64 - # runner, so the install must carry both variants of the native optional deps. + # Install both x64/arm64 optional natives for both macOS packages. - name: Install dependencies uses: nick-fields/retry@v4 with: @@ -214,12 +171,10 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 - # Why here: electron-builder's beforePack requires out/mobile-web, and the bundle - # build resolves React Native and Expo from mobile/node_modules. + # Build required out/mobile-web from mobile’s React Native/Expo install. - uses: ./.github/actions/install-mobile-dependencies - # Why: signing is what makes an hourly installable over an existing Orca, so - # a missing cert must fail here rather than after a 20-minute build. + # Require cert before expensive build; signing enables installation over existing Orca. - name: Verify macOS signing environment run: node config/scripts/verify-macos-release-env.mjs env: @@ -237,30 +192,17 @@ jobs: MAIN_REPO_TOKEN: ${{ github.token }} run: | set -euo pipefail - # Existing titles, which carry the build number this series continues - # from. The script picks the number, because it restarts per base version - # and only the script knows which base this build resolved to. - # - # Why drafts count here but not in the freshness check: that check asks - # "did this commit ship", where a draft is a no. This one asks "is the - # number free", where a stranded draft still holds one. + # The script continues numbering for its resolved base version; stranded drafts still reserve numbers, + # although freshness checks exclude them because they never shipped. names="$(gh release list --repo "$HOURLY_REPO" --limit 200 --json name \ --jq '.[].name // empty')" - # Why git tags, not GitHub releases: unpublishing a buggy cut deletes the - # GitHub release and leaves the tag. On 2026-09-14 we deleted v1.4.202's - # release for a bug; hourlies had already climbed to 1.4.203, then - # `gh release list` fell back to v1.4.201 and the next hourlies shipped - # as 1.4.202-hourly — which electron-updater will not install over - # 1.4.203-hourly or over the still-tagged 1.4.202. A separate token - # because GH_TOKEN above is the App's, scoped to the hourly repo. Empty - # on failure — the script then falls back to package.json, which is - # stale but never wrong enough to fail a build. + # Tags survive release deletion, preventing updater version regression. Use a separate token because + # GH_TOKEN’s App grant is scoped to the channel repo; lookup failure falls back to package.json. + # 2026-09-14: deleting v1.4.202 made release-list fallback ship 1.4.202-hourly below installed 1.4.203-hourly. main_tags="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \ "repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \ --jq '.[].ref | sub("^refs/tags/"; "")' || true)" - # Already-shipped channel tags are a second floor: even if main's tag - # list is empty this run, a 1.4.203-hourly already out must not be - # followed by a 1.4.202-hourly. + # Already-shipped channel tags also floor versions: never follow installed 1.4.203-hourly with 1.4.202-hourly. channel_tags="$(gh release list --repo "$HOURLY_REPO" --limit 200 --json tagName \ --jq '.[].tagName' || true)" published="$main_tags"$'\n'"$channel_tags" @@ -269,11 +211,7 @@ jobs: node config/scripts/hourly-build-version.mjs \ >"$RUNNER_TEMP/hourly-identity.txt" grep -E '^(version|build_number)=' "$RUNNER_TEMP/hourly-identity.txt" - # Why check rather than trust: the checkout above pins the resolved main commit, but a - # workflow_dispatch runs this file from whatever branch was dispatched. A - # branch that edits this step while main still has the old script yields - # an empty name and an untitled release — silent, and only visible once - # someone opens the releases page. Fail here instead. + # Dispatch YAML can be newer than the pinned source’s version script; reject missing output before publishing an untitled release. if ! grep -q '^name=' "$RUNNER_TEMP/hourly-identity.txt"; then echo "::error::hourly-build-version.mjs emitted no release name; this workflow and main's copy of the script are out of sync." exit 1 @@ -284,9 +222,7 @@ jobs: run: pnpm build:release env: NODE_OPTIONS: --max-old-space-size=4096 - # Why: hourly builds are not an official channel — telemetry's transport - # gate accepts only 'stable' or 'rc', so leaving this unset keeps them - # silent, which is correct for unvetted dev artifacts. + # Unset dev identity so official-only stable/rc telemetry stays silent. ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token # After the app build so nothing that cleans out/ can drop it; electron-builder ships it. @@ -296,11 +232,7 @@ jobs: name: orcad-template path: out/orcad-template - # Why a second mint: everything from here on writes to the hourly repo, and - # the notary round trip inside the publish step can be tens of minutes. The - # token minted at the top has already spent install + build of its one hour - # on steps that never touched it; restarting the clock here gives the slow - # part the full budget. + # Refresh write token after build to give long notary/upload work a full hour. - name: Re-mint hourly repo token for publish id: app_token_publish uses: actions/create-github-app-token@v2 @@ -320,19 +252,10 @@ jobs: SHA: ${{ needs.preflight.outputs.head_sha }} run: | set -euo pipefail - # Kept at 12 even though the title shows 7: the freshness check above - # parses this back out of the body to decide whether main has moved. + # Freshness parses the 12-character source SHA from the body even though titles show seven. short_sha="${SHA:0:12}" - # Why create it up front: electron-builder then uploads into a known tag - # rather than inferring one from package.json. - # - # Why --draft: everything between here and the manifest check is a window - # where the release exists but has no installable assets. A draft is - # absent from the releases list and from listReleaseBuilds, so a job that - # dies in that window — including a hard kill by the job timeout, which - # runs no cleanup step at all — leaves something invisible rather than a - # tag the picker offers and the download 404s on. It is flipped live only - # after the manifest is verified. + # Precreate a known tag as draft so builder cannot infer it from package.json. + # Keep incomplete assets invisible until manifest verification, even if timeout kills the job without cleanup. gh release create "$TAG" \ --repo "$HOURLY_REPO" \ --title "$NAME" \ @@ -355,17 +278,13 @@ jobs: - name: Publish hourly macOS artifacts uses: nick-fields/retry@v4 with: - # Why 45 like the release pipeline: an attempt is pack + notarize + - # upload, and the notary queue is the unbounded part. Why 2 attempts and - # not 3: a missed hourly costs an hour, and the next cron picks the same - # commit up, so a third attempt buys less than it costs in runner time. + # Allow two 45min pack/notarize/upload attempts; next hourly picks the same commit rather than paying for a third. timeout_minutes: 45 max_attempts: 2 retry_wait_seconds: 30 command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_HOURLY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_HOURLY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi env: - # Why: electron-builder's github publisher targets the repo named in the - # config; the token must therefore carry write access to orca-hourly. + # Give publisher token write access to configured orca-hourly repo. GH_TOKEN: ${{ steps.app_token_publish.outputs.token }} ORCA_HOURLY_BUILD_VERSION: ${{ steps.hourly.outputs.version }} ORCA_BUILD_COMMIT: ${{ steps.hourly.outputs.commit }} @@ -373,17 +292,12 @@ jobs: ORCA_REQUIRE_ORCAD_TEMPLATE: '1' CSC_LINK: ${{ secrets.MAC_CERTS }} CSC_KEY_PASSWORD: ${{ secrets.MAC_CERTS_PASSWORD }} - # Why all three: electron-builder's notarize step authenticates to the - # Apple notary service with the app-specific password, not with the - # signing cert. Omitting them fails the build rather than skipping it, - # since `notarize` is now on for this path. + # Require Apple id/app-specific password/team for notarization, separate from signing cert. APPLE_ID: ${{ secrets.APPLE_ID }} APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} - # Why: the updater resolves a tag, then fetches latest-mac.yml from it. A - # release missing that manifest is a tag the picker offers and the download - # 404s on, so fail loudly instead of leaving a broken entry. + # Verify latest-mac.yml before exposing release; missing manifests cause picker download 404s. - name: Verify update manifest published shell: bash env: @@ -394,10 +308,7 @@ jobs: assets="$(gh release view "$TAG" --repo "$HOURLY_REPO" --json assets --jq '.assets[].name')" echo "Published assets:" echo "$assets" - # Why exit 1 without deleting here: the release is still a draft, so it is - # already invisible to users, and the failure handler below owns cleanup. - # Deleting inline under `set -e` would also let the delete's exit code - # preempt this explicit failure. + # Drafts are already invisible; let failure cleanup delete them so set -e cannot mask this explicit failure. for required in latest-mac.yml; do if ! grep -qx "$required" <<<"$assets"; then echo "::error::Hourly draft $TAG is missing $required; the updater could not install it." @@ -409,9 +320,7 @@ jobs: exit 1 fi - # Why this is the last mutating step: publishing the draft is what makes the - # build visible to listReleaseBuilds. Doing it only after the manifest check - # means the picker can never offer a release whose assets are incomplete. + # Publish only after manifests verify so picker never offers incomplete assets. - name: Publish the verified release id: publish_live shell: bash @@ -421,21 +330,11 @@ jobs: NAME: ${{ steps.hourly.outputs.name }} run: | set -euo pipefail - # --title again: electron-builder resolves this draft by tag and may - # rewrite its title on upload. Re-asserting here means the name the - # picker reads is the one composed above, whatever it did in between. + # Reassert the picker title after electron-builder may rewrite the tag’s draft on upload. gh release edit "$TAG" --repo "$HOURLY_REPO" --draft=false --prerelease --title "$NAME" echo "Published $TAG as \"$NAME\"" - # Why: a draft left behind by a failed publish is invisible to users but still - # holds its tag name, so the next run for the same minute would collide. - # - # Why it is gated on publish_live not having succeeded: a later failure (the - # prune step) must not delete a release that already went live and that users - # may already be installing. A job killed by the outer timeout runs no steps - # at all — which is exactly why the release stays a draft until verified. - # Why cancelled() too: a run stopped from the Actions UI is not a failure(), - # so without it a manual cancel mid-publish would strand the draft. + # Delete failed/cancelled drafts but never publish_live releases, including after prune failure; outer timeouts run no cleanup. - name: Discard the draft release on failure if: >- (failure() || cancelled()) && steps.release.outputs.tag != '' && @@ -452,34 +351,18 @@ jobs: echo "::warning::Could not discard draft $TAG; remove it manually." - name: Prune old hourly releases - # Only after a live publish: $TAG is then a non-draft we must not delete, - # and failed runs should not reshuffle retention around a draft that the - # failure path is about to discard. + # Prune only after live publish; failed runs leave draft disposal to cleanup. if: steps.publish_live.outcome == 'success' shell: bash env: GH_TOKEN: ${{ steps.app_token_publish.outputs.token }} - # Protect the tag this run just shipped; at the retain cap, a bad sort - # can otherwise mark the newest release as stale and delete it. + # Exclude the just-published tag from retention pruning. TAG: ${{ steps.release.outputs.tag }} run: | set -euo pipefail - # Why: --cleanup-tag so pruning does not leave orphan tags behind that - # keep showing up in tag lists with no release or assets attached. - # Drafts are excluded so retention counts shipped builds only; a stale - # draft is handled by the failure path, not by the retention window. - # - # Sort by publishedAt (not createdAt). Many non-draft hourlies share one - # createdAt (bulk import / re-create), so createdAt ranking is unstable: - # reverse of a stable sort then drops the just-published tag past the - # retain window. publishedAt is real recency; tagName (...YYYYMMDDHHMM) - # is the deterministic tie-break. - # - # Why force $TAG to the front before slicing: a hard retain-window seat - # for this run's release. Dropping $TAG from the list *before* the slice - # would permanently keep retain+1 releases; skipping it only in the - # delete loop would under-prune when the sort is still wrong. Partition - # keeps relative order of every other tag. + # Delete pruned tags too; exclude drafts owned by failure cleanup. Rank by publishedAt/tagName: + # createdAt ties (bulk import/recreation) can evict the just-published release. Partition $TAG to + # the front before slicing so it owns exactly one retained seat without under-pruning or keeping retain+1. jq_filter='map(select(.isDraft | not)) | sort_by(.publishedAt // "", .tagName) | reverse' if [[ -n "${TAG:-}" ]]; then jq_filter+=" | (map(select(.tagName == \"${TAG//\"/\\\"}\")) + map(select(.tagName != \"${TAG//\"/\\\"}\")))" @@ -503,21 +386,10 @@ jobs: echo "::warning::Could not prune $tag" done <<<"$stale" - # Why this runs after the mac leg rather than beside it: the tag and version - # are computed inside that job, so until it has run nothing else can name the - # release to upload into. The cost is small enough not to matter — the Windows - # leg measures ~7.5 min (install 2m45, build 35s, NSIS package 3m) against a - # ~9.5 min mac run, which keeps a hourly run far inside its interval. - # - # Why `./` rather than a pinned `@main`: `uses:` resolves against the ref this - # file itself came from, which for an ordinary dispatch is main. Someone who - # deliberately points the Actions "Use workflow from" picker at a branch - # already gets that branch's copy of this entire file, so this follows the same - # rule instead of inventing a second one. + # Wait for mac tag/version before Windows upload; local ./ workflow consistently follows the dispatch ref. build-hourly-win: needs: build-hourly-mac - # Only once the mac release is actually live: there is no release to upload - # into otherwise, and the Windows workflow refuses to create one. + # Publish Windows only into the existing live mac release. if: needs.build-hourly-mac.outputs.published == 'true' permissions: contents: read diff --git a/.github/workflows/mobile-android-release.yml b/.github/workflows/mobile-android-release.yml index a9e269c0b75..b79a58966c1 100644 --- a/.github/workflows/mobile-android-release.yml +++ b/.github/workflows/mobile-android-release.yml @@ -1,8 +1,6 @@ name: Mobile Android Release -# Why a separate workflow from iOS: iOS releases go through App Store review, -# which can take days. Decoupling the triggers lets an Android release ship -# immediately without waiting on iOS, and vice versa. +# Release Android independently so App Store review cannot block its shipping cadence. on: push: tags: @@ -23,10 +21,7 @@ jobs: android-build: runs-on: ubuntu-latest - # Why: the "Create GitHub Release" step below uses the default GITHUB_TOKEN - # to call `gh release create`, which requires contents:write. Without this, - # the job builds the APK fine but fails at release time with - # "HTTP 403: Resource not accessible by integration". + # Grant contents:write for gh release create; build success alone does not authorize publication. permissions: contents: write @@ -75,10 +70,7 @@ jobs: - name: Build Android release APK run: cd android && ./gradlew assembleRelease - # Why: an install that fails with a missing certificate or a package-parse error is - # usually a download that died near the end, and the signature block sits in the last - # ~100 KB. Publishing the size and digest is what lets a reporter tell a truncated - # download apart from a bad build without anyone re-deriving them from the asset. + # Publish size/digest to distinguish truncated downloads (missing tail signature block) from bad APK builds. - name: Checksum the APK id: apk run: | @@ -93,10 +85,8 @@ jobs: apk="${apks[0]}" bytes="$(wc -c < "$apk" | tr -d ' ')" sha256="$(sha256sum "$apk" | cut -d ' ' -f 1)" - # A sibling file in `sha256sum -c` format, so verifying a download is one command - # and no retyped digest. The asset globs below match *.apk and skip it. The ` *` - # marker is binary mode: Git Bash's sha256sum reads text mode as a licence to - # translate line endings while hashing, which would fail on a valid APK. + # Publish sha256sum -c sidecars, excluded by *.apk globs. Use binary “ *” mode: + # Git Bash text-mode newline translation can reject a valid APK. printf '%s *%s\n' "$sha256" "$(basename "$apk")" > "$apk.sha256" { @@ -144,18 +134,13 @@ jobs: '${{ steps.apk.outputs.bytes }}' '${{ steps.apk.outputs.sha256 }}' } - # The section is always last, so dropping from its heading to EOF leaves the - # generated notes intact. \r* because a body round-tripped through the API has CRLFs. + # Verification is last; replace from heading to EOF, accepting API-roundtripped CRLFs. drop_verification_section() { sed '/^### Verify your download\r*$/,$d' } - # Why: reuse the desktop release path's character-safe truncation so a multi-byte - # character cannot be split at the cap. The section's own length is reserved out of - # that cap, because appending after truncating would push a near-limit body past - # GitHub's API limit and fail the call — on the upload path, after --clobber has - # already replaced the assets. `wc -c` counts bytes, so the section's multi-byte - # characters over-reserve, which errs toward a shorter body. + # Truncate by characters like desktop releases and reserve the appended section inside GitHub’s cap, + # or --clobber could replace assets before notes fail. wc -c over-reserves multibyte section text safely. write_notes_with_verification() { NOTES_FILE="$notes_file" \ NOTES_RESERVE="$(verification_section | wc -c | tr -d ' ')" \ @@ -179,26 +164,21 @@ jobs: --clobber \ android/app/build/outputs/apk/release/*.apk \ "${{ steps.apk.outputs.checksum }}" - # Why the body is rewritten too: --clobber replaced the APK, so a digest left - # over from the previous build now describes a file nobody can download, and a - # reader comparing against it would reject a good APK. + # --clobber replaced the APK; rewrite its digest so users cannot compare against obsolete bytes. gh release view "$tag" --repo "$GITHUB_REPOSITORY" --json body --jq .body \ | drop_verification_section > "$notes_file" write_notes_with_verification gh release edit "$tag" --repo "$GITHUB_REPOSITORY" --notes-file "$notes_file" else - # Why: release tags live on side branches, so GitHub's automatic - # previous-tag detection reaches back several releases; that body - # already exceeds the 125000-character API limit and grows each - # release. Pin the comparison base and cap the size. + # Side-branch release tags make automatic previous-tag detection span multiple releases and exceed + # the 125000-character API cap; pin the comparison base and truncate. previous_tag="$( gh release list --repo "$GITHUB_REPOSITORY" --limit 200 --json tagName --jq '.[].tagName' \ | grep '^mobile-android-v' | grep -Fxv "$tag" | sort -V | tail -1 || true )" if [ -n "$previous_tag" ]; then - # Why: gh writes the JSON error body to stdout on an HTTP error, so a - # non-empty file is not proof of success — gate on exit status. + # gh writes HTTP error JSON to stdout too; require exit success, not merely a nonempty file. if ! gh api "repos/$GITHUB_REPOSITORY/releases/generate-notes" -X POST \ -f tag_name="$tag" \ -f target_commitish="$GITHUB_SHA" \ @@ -222,8 +202,7 @@ jobs: "${{ steps.apk.outputs.checksum }}" fi - # Why a separate job: the release is already published here, so a failed bump only leaves - # the site's APK download on the previous release and must not fail the release run. + # Keep site bump advisory after publication; failure only leaves its download link on the prior APK. bump-apk-download: needs: android-build if: needs.android-build.outputs.publish_release == 'true' diff --git a/.github/workflows/mobile-ios-release.yml b/.github/workflows/mobile-ios-release.yml index 94de97d4954..2d2ab4d4037 100644 --- a/.github/workflows/mobile-ios-release.yml +++ b/.github/workflows/mobile-ios-release.yml @@ -1,8 +1,6 @@ name: Mobile iOS Release -# Why a separate workflow from Android: iOS releases go through App Store -# review, which can take days. Decoupling the triggers lets an Android release -# ship immediately without waiting on iOS, and vice versa. +# Release iOS independently so App Store review cannot block Android. on: push: tags: @@ -28,19 +26,13 @@ permissions: jobs: ios-build: - # GitHub-hosted macOS runner: required for Xcode. Expo SDK 55's - # expo-modules-core declares swift_version 6.0 and uses Swift 6 syntax - # (@MainActor isolation). Xcode 16.x (macos-15) can't even parse it - # ("unknown attribute 'MainActor'"), so we need Xcode 26.x → macos-26. + # Use macos-26/Xcode 26 for Expo SDK 55’s Swift 6 syntax/isolation; Xcode 16 runners cannot parse it. runs-on: macos-26 # Archive + upload is ~30–40m when healthy; 90m leaves margin for setup. timeout-minutes: 90 env: - # Why: this job and ios-distribute resolve gems ~25 minutes apart, so both - # must install the committed Gemfile.lock exactly. Frozen turns a lockfile - # drift into a setup failure instead of two different fastlane versions in - # one release. + # Freeze Gemfile.lock in both jobs; installs ~25min apart must not resolve different fastlane versions. BUNDLE_FROZEN: 'true' outputs: @@ -106,12 +98,7 @@ jobs: working-directory: mobile/ios run: bundle exec pod install - # Why: `-allowProvisioningUpdates` + the App Store Connect API key can - # create/refresh provisioning profiles, but it cannot recreate the - # distribution certificate's PRIVATE KEY across runs. So we import a - # pre-exported distribution .p12 (created once via Apple Developer) into a - # throwaway keychain. The keychain is ephemeral to the runner and torn - # down with the VM; nothing secret is written to the repo. + # Import exported distribution .p12 into ephemeral runner keychain: provisioning updates/API key cannot recreate its private key. - name: Import distribution certificate env: IOS_DIST_CERT_P12: ${{ secrets.IOS_DIST_CERT_P12 }} diff --git a/.github/workflows/mobile.yml b/.github/workflows/mobile.yml index 1c1f4057882..541ecef9227 100644 --- a/.github/workflows/mobile.yml +++ b/.github/workflows/mobile.yml @@ -10,9 +10,7 @@ on: - converted_to_draft paths: - 'mobile/**' - # Why all of src/shared: mobile imports hundreds of its modules, directly and through the RPC - # recordings and the host params check, so any shared edit can move a golden or break mobile's - # typecheck. Why the root lockfile: those modules resolve their packages from the root install. + # Route shared edits to mobile’s imports/RPC recordings/typecheck; root lockfile changes affect those modules’ root-resolved packages. - 'src/shared/**' # Mobile's Node RPC recording tools import this package. - 'src/packages/process-host/**' @@ -27,8 +25,7 @@ on: - 'src/main/runtime/agent-session-*.ts' - 'src/main/native-chat/agent-session-wire/**' - 'src/main/runtime/structured-attention-original-delivery.test-fixture.ts' - # Why: this job holds the only checks that load the Fastfile, so edits to - # it or to the release workflow it guards must re-run them. + # Rerun the only Fastfile-loading checks for release-workflow/Fastfile edits. - '.github/workflows/mobile.yml' - 'config/mobile-ci.vitest.config.mjs' - 'config/scripts/run-mobile-ci-tests.mjs' @@ -57,9 +54,7 @@ on: - '!src/main/**/*.test.[jt]s' - '!src/main/**/*.test.[jt]sx' - '!src/main/**/*.test.[cm][jt]s' - # Why main too: two pull requests can each pass against their own base and disagree once both - # land, and `verify` never runs on main. The same source paths as above, less the CI inputs only - # `verify` reads. + # Check matching source inputs on main too: independently passing PRs can conflict after merge, and verify is PR-only. push: branches: - main @@ -98,9 +93,7 @@ on: - '!src/main/**/*.test.[cm][jt]s' concurrency: - # Per commit on main, not per branch. GitHub cancels any PENDING run in a group when a new one - # queues, whatever `cancel-in-progress` says, so one shared main group drops the middle merge of - # three -- and a merge that breaks main there is exactly what `main-tests` checks for. + # Key main concurrency by commit: GitHub replaces pending runs even with cancel-in-progress:false, hiding middle-merge regressions. group: mobile-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true @@ -111,15 +104,11 @@ jobs: verify: name: Mobile verify if: github.event_name == 'pull_request' && github.event.pull_request.draft != true - # Why ARM: 209s of this job is Vitest and nothing here needs x86: no Android SDK, emulator, gradle, - # Hermes or Watchman, no docker, and no artifacts. The Gemfile.lock lists the generic `ruby` - # platform, so frozen bundler installs without an aarch64-linux entry. + # Use ARM for pure Vitest; no x86 native toolchain is needed and Gemfile.lock’s ruby platform supports frozen install. runs-on: blacksmith-4vcpu-ubuntu-2404-arm env: - # Why: an unfrozen bundler silently re-resolves when Gemfile.lock drifts - # from the Gemfile, which is how the release jobs could land on different - # fastlane versions in the first place. Fail here instead. + # Freeze bundler to reject Gemfile drift before release jobs resolve different fastlane versions. BUNDLE_FROZEN: 'true' ORCA_MOBILE_TYPECHECK_COMPILER: 'bun' @@ -152,9 +141,7 @@ jobs: echo 'should_run=true' >> "$GITHUB_OUTPUT" fi - # bundler-cache installs mobile/Gemfile.lock, so this job is also what - # proves the pinned fastlane the release workflow depends on still - # resolves — before a release run finds out. + # Bundler-cache proves the release’s pinned fastlane still resolves before a release needs it. - name: Setup Ruby and fastlane id: ruby-tools background: true @@ -168,8 +155,7 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - # Why here: pr.yml runs these import rules before mobile's install, when mobile's tsconfig - # (which extends expo's) can't load, so its import cycles were never seen. Zero tolerance. + # Check mobile import cycles after Expo deps install; pre-install pr.yml cannot load its tsconfig. - name: Lint mobile imports with mobile dependencies installed working-directory: . run: node node_modules/oxlint/bin/oxlint --config config/oxlint-code-quality-native-plugins.json mobile --deny-warnings @@ -189,12 +175,7 @@ jobs: background: true run: node scripts/check-tests-typecheck-ratchet.mjs - # This includes the bridged replay of the whole recording corpus, which used to be a second - # step of its own behind RPC_FOUNDATION_BRIDGE=1. A gate nobody can forget to set is the point: - # it fails when a divergence class grows, when a divergence lands in no class at all, or when - # one of the 103 goldens inside the C1 page closure changes the verdict it is pinned to. It is - # ~3 min of test time on its own, and Vitest runs it on a worker beside the rest of the suite, - # so folding it in costs a fraction of that in wall time and one step less to skip. + # Run bridged recording corpus unconditionally: reject growing/unclassified divergence and changed verdicts among 103 C1 goldens. Share worker time instead of a separate skippable env-gated step. - name: Detect mobile test inputs id: test-scope shell: bash @@ -221,9 +202,7 @@ jobs: # All checks read the completed install; compilers need not hold up test workers. - wait: [production-types, test-types, ruby-tools] - # Reports, never gates: the goldens are content-addressed JSON, so their raw diff is hashes. - # This decodes which recorded behaviour the pull request moves into the job summary, and runs - # after a red Test too, when a reviewer most wants it. + # Decode hash-addressed golden behavior changes for review, including after failed tests; advisory only. - name: Summarize RPC recording changes if: ${{ !cancelled() }} continue-on-error: true @@ -237,11 +216,7 @@ jobs: if: steps.ruby-scope.outputs.should_run != 'false' run: ruby fastlane/fastfile_testflight_arguments_test.rb - # Why: nothing else in CI loads the Fastfile, so a syntax error, a broken - # require, or an undefined constant only surfaces mid-release — the - # ios-distribute job failed every run for six days that way. `lanes` just - # loads and lists, so it needs no App Store Connect credentials and makes - # no network calls to Apple. + # Load/list Fastfile lanes without Apple credentials/network to catch syntax/require/constants before release; ios-distribute failed six days without this guard. - name: Smoke-check the Fastfile if: steps.ruby-scope.outputs.should_run != 'false' env: @@ -341,9 +316,7 @@ jobs: background: true run: node scripts/check-tests-typecheck-ratchet.mjs - # The whole suite, not only the recordings: the mutant and page-bridge suites read the goldens - # too. A red run here names each golden and the `rpc:record` command; the author of the merge - # that turned it red re-records in a follow-up. + # Run all golden consumers, including mutant/page-bridge suites; failures name the recording and rpc:record command. - name: Setup Bun for tests uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: diff --git a/.github/workflows/pr-test-loc.yml b/.github/workflows/pr-test-loc.yml index 6c284cb4102..da82b6c1d6a 100644 --- a/.github/workflows/pr-test-loc.yml +++ b/.github/workflows/pr-test-loc.yml @@ -22,10 +22,7 @@ jobs: runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 2 steps: - # Why no checkout: the Files API already has per-file additions/deletions. - # Why the default branch and never pull//head: this job holds a write-scoped - # GITHUB_TOKEN, so it may only execute reviewed code. A PR that edits these - # scripts takes effect once merged. + # Read file counts through API without checkout; execute only default-branch scripts because GITHUB_TOKEN has write scope. PR edits take effect after merge. - name: Count test vs non-test LoC env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4b41b45bd88..8245654acae 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -18,12 +18,7 @@ permissions: contents: read jobs: - # Why: a README/docs-only PR used to start the full matrix (test shards, - # two package jobs, typecheck, git compat, xterm, shell contracts). Path - # filters on `on.pull_request` would drop the `verify` check entirely; this - # detector keeps verify as the required aggregate and skips the expensive jobs. - # Per-job outputs also skip git-compat/xterm/packaging/shell when those - # inputs are unchanged; empty diffs fail closed and run everything. + # Keep required verify present for docs-only PRs while routing expensive jobs by inputs; empty diffs fail closed to full checks. code_paths: if: github.event_name != 'pull_request' || github.event.pull_request.draft != true name: detect changes and check repository guards @@ -67,11 +62,7 @@ jobs: - name: Checkout uses: actions/checkout@v6 with: - # Why depth 50 and not 0: every diff below resolves to HEAD^1, so only the merge commit - # and a little slack are needed. Fetching all 8127 refs' commit graph cost ~20s here and - # is charged to the start of all 22 jobs, since they all need this one. - # Why blob:none stays: the sparse tree below is ~7 files, so there are no blobs to - # materialize and no promisor refetch. Measured 9.5s -> 1.6s against 20.7s today. + # Fetch 50 commits for merge HEAD^1 diffs instead of all 8127 refs (~20s); sparse ~7-file blob:none checkout avoids materialization/refetch (9.5s→1.6s). fetch-depth: 50 filter: blob:none sparse-checkout: | @@ -128,12 +119,9 @@ jobs: BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | set -euo pipefail - # Why HEAD^1 and not --merge-base: HEAD is the pull request merge commit, so its first - # parent is the base side already. Computing a merge base instead would require the - # payload base SHA to be in the graph, which is what forced a full-history checkout. + # PR merge HEAD’s first parent is the base; --merge-base would require the payload SHA/full history. DIFF_BASE="$(node config/scripts/git-pull-request-diff-base.mjs "$BASE_SHA")" - # Why --no-renames: name-only rename detection can report only the destination. - # A code file moved under docs/ must still expose its code-side deletion. + # --no-renames retains code-side deletions when files move into docs. CHANGED="$(git diff --name-only --no-renames --diff-filter=ACDMR "$DIFF_BASE" HEAD)" echo "Changed paths:" printf '%s\n' "$CHANGED" @@ -156,13 +144,11 @@ jobs: if git diff --quiet --no-renames --diff-filter=ACDRTUXB "$DIFF_BASE" HEAD; then E2E_SCOPE_ARGS=(--daemon-init-test-support) fi - # Source routes are executable contracts so a test can prove exact - # authorities, exclusions, and sentinels without evaluating workflow shell. + # Executable routes let tests prove authorities/exclusions/sentinels without evaluating shell. TEST_FILES_JSON="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs "${E2E_SCOPE_ARGS[@]}")" echo "test_files=$TEST_FILES_JSON" >> "$GITHUB_OUTPUT" printf '%s\n' "$TEST_FILES_JSON" | node config/scripts/ci-e2e-job-selection.mjs --job-outputs >> "$GITHUB_OUTPUT" - # Why its own signal: the real-IME lane is a whole ibus session, not a spec, so it must - # trigger on IME source rather than on a spec name in some route's list. + # Real IME runs an entire IBus session; route it by IME source, independently of spec lists. NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)" echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT" WSL_CHANGED="$(git diff --name-only --no-renames --diff-filter=ACDMR "$DIFF_BASE" HEAD)" @@ -192,11 +178,7 @@ jobs: - name: Checkout uses: actions/checkout@v6 with: - # Why depth 50: the gates below diff against HEAD^1, so no merge base is computed and - # the payload base SHA need not be in the graph. - # Why no blob:none here, unlike code_paths: this job checks out all 30,226 files, and - # the filter then forces a second promisor fetch of nearly every blob. Measured 23s - # blobless against ~11s without it. + # Fetch 50 for merge HEAD^1, no merge-base graph needed. Full-tree checkout omits blob:none to avoid refetching ~30k blobs (23s versus ~11s). fetch-depth: 50 persist-credentials: false @@ -216,10 +198,7 @@ jobs: continue-on-error: true run: node -p '"executable=" + process.execPath' >> "$GITHUB_OUTPUT" - # Why two guarded installs: the mixed root+mobile store entry is 537 MB against - # 321 MB for root alone, and restoring it costs 8.6s against 4.6s. Most PRs skip the - # mobile install below, so they were paying 216 MB for packages they never link. The - # root-only key is also the one the hourly warmer reseeds. Only one of these runs. + # Choose one root/mobile install: root-only store is 321MB/4.6s versus mixed 537MB/8.6s, saving unused mobile downloads on most PRs. - uses: ./.github/actions/install-node-dependencies if: needs.code_paths.outputs.mobile_dependencies != 'true' with: @@ -325,9 +304,7 @@ jobs: # Mobile installation changes import resolution for the native cycle check. - wait: native-code-quality - # Why here: the changed-code gate lints mobile files too, and its type-aware pass - # resolves types from mobile/node_modules. Without the install every mobile type - # degrades to an `error` type — reported as phantom findings against the changed lines. + # Install mobile before type-aware changed-code lint or unresolved mobile types become phantom error-type findings. - uses: ./.github/actions/install-mobile-dependencies if: needs.code_paths.outputs.static_analysis == 'true' && needs.code_paths.outputs.mobile_dependencies == 'true' @@ -419,9 +396,7 @@ jobs: if: needs.code_paths.outputs.static_analysis == 'true' run: node config/scripts/check-mobile-shared-dependency-versions.mjs - # Why: extraction writes sorted evidence to an isolated temporary path, - # so feature PRs need one normalized AST pass rather than a three-OS matrix. - # Use preflight's pinned Bun even when the optional local pin differs. + # Extract sorted evidence to isolated temp paths with one normalized platform-independent AST pass; use preflight’s pinned Bun. - name: Verify localization extraction if: '!cancelled()' id: localization-extraction @@ -461,9 +436,7 @@ jobs: - uses: ./.github/actions/build-mobile-web-bundle if: steps.orcad-smoke-scope.outputs.run == 'true' - # Why both: the ratchet proves nothing reachable from the runtime imports electron, - # which is a property of the import graph. This proves the Node artifact it enables - # actually boots, pairs, creates a worktree and round-trips a real PTY. + # Pair import-graph electron exclusion with built-Node boot/pair/worktree/PTY smoke; graph purity alone does not prove runtime operation. - name: Boot orcad and round-trip a terminal if: steps.orcad-smoke-scope.outputs.run == 'true' env: @@ -503,11 +476,7 @@ jobs: - wait: [localization-catalogs, localization-extraction] - # Why: project-owned type declarations must live in .ts so tsc - # actually checks them. TypeScript's skipLibCheck: true (inherited - # from @electron-toolkit/tsconfig) silently widens unresolved names - # in .d.ts to `any`, which is how #1186 shipped a broken IPC signature - # past typecheck. See .github/CONTRIBUTING.md#type-declarations-prefer-ts-over-dts. + # Keep project declarations in .ts: skipLibCheck silently widened unresolved .d.ts names to any, hiding #1186 IPC breakage. See .github/CONTRIBUTING.md#type-declarations-prefer-ts-over-dts. - name: Guard against project-owned .d.ts in preload/shared if: needs.code_paths.outputs.static_analysis == 'true' run: | @@ -618,8 +587,7 @@ jobs: "alpine/git:edge-2.38.1|2.38.1" "alpine/git:v2.49.1|2.49.1" ) - # Why pull up front: a lane's first `docker run` otherwise pulls its image - # while the sibling lane is mid-test, and that stall is charged to the test. + # Pull before tests so a sibling’s timed lane cannot be stalled by the first image download. for spec in "${specs[@]}"; do docker pull --quiet "${spec%%|*}" done @@ -655,12 +623,7 @@ jobs: done exit "$status" - # Why this job: Orca's session index-heal depends on a Codex behavior — a - # `thread/read` of an unindexed rollout performs a read-repair that inserts the - # `threads` row. Every unit test drives a stub app-server and asserts only that the - # call did not error, so if Codex dropped the repair they would all stay green while - # the subsystem went inert. This runs the pinned real binary and fails when the - # repair stops happening. Pinned because the binary is the thing expected to drift. + # Run pinned real Codex: thread/read must repair an unindexed rollout’s threads row; stub-only units cannot detect upstream removal of this behavior. codex_index_heal_contract: name: Codex index-heal contract needs: [code_paths, preflight] @@ -692,9 +655,7 @@ jobs: - name: Verify Codex index-heal contract env: - # Why REQUIRED: without a binary the suite skips, and a job that skips - # reports success. This turns a failed or missing install into a red test - # instead of a green no-op. + # Require installed binary or skip would report green without exercising the contract. ORCA_CODEX_CONTRACT_REQUIRED: '1' ORCA_CODEX_CONTRACT_VERSION: ${{ env.CODEX_CLI_VERSION }} run: | @@ -719,9 +680,7 @@ jobs: pnpm exec vitest run --config config/vitest.config.ts \ src/main/pty/codex-no-daemon-binary-contract.test.ts - # Why both pins and latest: Orca approves its managed-home hook entry with a hash - # Codex computes, so a release that changes it must redden here before users - # meet a review screen. + # Test pins plus latest because Codex’s managed-home approval hash can change before users encounter review prompts. - name: Verify Codex hook file-entry contract (pinned) env: ORCA_CODEX_HOOK_CONTRACT_REQUIRED: '1' @@ -768,9 +727,7 @@ jobs: needs.code_paths.result == 'success' && needs.code_paths.outputs.xterm_patch_sync == 'true' && needs.preflight.result == 'success' - # Why ARM: the patch check rebuilds 4 packages x 2 builds and byte-compares against the - # checked-in bundles. Those were generated on Linux x64 and reproduce byte-for-byte on - # darwin-arm64, so the output is neither host-arch nor host-OS dependent. + # Use ARM for four-package pristine/patched rebuild byte comparison: output generated on Linux x64 reproduces on darwin-arm64. runs-on: blacksmith-4vcpu-ubuntu-2404-arm steps: @@ -807,10 +764,7 @@ jobs: # Bound stalled prerequisite downloads instead of waiting for GitHub's 6-hour default. timeout-minutes: 15 env: - # Why: the suites below gate their live fish tests on the binary, which is - # right on a developer machine and wrong here — this job is a required check - # and its fish lane is the only end-to-end guard for #9993, so a skip would - # report green with nothing exercised. Turns those skips into failures. + # Require live fish instead of developer-friendly skips: this is the only full #9993 guard. ORCA_REQUIRE_FISH: '1' # Why: the runner image ships pwsh, and the codex wrapper's PowerShell lane must not skip. ORCA_REQUIRE_PWSH: '1' @@ -867,9 +821,7 @@ jobs: - wait: shells - # Separate from the install so the failure names the contract, not an apt error. - # ORCA_REQUIRE_FISH re-checks this at test time; this step just fails in seconds - # instead of after a full dependency install. + # Fail fish prerequisites separately before dependency install; ORCA_REQUIRE_FISH checks again in tests. - name: Require fish 4+ run: | version="$(fish --version 2>/dev/null || true)" @@ -926,17 +878,13 @@ jobs: run_relay_integration: ${{ needs.code_paths.outputs.relay_integration != 'false' || needs.code_paths.outputs.orcad_browser == 'true' }} browser_contract: ${{ needs.code_paths.outputs.orcad_browser == 'true' }} - # Why a sibling and not part of the test workflow: it is advisory, so it must not delay the - # gate. Inside unit-tests.yml a caller's `needs: test` waited for it, holding verify ~36s - # after the last shard. Deliberately absent from verify's needs for the same reason. + # Keep advisory evidence sibling to tests and outside verify.needs; reusable-workflow dependency otherwise added ~36s after last shard. unit_selection_evidence: needs: [test] if: ${{ !cancelled() && (needs.test.result == 'success' || needs.test.result == 'failure') }} uses: ./.github/workflows/unit-selection-evidence.yml - # Why its own job: it needs mobile/node_modules and a real browser, and the sharded `test` - # matrix would pay for both on every shard to run two files. It builds the same bundle the - # package job ships, and adds the render and census checks packaging does not run. + # Isolate two browser/mobile files to avoid every unit shard paying for their installs; test the same shipped bundle plus render/census. mobile_web_app: name: mobile web app bundle needs: [code_paths] @@ -958,9 +906,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # The drawer check runs on WebKit as well as Chrome, because the shell's iOS WebView is - # WebKit and the Chrome above cannot stand in for it. Downloaded rather than resolved from - # the runner: Ubuntu ships no WebKit build to point at. + # Test drawer with WebKit as iOS WebView uses it; runner Chrome cannot substitute. - name: Install WebKit for the drawer check id: webkit background: true @@ -976,10 +922,7 @@ jobs: background: true run: node config/scripts/run-mobile-web-app-checks.mjs --prepare-route-snapshot "$RUNNER_TEMP/mobile-routes.json" - # Why the runner's Google Chrome and not a downloaded chromium: same reason as the orcad - # browser job -- Ubuntu 24.04 only ships an AppArmor userns profile for the Chrome .deb. - # Why fail instead of skip: a silently skipped render check is the failure this job exists - # to prevent. + # Require runner Google Chrome: Ubuntu’s AppArmor userns profile covers its .deb, unlike downloaded Chromium; missing render checks must fail. - name: Resolve Chrome for the render check run: | set -euo pipefail @@ -997,16 +940,7 @@ jobs: # Browser checks must observe both a finished install and the built bundle. - wait: [webkit, mobile-routes] - # The bundling tests skip themselves where mobile dependencies are absent, which is how they - # stay green in the sharded `test` job. This is the job that installs them, so here a missing - # install has to fail rather than skip everything the job exists to run. - # Why a prefix and not a file list: the list this replaces had gone stale twice without - # anyone noticing, because a census whose closure block skips without the env flag below is - # green in the sharded `test` job whether or not it ever runs here. The prefix is the same - # one `pr-code-change-scope.mjs` fires this job on, so naming a test into the family is all - # it takes to have it run. Quoted because these are vitest filename filters, matched as - # substrings against the discovered files, and the shell must not touch them. - # Route censuses share fresh dependency lists for this invocation; scratch builds stay independent. + # Require mobile install and select census family by quoted filename prefix, matching routing; stale file lists/env-gated skips previously hid missing coverage. - name: Builder, override census and render checks env: ORCA_MOBILE_WEB_APP_DEPS_REQUIRED: '1' @@ -1026,15 +960,11 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404-arm steps: - # Why fetch-depth 0: the harness extracts the newest release tag to skew - # current code against it. The default shallow clone has no tags, which is - # why this cannot ride along in the sharded `test` job. + # Fetch tags/full history for newest-release skew; shallow unit shards cannot supply it. - name: Checkout uses: actions/checkout@v6 with: - # Why blob:none: full history is needed for the merge-base diff, but historical - # file contents are not. Blobs are ~89% of this repo's pack, and Git fetches the - # few this job actually reads on demand. + # Use blob:none for merge-base history because blobs are ~89% of pack; fetch rare historical contents on demand. fetch-depth: 0 filter: blob:none persist-credentials: false @@ -1057,8 +987,7 @@ jobs: DIFF_BASE="$(node config/scripts/git-pull-request-diff-base.mjs "$BASE_SHA")" pnpm run check:runtime-launcher-protocol-ratchet -- --base "$DIFF_BASE" - # Runs the whole directory so a new file there cannot be left out; an empty match - # exits 1 ("No test files found"), so this lane cannot pass while running zero tests. + # Run the entire directory; zero matches fail so new contracts cannot be omitted silently. - name: Old/new client and server compatibility journeys run: pnpm exec vitest run --config config/vitest.config.ts tests/e2e/cross-version-wire/ @@ -1141,9 +1070,7 @@ jobs: ORCA_DAEMON_SHUTDOWN_FIXTURE_CACHE_IMAGE: ${{ steps.daemon-fixture-cache.outputs.image }} run: node config/scripts/run-daemon-shutdown-descendants-docker.mjs - # Why --no-file-parallelism: every file here launches a full Electron stack twice, and each - # probe carries its own in-process deadline. Four at once on a 4-vCPU runner starve each other - # past those deadlines; serial, every probe owns the runner. + # Serialize dual-Electron lifecycle files: four concurrent probes on four vCPUs starve each other past their own deadlines. - name: Test Linux Electron lifecycle boundary env: ORCA_BACKGROUND_LAUNCH: '1' @@ -1243,9 +1170,7 @@ jobs: background: true run: pnpm run build:web-from-renderer - # Why here and not inside "Build package inputs": this job assembles packaging inputs step by - # step instead of calling build:release, and electron-builder's beforePack guard hard-fails - # without out/mobile-web. This is the real page, ~8 MB, not a bootstrap document. + # Build the real ~8MB mobile page because stepwise package assembly bypasses build:release and beforePack refuses its absence. - name: Build mobile web bundle run: pnpm run build:mobile-web @@ -1341,8 +1266,7 @@ jobs: restore-keys: | electron-builder-win- - # Why persist-native-cache false: this job later rebuilds the same path for - # Electron. A post-job save would store the Electron ABI under the Node key. + # Do not save Node cache post-job: later Electron rebuild mutates the same paths to another ABI. - uses: ./.github/actions/install-node-dependencies id: deps with: @@ -1379,9 +1303,7 @@ jobs: - name: Cache Windows CLI launcher uses: actions/cache@v5 with: - # Why the cargo directories ride along: a hit on .build skips the build - # entirely, but a miss otherwise recompiles the resource crate from a - # cold registry. + # Cache cargo registry with .build so misses do not rebuild resource crates cold. path: | native/windows-cli-launcher/.build native/windows-cli-launcher/target @@ -1390,9 +1312,7 @@ jobs: ~/.cargo/registry key: windows-cli-launcher-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('native/windows-cli-launcher/**', 'native/windows-agent-hook/**', 'config/scripts/build-windows-cli-launcher.mjs', 'config/scripts/build-windows-agent-hook.mjs', 'config/scripts/windows-rust-executable-build.mjs', 'resources/build/icon.ico', 'package.json') }} - # Why an explicit step rather than trusting the runner image: the packaged - # CLI launcher is a native Rust binary, and a host without cargo fails deep - # inside electron-builder's native hook instead of here. + # Require cargo early; otherwise packaged Rust launcher fails deep inside builder native hooks. - name: Ensure the Rust toolchain for the Windows CLI launcher shell: pwsh run: | @@ -1402,8 +1322,7 @@ jobs: } cargo --version - # Why before the tests: the orca-agent-hook.exe suites then share one prebuilt binary rather - # than each starting its own build mid-run. + # Build agent-hook once before tests so suites share the binary instead of compiling concurrently. - name: Build the Windows agent hook run: node config/scripts/build-windows-agent-hook.mjs @@ -1420,8 +1339,7 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts @files - # Why its own step: the CPU burners that make taskkill slow would starve - # every other test file sharing the runner. + # Isolate slow-taskkill CPU burners so they cannot starve unrelated test files. - name: Test Windows tree kill under CPU load env: ORCA_WINDOWS_TREE_KILL_LOAD: '1' @@ -1429,9 +1347,7 @@ jobs: pnpm exec vitest run --config config/vitest.config.ts src/shared/__tests__/process-host/windows-tree-kill-late-taskkill.win32.test.ts - # Why the :parallel variant: identical to build:release except the three - # electron-vite targets overlap instead of running back to back. The Linux package - # job already packages and smoke-tests an AppImage built that way. + # Use parallel Vite targets as already packaged/smoked on Linux; release behavior is otherwise identical. - name: Build package inputs env: ORCA_CI_MAIN_SOURCEMAPS: '0' @@ -1482,10 +1398,7 @@ jobs: needs_build: ${{ needs.code_paths.outputs.e2e_needs_build != 'false' }} managed_ssh_shards: ${{ needs.code_paths.outputs.e2e_ssh_shards || '["1/2","2/2"]' }} - # Why its own required job, apart from `e2e`: the rest of the suite is red on main, but the - # layout oracle is green there, and its known-on-main list absorbs today's findings. Only the two - # oracle specs run, through the same reusable workflow: the window spec in the changed-spec lane, - # the headless spec in the orcad lane on both serve hosts. + # Gate the green layout oracles separately from main’s red broader E2E suite: changed-spec window plus both orcad serve hosts. layout_oracle: name: layout oracle needs: [code_paths, preflight] @@ -1505,12 +1418,7 @@ jobs: needs_build: true artifact_suffix: -layout-oracle - # Why this is not in verify's needs: it is the first PR-gate run of a harness whose reliability - # is only known from nightly main runs (20/20 green, 2026-08-09..2026-08-29, p50 3m25s). It - # reports a red X on the PR without blocking, exactly like `e2e` above. Deliberately no - # continue-on-error: that renders the check green and hides the signal it exists to give. To - # make it blocking, add it to verify.needs, add TERMINAL_IME_NATIVE to the env below, and - # require `success || skipped` outside the strict loop — see the note on `e2e`. + # Keep native IME advisory while PR reliability is established (nightly 20/20, 2026-08-09..29, p50 3m25s); do not hide red via continue-on-error. To gate, add verify.needs, TERMINAL_IME_NATIVE env and separate success-or-skipped validation. terminal_ime_native: name: real IME needs: code_paths @@ -1530,9 +1438,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - # Why: pnpm's global virtual store links each package to only its declared dependencies, which - # is how developers install. Hoisting hides an undeclared one everywhere else, so install that way - # here with HOME and the store in an empty temp dir, where no stray node_modules can mask it. + # Install like developers’ global virtual store in empty HOME/store; hoisting or stray dependencies can mask undeclared imports. dependency_isolation: name: dependency isolation under the global virtual store needs: [code_paths] @@ -1546,8 +1452,7 @@ jobs: ORCA_PNPM_FORCE_GLOBAL_VIRTUAL_STORE_IN_CI: '1' steps: - # Job-wide (via GITHUB_ENV, since job env cannot read runner.temp) so every pnpm command, - # including pnpm run's own install check, keeps this layout. + # Persist layout via GITHUB_ENV (job env cannot read runner.temp) for every pnpm command/install check. - name: Point HOME and the pnpm store at empty temp dirs run: | mkdir -p "$RUNNER_TEMP/empty-home" @@ -1613,14 +1518,7 @@ jobs: timeout-minutes: 5 steps: - # Why: e2e is deliberately absent from needs. The suite is currently red on - # main (every scheduled run), so gating merges on it would block any PR that - # touches tests/e2e/** — including the ones fixing the suite. Until it is - # green the job runs and reports for E2E-path PRs without blocking. To flip - # it on: add `e2e` to needs, add E2E to the env below, and require - # `"$E2E" = success || skipped` after the loop — skipped is the normal - # result for a path-filtered job and must keep passing, so it has to be - # checked outside the loop or it would excuse the jobs above. + # Keep E2E advisory while main’s scheduled suite is red, allowing fixes to merge. To gate, add e2e to needs/E2E env and check success-or-skipped outside the strict required-job loop. - name: Require successful checks env: CODE_PATHS: ${{ needs.code_paths.result }} diff --git a/.github/workflows/release-cut.yml b/.github/workflows/release-cut.yml index 33ac5e78213..7ae1f1d613e 100644 --- a/.github/workflows/release-cut.yml +++ b/.github/workflows/release-cut.yml @@ -1,22 +1,6 @@ name: Cut Release -# Why: single entry point for manually cutting releases. -# Replaces the old local `pnpm release:*` scripts and the standalone scheduled -# RC workflow so releases are always reproducible from CI and can never be -# accidentally tagged against an uncommitted or non-main working tree. -# -# Flow: -# 1. Resolve `ref` to a SHA. -# 2. Read the latest stable release from GitHub. -# 3. Compute the next version from `kind` (rc | patch | minor | major). -# 4. For stable kinds, REFUSE if the new version is <= the latest stable. -# This is the only guard electron-updater actually needs — it compares -# semver within a channel, so a regressing "latest" is the one thing -# that breaks auto-update for fresh installs. -# 5. Write package.json, commit (detached), tag, push tag. -# 6. If ref was the tip of origin/main, try to fast-forward its version; -# a pull-request-only rule leaves main unchanged and still pushes the tag. -# 7. Build and publish artifacts from the tag. +# Resolve ref/latest stable, compute next kind, reject stable version regression, commit/tag detached and build/publish from tag. Fast-forward main’s version only when source was its tip and policy permits; PR-only rules leave main unchanged. CI replaces local/scheduled release paths to avoid accidental working-tree releases. on: workflow_dispatch: @@ -61,10 +45,7 @@ concurrency: jobs: cut: - # Why: this job bumps package.json and fast-forwards main. On a fork with - # Actions enabled, the scheduled cut would run against the fork's main and - # diverge it (version line) every slot, conflicting every PR back upstream. - # Gate to the canonical repo so the workflow no-ops on forks. + # Limit version/tag mutations to canonical repo; fork schedules must not diverge their main and conflict upstream. if: github.repository == 'stablyai/orca' runs-on: ubuntu-latest timeout-minutes: 15 @@ -81,13 +62,7 @@ jobs: source_short_sha: ${{ steps.resolve.outputs.short_sha }} ships_orcad_template: ${{ steps.orcad-template-support.outputs.ships }} steps: - # Why inlined (not m-s-abeer/update-gha-summary-with-workflow-inputs): - # this job runs with contents:write and secret scope, so avoid executing - # any external (mutable @v1) action here. Surfaces every - # workflow_dispatch input as a table for audit; the resolved commit / - # branch / tag enrichment is written later in "Resolve ref SHA". - # Inputs are passed as JSON via env and parsed by jq as data — never - # interpolated into the shell — to avoid injection from dispatch values. + # Render dispatch audit summary locally rather than run mutable external action beside write token/secrets; parse env JSON with jq as data. - name: Summarize workflow inputs if: github.event_name == 'workflow_dispatch' env: @@ -98,8 +73,7 @@ jobs: echo "" echo "| Input | Value |" echo "| --- | --- |" - # Values are data from env JSON; wrap in backticks for readability. - # Newlines collapsed so a multi-line input cannot break the table. + # Treat env JSON as data; collapse newlines so inputs cannot break the summary table. jq -r '(. // {}) | to_entries[] | "| `\(.key)` | `\(.value | tostring | gsub("\n"; " "))` |"' <<<"$INPUTS_JSON" } >> "$GITHUB_STEP_SUMMARY" @@ -110,9 +84,7 @@ jobs: fetch-depth: 0 # Version recovery needs complete history and tags, but not historical file contents. filter: blob:none - # Why: version math recovers unpublished tags; checkout's default - # fetch-tags:false hides them, so a patch cut recreates vX.Y.Z and - # `git push` overwrites the existing tag. + # Fetch unpublished tags for version recovery; hidden tags can recreate/overwrite an existing version. fetch-tags: true # Why: a relay-era ref (before #27215) needs the Windows relay addon this build no longer compiles. @@ -138,9 +110,7 @@ jobs: - name: Resolve ref SHA id: resolve env: - # Why: keep the caller's ref as data (env) so we can label it in the - # summary without shell-interpolating a dispatch-controlled string - # into the script body. + # Keep caller ref in env data so summary labels cannot inject shell. INPUT_REF: ${{ github.event_name == 'schedule' && 'main' || inputs.ref }} REPO: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} @@ -153,9 +123,7 @@ jobs: echo "sha=$sha" >>"$GITHUB_OUTPUT" echo "short_sha=$short_sha" >>"$GITHUB_OUTPUT" - # Why: only push the version-bump commit back to main when the - # caller is releasing the exact tip of main. For any older or - # off-main ref we leave main alone and only publish the tag. + # Push the version bump to main only when cutting its exact tip; older/off-main refs publish tags only. git fetch origin main --quiet main_sha="$(git rev-parse origin/main)" if [[ "$sha" == "$main_sha" ]]; then @@ -164,11 +132,7 @@ jobs: echo "push_main=false" >>"$GITHUB_OUTPUT" fi - # Always surface the resolved commit in the job summary, plus any - # branches/tags that currently point at it (clickable). The raw - # `ref` input alone is ambiguous (branch vs tag vs SHA); for SHA - # inputs it also hides the human-readable names operators need - # when auditing RC cuts. + # Show the immutable commit and matching ref links: raw ref input is ambiguous and bare SHAs hide operator-readable names. repo_url="${SERVER_URL}/${REPO}" branches="$( @@ -184,8 +148,7 @@ jobs: || true )" - # Build comma-separated markdown links. Branch/tag names go in the - # URL path as-is (slashes must stay literal for GitHub tree URLs). + # Preserve literal slashes in branch/tag tree URLs. linkify_names() { local url_kind="$1" local names="$2" @@ -215,8 +178,7 @@ jobs: branch_md="$(linkify_names branch "$branches")" tag_md="$(linkify_names tag "$tags")" - # When no branch tip matches (historical SHA cuts), fall back to - # name-rev so the summary still shows something like `main~3`. + # Historical SHAs may lack a matching branch tip; name-rev supplies an auditable main~N label. contains_md="_none_" if [[ "$branch_md" == "_none_" ]]; then approx="$(git name-rev --name-only --no-undefined --refs='refs/remotes/origin/*' "$sha" 2>/dev/null || true)" @@ -274,8 +236,7 @@ jobs: pt_hour=$(TZ=America/Los_Angeles date '+%H') pt_minute=$(TZ=America/Los_Angeles date '+%M') - # Why: GitHub may deliver a scheduled event long after the intended - # time, so delayed 4:16 AM runs must not cut the 3:00 AM release. + # Delayed schedule delivery (e.g. 4:16 AM) must not cut the 3:00 AM slot. if [[ "$pt_hour" == "03" || "$pt_hour" == "15" ]]; then echo "allowed=true" >>"$GITHUB_OUTPUT" echo "reason=target_hour:${pt_hour}:${pt_minute}" >>"$GITHUB_OUTPUT" @@ -289,15 +250,13 @@ jobs: id: existing if: github.event_name == 'schedule' && steps.window.outputs.allowed == 'true' run: | - # Why: scheduled runs retry inside each target hour, so make the - # schedule idempotent by embedding a slot marker in the release commit. + # Repeated target-hour schedules deduplicate via a slot marker in the release commit. if git log origin/main --grep="\\[rc-slot:${{ steps.slot.outputs.value }}\\]" -n 1 --format=%H | grep -q .; then echo "already_ran=true" >>"$GITHUB_OUTPUT" exit 0 fi - # Why: this preserves dedupe across the older scheduled workflow's - # first runs, before all RC cuts shared release-cut's slot marker. + # Preserve dedupe for older scheduled workflows that predate the shared slot marker. latest_rc_tag="$(git for-each-ref --sort=-creatordate --format='%(refname:short) %(creatordate:iso-strict)' 'refs/tags/v*-rc.*' | head -n 1)" if [[ -n "$latest_rc_tag" ]]; then latest_rc_tag_name="${latest_rc_tag%% *}" @@ -334,8 +293,7 @@ jobs: - name: Publish complete release-cut RC drafts from prior runs id: publish_drafts - # Why: a manual RC dispatch should unstick any complete RC draft before - # deciding whether to cut another tag. + # Recover complete RC drafts before deciding whether manual dispatch needs another tag. if: steps.window.outputs.allowed == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.dry_run) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -343,9 +301,7 @@ jobs: - name: Compute next version id: version - # Why: if an RC run only had complete drafts to publish, stop there - # instead of immediately cutting another RC after the recovered one. - # Stable dispatches should still cut the requested stable release. + # Stop after RC-only draft recovery; stable dispatch still cuts its requested version. if: steps.window.outputs.allowed == 'true' && steps.existing.outputs.already_ran != 'true' && !(github.event_name == 'workflow_dispatch' && inputs.dry_run) && !((github.event_name == 'schedule' || inputs.kind == 'rc') && steps.publish_drafts.outputs.published_count != '0' && steps.publish_drafts.outputs.skipped_count == '0') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -355,35 +311,16 @@ jobs: run: | set -euo pipefail - # Latest stable release tag, picked by *tag shape* and semver max: - # - must start with `v` (desktop convention, e.g. v1.3.32) - # - must NOT contain `-rc.` (not a prerelease) - # - # Why not the GitHub `isPrerelease` flag: electron-builder's publish - # step has flipped that flag back to `false` on RC releases before - # (v1.3.22-rc.2 on 2026-04-27 briefly became "latest" on GitHub and - # poisoned the math here). Tag format is authoritative. - # - # Why the `^v[0-9]` prefix: other products shipped from this repo - # use their own prefixes (e.g. `mobile-v0.0.1`). Without the prefix - # gate the latest mobile release would be selected as "latest - # stable", then `strip_pre()` would reduce `mobile-v0.0.1` to - # `mobile`, `Number("mobile")` → NaN → 0, and a patch-bump would - # produce `0.0.1` — exactly the wedge on 2026-05-04 (run - # 25304336767). Any non-desktop tag shape must be excluded here. - # - # Why not `gh release list` order: GitHub can list a newer published - # stable after older releases. On 2026-06-04, v1.4.44 existed but - # the list returned v1.4.42 first, causing a manual RC cut to reopen - # the already-shipped 1.4.43 series as v1.4.43-rc.0. + # Use desktop v tag shape and numeric semver max, excluding RC/mobile tags. + # Builder flipped v1.3.22-rc.2’s prerelease flag on 2026-04-27; that flag cannot authorize stable math. + # 2026-05-04/run25304336767: mobile-v0.0.1 collapsed through NaN to 0.0.1 without the desktop prefix. + # 2026-06-04: release-list order put v1.4.42 before v1.4.44 and reopened 1.4.43 as rc.0. latest_stable="$(node config/scripts/latest-stable-release.mjs)" latest_stable="${latest_stable#v}" echo "Latest stable: ${latest_stable:-}" - # Strip any prerelease suffix before numeric math. Without this, - # `Number("1-rc")` returns NaN and `(NaN||0)+1` silently collapses - # to 1 — exactly the path that produced v1.3.1-rc.4 on 2026-04-27 - # when latest_stable was misread as a prerelease tag. + # Strip prerelease suffixes before numeric math: Number("1-rc") becomes NaN and silently collapsed + # to 1, producing v1.3.1-rc.4 on 2026-04-27 when latest_stable was misclassified. strip_pre() { echo "${1%%-*}"; } semver_gt() { @@ -415,16 +352,8 @@ jobs: } require_valid_version_suffix() { - # Why a dot-appended identifier (rc.N.perf): it sorts just - # above its own base rc.N but BELOW rc.N+1, so suffixed side- - # branch builds never outrank the main RC series and cannot - # hijack the update channel; clients find them by matching the - # identifier ("perf") in the prerelease components. - # Why the numeric alternation rather than plain [0-9A-Za-z]+: - # semver forbids a leading zero on an all-digit identifier, and - # `npm version` silently renormalizes rc.4.01 to rc.4.1 while the - # tag step keeps the literal input — so the shipped package.json - # version and its own release tag would name different releases. + # Append rc.N. so side builds sort above rc.N but below rc.N+1; clients match the suffix. + # Reject leading-zero numeric identifiers: npm normalizes rc.4.01 to rc.4.1 while tagging keeps the input. if [[ ! "$1" =~ ^(0|[1-9][0-9]*|[0-9A-Za-z]*[A-Za-z][0-9A-Za-z]*)$ ]]; then echo "::error::version_suffix (or the trailing .identifier in version) must be alphanumeric with no leading zero on an all-digit identifier, got: $1" >&2 exit 1 @@ -502,18 +431,10 @@ jobs: package_stable="$(current_package_stable)" if [[ -n "$package_stable" ]]; then - # Why: if a stable release is deleted after its version-bump commit - # reached main, GitHub's release list regresses. package.json is the - # floor for the current ref so the next cut cannot reuse an older - # stable number just because the public release was nuked. + # Deleted releases can regress GitHub’s version list after their bump reached main; package.json floors the current ref. if semver_gt "$package_stable" "$latest_stable"; then - # Skip floor-tag recovery when an explicit version is requested: - # recover_unpublished_tag can exit 0, which would recover the - # package-floor tag instead of cutting the requested version — - # defeating the very rollback scenario the override exists for. - # We still raise latest_stable to the floor below so the explicit - # version is gated against it; the collision recovery for the - # requested tag runs later. + # Explicit versions must skip floor-tag recovery (which can exit successfully with the wrong tag). + # Still enforce the package floor; recover collisions for the requested tag later. if [[ "$KIND" != "rc" && -z "${EXPLICIT_VERSION:-}" ]]; then package_tag="v$package_stable" if git rev-parse "$package_tag" >/dev/null 2>&1; then @@ -526,84 +447,44 @@ jobs: fi fi - # Explicit version override (manual dispatch only). - # - # Why: kind-based math derives the next number from the latest - # *published* stable. When a shipped stable is deleted (e.g. a - # rolled-back 1.4.154), the release list regresses to the prior - # stable, so a kind cut recomputes a number at or below the nuked one - # and strands every client that already installed the deleted build. - # The package.json floor above only recovers this when the deleted - # version's bump commit is on the ref being cut, which a hotfix cut - # from an older RC ref does not carry. An explicit version lets a - # human assert the exact target (e.g. leapfrog to 1.4.155); the - # updater-safety gate and tag-collision recovery below still apply. + # Manual explicit versions let hotfixes from older refs leapfrog deleted shipped versions (e.g. 1.4.154 → 1.4.155) + # when published-release math regresses and the ref lacks the newer package floor. Keep updater/tag-collision gates. new="" if [[ -n "${EXPLICIT_VERSION:-}" ]]; then explicit="${EXPLICIT_VERSION#v}" - # Why the optional trailing identifier: it lets an operator name a - # suffixed side-branch RC (X.Y.Z-rc.N.perf) directly, the same shape - # the rc path cuts. Note this only ever admits one *above* the - # series head — the gate below refuses a suffixed rc at or below it - # just like a bare one, so this is a second spelling of - # `version=X.Y.Z-rc.N` + `version_suffix`, not a way back into a - # series that already shipped. - # Why rc.(0|[1-9][0-9]{0,8}): the `-le` below compares with bash's - # machine-width integers, so both ends of that range fall *open* on - # exactly the RCs this gate must catch. A leading zero (rc.08) is an - # invalid octal literal, and the failed test makes the `if` false. - # Past INTMAX the literal wraps two's-complement, so whether it - # reads as above or below the published rc depends on the value: - # rc.99999999999999999999 wraps to 7766279631452241919 and sails - # through. The cut then lands a tag that pins highest_rc_for_base - # at 1e20 forever, and every later cut wraps to a *lower* rc that - # sorts below it, so the fleet never updates again. Nine digits is - # far above any real series and exact in bash math either way. + # Accept one optional side-branch RC identifier, still strictly above the series head; it grants no rollback exception. + # Bound rc ordinals to nine digits without leading zeros: Bash -le misreads rc.08 as octal and wraps INTMAX, + # letting rc.99999999999999999999 pass as 7766279631452241919 and pin highest_rc_for_base at 1e20 forever. if [[ ! "$explicit" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-rc\.(0|[1-9][0-9]{0,8})(\.[0-9A-Za-z]+)?)?$ ]]; then echo "::error::version must be X.Y.Z, X.Y.Z-rc.N, or X.Y.Z-rc.N.suffix, got: $EXPLICIT_VERSION" >&2 exit 1 fi - # Why route the embedded identifier through the same validator the - # kind path uses: the regex above only checks shape, and rc.4.01 - # is a shape-valid identifier that is not valid semver. + # Reuse identifier validation: rc.4.01 matches shape but is invalid semver. if [[ "$explicit" == *-rc.*.* ]]; then require_valid_version_suffix "${explicit##*.}" fi - # Same updater-safety gate the kind path enforces: stable line must - # strictly increase over the latest published stable (prerelease - # identifiers ignored for the comparison). + # Explicit stable lines must strictly exceed published stable, ignoring prerelease identifiers. if ! semver_gt "$explicit" "$latest_stable"; then echo "::error::Refusing explicit version $explicit: not greater than latest stable $latest_stable." >&2 exit 1 fi - # Why a second gate for prereleases: semver_gt compares through - # strip_pre(), so the stable-line check reads 1.4.156-rc.0 as - # 1.4.156 and waves it past a 1.4.155 stable even when rc.0..rc.3 - # already shipped — republishing an RC *below* what clients run, - # the same regression class as the rc.4 cut that orphaned live - # daemons. Anchor on the same rc history the kind path uses so the - # override can only ever advance the series it targets. + # Stable-line comparison alone admits rc.0 after rc.3 shipped. Require the override to advance its + # own series too, avoiding the rc.4 regression that orphaned live daemons. if [[ "$explicit" == *-rc.* ]]; then explicit_base="${explicit%%-*}" explicit_rc="${explicit#*-rc.}" explicit_rc="${explicit_rc%%.*}" highest_explicit_rc="$(highest_rc_for_base "$explicit_base")" if [[ -n "$highest_explicit_rc" && "$explicit_rc" -le "$highest_explicit_rc" ]]; then - # Why the remedy is spelled this narrowly: kind=rc derives its - # base from bump(latest_stable, patch), so it can only resume a - # series on that base. A minor/major series (1.5.0-rc.N) exists - # only because this override created it, and pointing an - # operator at kind=rc there would cut an unrelated release. + # kind=rc resumes only latest stable’s next patch; minor/major RC series require explicit overrides, + # or the suggested remedy would cut an unrelated release. echo "::error::Refusing explicit version $explicit: rc.$explicit_rc is not above rc.$highest_explicit_rc, the highest already cut for $explicit_base. Request rc.$((highest_explicit_rc + 1)) or higher. If you are resuming an unpublished tag and $explicit_base is the next patch after latest stable $latest_stable, dispatch kind=rc instead, which recovers that tag when it was cut from the ref you dispatch; otherwise cut rc.$((highest_explicit_rc + 1)) and leave the unpublished tag alone." >&2 exit 1 fi fi new="$explicit" - # Why here too: the suffix append below lives in the kind path the - # override skips, so an operator passing both inputs used to get - # their suffix silently dropped. Only a bare rc can take one — a - # stable X.Y.Z.perf is not valid semver, and re-suffixing an - # already-suffixed rc would produce rc.N.perf.perf. + # The override bypasses kind-path suffixing: append here too, only to bare RCs, avoiding invalid + # stable X.Y.Z.perf or repeated rc.N.perf.perf suffixes. if [[ -n "${VERSION_SUFFIX:-}" ]]; then # Same bounded rc pattern as the shape check above, so the two # cannot drift apart under a later edit. @@ -620,23 +501,15 @@ jobs: if [[ -z "$new" ]]; then case "$KIND" in rc) - # Why: RCs always stabilize the *next* patch after whatever - # is currently published as stable. Earlier logic tried to - # "continue the current series" by reading the highest git - # tag, which silently reopened a series that had already - # shipped (e.g. cutting v1.3.21-rc.7 after v1.3.21 stable - # was out). Anchoring to latest_stable + patch eliminates - # that class of bug; minor/major RCs are cut by running - # that stable kind first. + # RCs target published stable + next patch. Highest-tag continuation reopened shipped series + # (v1.3.21-rc.7 after v1.3.21 stable); cut minor/major stable first to start those RC bases. base="$(bump "$latest_stable" patch)" highest_rc="$(highest_rc_for_base "$base")" if [[ -z "$highest_rc" ]]; then new="${base}-rc.0" else existing_rc_tag="v${base}-rc.${highest_rc}" - # Why: a failed or GitHub-stuck run can leave the highest RC - # tag attached to a draft/missing release. Resume only when it - # was cut from this ref; stale attempts advance to rc.N+1. + # Resume highest draft/missing RC only when cut from this ref; stale attempts advance to rc.N+1. if git rev-parse "$existing_rc_tag" >/dev/null 2>&1; then recover_unpublished_tag "$existing_rc_tag" "latest RC in series" || true fi @@ -655,10 +528,7 @@ jobs: exit 1 fi - # Why: a stale orphan stable tag can exist from an older release - # ref after main has moved on. If it cannot be recovered for the - # current ref, advance to the next stable version instead of - # wedging every future patch cut on the same collision. + # If an old orphan stable tag cannot be recovered for this ref, advance rather than wedge future patch cuts. for _ in {1..100}; do candidate_tag="v$new" if ! git rev-parse "$candidate_tag" >/dev/null 2>&1; then @@ -689,29 +559,9 @@ jobs: esac fi - # Orphan-tag recovery. - # - # Why: if a previous cut pushed the tag but was cancelled (or the - # dependent release build jobs otherwise failed to start) before the - # GitHub Release was published, the tag now exists on the remote - # but "latest stable" still points at the prior version. Every - # subsequent patch cut then recomputes the same version and dies - # on "Tag already exists." This exact sequence wedged the cut - # pipeline on 2026-05-01 when v1.3.26 was pushed by a cancelled - # run (25237882049) — every patch cut after that rehit the same - # tag for hours until the orphan release was dispatched by hand. - # - # Recovery policy: if the tag exists AND no GitHub release has - # been published for it (draft-or-absent both count as "not - # shipped"), treat this as a resumable state: emit the existing - # tag as the job output so the downstream release build jobs run - # against it and finishes what the earlier attempt started. The - # bump/commit/push steps are skipped in that case — there is - # nothing to bump; the tag is already on the remote. - # - # Refuse collisions only when the tag *and* a published release - # already exist — that's a real conflict (someone tagged manually - # over a shipped version) and needs human attention. + # Recover existing draft/absent releases by emitting the pushed tag and rerunning builds without bump/commit/push. + # 2026-05-01/run25237882049: cancellation after pushing v1.3.26 wedged every patch cut on that orphan tag + # until manual release dispatch. A tag plus published release is a real collision requiring operator attention. if git rev-parse "v$new" >/dev/null 2>&1; then recover_unpublished_tag "v$new" "tag collision" || { echo "::error::Tag v$new already exists and cannot be recovered for this ref. Refusing to re-cut over an existing version." >&2 @@ -731,12 +581,9 @@ jobs: VERSION: ${{ steps.version.outputs.version }} run: | set -euo pipefail - # Why: use npm version --no-git-tag-version so we control the commit - # message and tag name explicitly (avoids npm's `v1.2.3` prefix - # assumptions and any lifecycle scripts that would run on bump). + # Control commit message/tag ourselves via npm version --no-git-tag-version, avoiding npm’s prefix/lifecycle assumptions. npm version "$VERSION" --no-git-tag-version --allow-same-version - # Why: freshness compares installs against the bundled manifest, so a tag must - # never ship skill bytes it doesn't describe. Node built-ins only; no install. + # Tags must describe their bundled skill bytes; verify freshness with Node built-ins before install. if ! node config/scripts/generate-skill-bundle-manifest.mjs; then echo "::error::Refusing to tag v$VERSION: the committed skill manifest does not match skills/. Land a regeneration on main, then re-run the cut." >&2 exit 1 @@ -747,23 +594,14 @@ jobs: commit_message="$commit_message [rc-slot:$SLOT]" fi if git diff --cached --quiet; then - # Why: a failed cut can push the version bump to main before the - # release is published. Re-cutting then needs a fresh taggable - # release commit even though package.json is already at VERSION. + # A failed cut may already have bumped main; re-cut still needs a fresh taggable commit. git commit --allow-empty -m "$commit_message" else git commit -m "$commit_message" fi - # Why: a lint that greps this file cannot see a path built from an env - # var, a composite action, or concatenation, and `git commit` has forms - # (-a, -i, --only, a pathspec) that commit the working tree rather than - # the index. Assert what the commit actually carries, so the tag can - # only ever ship the version bump, no matter which step staged what or - # how the commit was spelled. - # -F because the allowlist is literal: unanchored, `.` would match any - # character and quietly admit a path like `packageXjson`. - # -m --first-parent: plain diff-tree prints NOTHING for a merge commit, - # which would make this guard pass silently rather than fail closed. + # Validate the committed tree, not staged intent: git commit -a/-i/--only/pathspec can bypass the index. + # Literal -F matching cannot accidentally admit packageXjson; -m --first-parent makes merge commits visible + # to diff-tree so the allowlist cannot silently pass an empty merge diff. committed="$(git diff-tree --no-commit-id --name-only -r -m --first-parent HEAD | grep -vxF -e 'package.json' || true)" if [[ -n "$committed" ]]; then @@ -924,10 +762,7 @@ jobs: npm install -g node-gyp@11.5.0 echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV" - # Why: this install runs lifecycle scripts, so node-gyp rebuilds - # native/windows-registry and fetches that Node version's headers from - # nodejs.org. One `read ECONNRESET` there failed this blocking gate and the - # whole cut. Retry like the release build's install below. + # Retry node-gyp header fetch ECONNRESET so transient install failures cannot fail the whole release. - name: Install dependencies uses: nick-fields/retry@v4 with: @@ -939,10 +774,7 @@ jobs: - name: Build Electron app for platform golden run: npx electron-vite build --mode e2e - # Why: this job is defined on the dispatch ref (usually main) but checks - # out the release tag. Cherry-pick / hotfix tags can predate a golden - # script that main already calls; --if-present keeps those cuts green - # instead of failing with ERR_PNPM_NO_SCRIPT. + # Use --if-present for golden scripts absent from older hotfix/cut tags, despite newer dispatch YAML. - name: Run terminal rendering golden on Linux if: runner.os == 'Linux' run: | @@ -1186,9 +1018,7 @@ jobs: retention-days: 7 if-no-files-found: ignore - # Design D2: every desktop build ships the orcad template (server JS plus every target's - # addons), merged from the node-server lanes that qualified each slot at this tag. It needs no - # signing quota, so it runs beside the release gates instead of behind them. + # Build every target’s qualified orcad slot at the tag beside release gates; unsigned template consumes no signing quota. orcad-template: needs: cut if: needs.cut.outputs.should_release == 'true' && needs.cut.outputs.ships_orcad_template == 'true' @@ -1200,9 +1030,7 @@ jobs: ref: refs/tags/${{ needs.cut.outputs.tag }} fast_runners: false - # Why: artifact jobs submit Windows binaries to SignPath. Keep every - # quota-consuming build behind all blocking release gates so a late test - # failure cannot create signing requests that can never be published. + # Gate SignPath artifact requests behind blocking tests to avoid approvals for unpublishable builds. release-preflight: needs: - cut @@ -1285,9 +1113,7 @@ jobs: ~/.cache/electron-builder runs-on: ${{ matrix.os }} - # Why: hosted runners hard-cap jobs at 6h; the Windows SignPath waits - # (1h inner + 4h installer) are budgeted to fit under this with the - # build itself, so a slow approval can't kill the job mid-flow. + # Fit 1h inner+4h installer SignPath waits and build under hosted runner’s 6h cap. timeout-minutes: 360 permissions: @@ -1310,16 +1136,9 @@ jobs: echo "Windows artifact/signing steps are disabled on reruns (attempt $GITHUB_RUN_ATTEMPT)." echo "Existing signed release assets must be reused; dispatch a fresh release only when a rebuild is required." >> "$GITHUB_STEP_SUMMARY" - # Why: `uses: ./…` resolves from the checked-out tag, not from the workflow - # ref, so cutting from an older/off-main ref whose tree predates a composite - # action would fail the step with "Can't find 'action.yml'". Restore the - # actions directory from the commit this workflow file itself came from. - # Not Windows-only: every platform now consumes install-mobile-dependencies, so - # any of them can be the one whose cut ref predates the action. + # Restore composite actions from workflow commit: uses:./ resolves checked-out tag, which may predate mobile/dependency actions. - name: Restore draft-publish scripts from the workflow ref - # Why: this job checks out the release tag, so a cut from an older SHA - # still has electron-builder releaseType:release and no re-draft helper. - # The workflow YAML is from main; restore the scripts it invokes. + # Restore current workflow helper scripts when old cut tags still publish live and lack re-draft support. shell: bash env: WORKFLOW_SHA: ${{ github.workflow_sha }} @@ -1370,23 +1189,14 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Why: release builds hit the same native-module postinstall path as - # PR CI, so keep the pinned node-gyp override here too instead of - # relying on pnpm's bundled copy. Scoped to Linux via runner.os (not - # a specific matrix image) because the failing postinstall has only - # been observed on Linux runners — see run 25081763129. The macOS - # and Windows release jobs exercise the same pnpm install path and - # have not reproduced it, so keep the gate narrow until we know why. - # Using runner.os instead of matrix.os == 'ubuntu-latest' means the - # gate still works if another Linux matrix entry is added later. + # Override non-executable pnpm node-gyp only on Linux (run 25081763129); runner.os keeps future Linux entries covered. - name: Use external node-gyp to avoid pnpm's bundled copy (Linux only) if: runner.os == 'Linux' run: | npm install -g node-gyp@11.5.0 echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV" - # Cache the Electron binary + electron-builder tool downloads - # (winCodeSign, nsis, squirrel, AppImage). Saves ~30-90s per job. + # Cache Electron and builder downloads (winCodeSign/NSIS/Squirrel/AppImage), saving ~30–90s per job. - name: Cache electron-builder downloads uses: actions/cache@v5 with: @@ -1405,12 +1215,7 @@ jobs: # The release cache above restores the same tools, so this only needs to save on a miss. lookup-only: true - # Why: pnpm install triggers electron's postinstall, which downloads the - # Electron binary from GitHub release assets. GitHub's download CDN - # occasionally returns 504s that fail the whole release. Retry on - # failure so transient network errors don't require a manual re-run. - # Why host-only: this job packages only for its own runner OS and - # architecture, so the default host-scoped install is deliberate. + # Retry transient Electron CDN 504s during install; keep host-scoped native dependencies for this platform’s package. - name: Install dependencies uses: nick-fields/retry@v4 with: @@ -1419,28 +1224,15 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile - # Why here: electron-builder's beforePack requires out/mobile-web, and the bundle - # build resolves React Native and Expo from mobile/node_modules. + # Build required out/mobile-web from mobile’s React Native/Expo dependencies. - uses: ./.github/actions/install-mobile-dependencies - # Why: `pnpm build:release` verifies the Linux computer-use provider by - # importing AT-SPI bindings, which are runtime package deps but are not - # present on stock GitHub Ubuntu release runners. - # Why: `rpm` is needed by electron-builder's fpm backend to produce the - # .rpm artifact. Stock Ubuntu runners do not ship it. + # Install AT-SPI runtime bindings for Linux provider verification and rpm for builder’s fpm backend; stock Ubuntu lacks them. - name: Install Linux computer-use provider dependencies if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y python3-gi gir1.2-atspi-2.0 at-spi2-core xclip xdotool rpm - # Why: telemetry's transport gate (`src/main/telemetry/client.ts:IS_OFFICIAL_BUILD`) - # requires the build identity to be the literal string `stable` or `rc`, - # substituted by electron-vite's `define` block at build time. Derive - # that identity from the release tag here — `stable` for plain semver - # (`vX.Y.Z`), `rc` for prerelease (`vX.Y.Z-rc.N`). The strict regex is - # a safety net: this workflow only fires on cut-tags that already match - # one of those shapes, but if a future change ever loosens that, we - # refuse to ship rather than let an unclassified build go out with - # `BUILD_IDENTITY = null`. + # Derive literal stable/rc telemetry identity from strict tag shape; refuse unclassified builds rather than ship silent null identity (#1385 follow-up). - name: Classify release tag for telemetry build identity id: tag-classify shell: bash @@ -1448,9 +1240,7 @@ jobs: TAG: ${{ needs.cut.outputs.tag }} run: | set -euo pipefail - # Why the optional trailing identifier: suffixed side-branch RCs - # (vX.Y.Z-rc.N.perf) are rc-channel prerelease builds — same telemetry - # identity as plain RCs. + # Suffixed vX.Y.Z-rc.N.perf builds keep the same rc-channel telemetry identity. if [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+(\.[0-9A-Za-z]+)?$ ]]; then identity=rc elif [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -1462,10 +1252,7 @@ jobs: echo "identity=$identity" >>"$GITHUB_OUTPUT" echo "Classified $TAG as $identity" - # Why an explicit step rather than trusting the runner image: the packaged - # CLI launcher is a native Rust binary, and a Windows host without cargo - # fails deep inside electron-builder's native hook instead of here. Real - # Windows dev machines do not have it by default either. + # Require cargo before builder invokes the packaged Rust launcher hook. - name: Ensure the Rust toolchain for the Windows CLI launcher if: matrix.platform == 'win' shell: pwsh @@ -1514,15 +1301,11 @@ jobs: - name: Gate runtime file-watcher process isolation if: runner.os == 'Linux' run: | - # Why: #8212 is a native-process crash contract. Prove both the Node - # host and the exact Electron runtime survive SIGSEGV before packaging. + # #8212: require Node and the exact Electron runtime to survive native SIGSEGV before packaging. node config/scripts/runtime-file-watcher-fault-harness.mjs ELECTRON_RUN_AS_NODE=1 pnpm exec electron config/scripts/runtime-file-watcher-fault-harness.mjs - # Why: main ships minified with sourcemap:'hidden' and packaging drops - # out/**/*.map from app.asar, so a crash trace from a released build is - # otherwise undecodable. The main bundle is platform-independent, so one - # leg publishes the maps for the whole release. + # Publish hidden main sourcemaps separately because app.asar drops maps; one platform suffices for platform-independent main bundle. - name: Bundle main-process source maps id: bundle-main-sourcemaps if: matrix.platform == 'linux-x64' @@ -1532,8 +1315,7 @@ jobs: run: | set -euo pipefail if [ -z "$(find out/main -name '*.js.map' -print -quit)" ]; then - # Older cut tags predate the hidden-source-map build setting. They - # are valid legacy releases, but have no map bundle to publish. + # Old valid cut tags predate hidden sourcemaps and have no bundle to publish. if grep -Eq "sourcemap:[[:space:]]*['\"]hidden['\"]" electron.vite.config.ts; then echo "::error::No main-process source maps in out/main despite build.sourcemap='hidden'." exit 1 @@ -1543,9 +1325,7 @@ jobs: exit 0 fi echo "has_maps=true" >>"$GITHUB_OUTPUT" - # Why: every entry in electron-builder's `files` is a negation, so - # app-builder prepends `**/*` and packs anything left in the workspace - # root into app.asar. Stage the bundle outside the checkout instead. + # All-negation builder files prepend **/*; stage outside checkout to avoid packing maps into app.asar. find out/main -name '*.js.map' -print | sort | zip -q -X "$RUNNER_TEMP/orca-sourcemaps-$TAG.zip" -@ ls -l "$RUNNER_TEMP/orca-sourcemaps-$TAG.zip" @@ -1596,16 +1376,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # The release command prepares and verifies this host's Electron native modules first. ORCA_REUSE_PREPARED_NATIVE_RUNTIME: '1' - # Why: the NSIS uninstaller only exists inside electron-builder's - # uninstaller pass, which deletes it right after embedding it. The sign - # hook in config/scripts/windows-uninstaller-signing.cjs copies it out - # here so it can ride the inner-binaries SignPath request below. - # Why runner.temp and never the workspace: `files` in - # config/electron-builder.config.cjs is all-negation, so app-builder - # prepends `**/*` and packs whatever is left in the checkout root. This - # step retries up to 3 times; attempt 1 writes the file after packing, - # but attempts 2 and 3 would then pack the unsigned uninstaller into - # app.asar - the exact defect this chain exists to remove. + # Capture NSIS uninstaller in sign hook before builder embeds/deletes it, including it in inner-binary signing. Keep capture under runner.temp: all-negation files includes checkout leftovers, and retry would pack unsigned uninstaller into app.asar. ORCA_WIN_UNINSTALLER_EXPORT_PATH: ${{ runner.temp }}\uninstaller-signing\unsigned\orca-uninstaller.exe - name: Verify Windows node-pty ConPTY runtime @@ -1630,18 +1401,9 @@ jobs: if: matrix.platform == 'win' && github.run_attempt == 1 uses: ./.github/actions/install-signpath-module - # ── Windows inner-binary signing (issue #7785) ───────────────────── - # Why: SignPath cannot deep-sign inside NSIS installers, so inner PE - # files (Orca.exe, node-pty *.node, DLLs) are signed via a separate zip - # request, then the installer is rebuilt from the signed tree before the - # existing installer signing request below. The NSIS uninstaller rides - # this same request (it is the MDE update cluster: old-uninstaller.exe / - # Uninstall Orca.exe), captured through electron-builder's sign hook and - # swapped back in during the rebuild — no third approval wait. Production - # releases require the entire signing chain to succeed (#23383). + # #7785/#23383: sign inner PE zip, including captured NSIS uninstaller, then rebuild/sign installer; SignPath cannot deep-sign inside NSIS and this avoids a third approval. - # Why: only unsigned PE files go to SignPath. Files that already carry a - # valid signature (Microsoft's OpenConsole.exe) must keep their signer. + # Stage only unsigned PE files; preserve existing vendor signatures such as Microsoft OpenConsole. - name: Stage unsigned inner PE files for signing id: stage-inner if: matrix.platform == 'win' && github.run_attempt == 1 @@ -1685,12 +1447,8 @@ jobs: Write-Host "Skipped $($skipped.Count) already-signed files:" $skipped | ForEach-Object { Write-Host " $_" } - # Why the uninstaller rides this request: it is the file MDE flagged in - # the whole update cluster (old-uninstaller.exe / Uninstall Orca.exe), - # and folding it in here costs no extra approval wait. Why it is kept - # out of inner-signing-list.txt: that list drives the copy-back into - # dist/win-unpacked, and the uninstaller does not live there — it is - # re-injected through the sign hook during the rebuild instead. + # Include MDE-flagged old-uninstaller.exe/Uninstall Orca.exe in this approval without another wait. + # Exclude it from win-unpacked’s copy-back list; the sign hook re-injects it during rebuild. $exportedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\unsigned\orca-uninstaller.exe' if (-not (Test-Path -LiteralPath $exportedUninstaller)) { throw "No exported NSIS uninstaller at $exportedUninstaller." @@ -1822,9 +1580,7 @@ jobs: New-Item -ItemType Directory -Path signed-inner -Force Expand-Archive -Path signed-inner.zip -DestinationPath signed-inner -Force - # Why: copy back strictly by the staged list so a layout mismatch in the - # returned artifact fails before publication instead of silently - # shipping a mix of signed and unsigned binaries. + # Copy back only staged paths; layout mismatch must fail before mixed signed/unsigned assets ship. - name: Restore signed inner binaries into unpacked app id: restore-signed-inner if: matrix.platform == 'win' && github.run_attempt == 1 && steps.download-signed-inner.outcome == 'success' @@ -1858,8 +1614,7 @@ jobs: throw "Signed inner artifact did not round-trip cleanly ($($failures.Count) failures)." } - # Why: SignPath rewrote the template's Windows binaries, and the client materializer checks - # each file against the template manifest, so it must record the signed bytes. + # Recompute template checksums after SignPath rewrites Windows binaries so materialization validates signed bytes. - name: Reseal the orcad template over its signed binaries id: reseal-orcad-template if: matrix.platform == 'win' && github.run_attempt == 1 && steps.restore-signed-inner.outcome == 'success' && needs.cut.outputs.ships_orcad_template == 'true' @@ -1891,21 +1646,7 @@ jobs: Copy-Item -LiteralPath $signed.FullName -Destination (Join-Path $signedDir 'orca-uninstaller.exe') -Force Write-Host ("{0,-14} uninstaller <{1}>" -f $signature.Status, $signature.SignerCertificate.Subject) - # Why this step exists: electron-builder's CopyElevateHelper re-copies a - # pristine elevate.exe from its download cache over resources\elevate.exe - # on EVERY nsis pack — including the --prepackaged rebuild below — which - # clobbered the SignPath signature in v1.4.129-rc.4. There is no supported - # way to disable just the copy, so we overwrite the cache's copy with our - # signed one (identical bytes plus signature) so the clobber becomes a - # no-op. Known quirk: the cache persists across releases via actions/cache, - # so later runs may see elevate.exe as already signed and skip staging it — - # that is fine (the signature is timestamped) and the evidence gate checks - # elevate.exe in the shipped installer unconditionally. - # - # The cache lookup lives in a script because the inline path this step used - # (`\nsis`) matches no app-builder-lib layout, and `SilentlyContinue` - # plus `exit 0` turned that miss into a green step — v1.4.193 and v1.4.194 - # shipped an unsigned elevate.exe that way. A miss now fails the step. + # Builder recopies cached elevate.exe on every NSIS pack, clobbering signature in v1.4.129-rc.4; replace cache copy with signed bytes before rebuild. Persisted timestamped signatures may skip later staging; shipped-installer evidence still checks them. Scripted lookup fails missing cache paths: old nonexistent /nsis plus silent success shipped unsigned elevate.exe in v1.4.193/v1.4.194. - name: Replace cached elevate.exe with the signed copy id: sign-elevate-cache if: matrix.platform == 'win' && github.run_attempt == 1 && steps.restore-signed-inner.outcome == 'success' @@ -1915,9 +1656,7 @@ jobs: if (-not (Test-Path $signed)) { throw 'No elevate.exe in win-unpacked resources.' } - # Why this guard stays: windows-signing-rehearsal.yml shares the - # electron-builder-win- cache key with this workflow, so a - # test-certificate elevate.exe must never be staged into a release cache. + # Rehearsal shares the builder cache: never seed a release with its test-certificate elevate.exe. $signature = Get-AuthenticodeSignature -FilePath $signed $subject = if ($null -eq $signature.SignerCertificate) { '' } else { $signature.SignerCertificate.Subject } if ($signature.Status -ne 'Valid' -or $subject -notlike '*CN=SignPath Foundation*') { @@ -2107,9 +1846,7 @@ jobs: } $signature.SignerCertificate | Format-List Subject,Issuer,NotBefore,NotAfter,Thumbprint - # Why: evidence gate for inner-binary signing (issue #7785, supersedes - # PR #7170's Orca.exe-only gate — this covers every staged .exe/.dll/.node - # by extracting the shipped installer). Unsigned binaries block publication. + # #7785 evidence extracts every staged exe/dll/node from shipped installer, superseding #7170’s Orca.exe-only gate; unsigned assets block publish. - name: Verify Windows inner binary signatures if: matrix.platform == 'win' && github.run_attempt == 1 shell: pwsh @@ -2162,10 +1899,8 @@ jobs: $report = New-Object System.Collections.Generic.List[string] $failures = New-Object System.Collections.Generic.List[string] - # Why: verify the files a user actually gets on disk, not the build - # tree — 7z parses the NSIS exe directly as its embedded payload. - # Resolve 7za via app-builder-lib; electron-builder 26.9+ dropped the - # bundled 7zip-bin package the old hardcoded path relied on (#6487). + # Verify shipped NSIS payload bytes via 7z, not the build tree. Resolve 7za through app-builder-lib: + # builder 26.9+ dropped bundled 7zip-bin (#6487). $7zaOutput = node config/scripts/resolve-7za-path.mjs $7zaExitCode = $LASTEXITCODE if ($7zaExitCode -ne 0) { @@ -2181,11 +1916,7 @@ jobs: $root = Resolve-Path 'inner-evidence-extract' # Verify the CLI even when a cached signed copy bypassed staging (#23383). - # Why elevate.exe is always appended: staging skips already-signed - # files, and the persisted electron-builder cache can carry a - # previously signed elevate.exe — so it may be absent from the list - # in some runs, yet it is the file most at risk of losing its - # signature in the NSIS rebuild. Verify it in every release. + # Always verify elevate.exe, even when cached signatures skipped staging; NSIS rebuild can clobber it. $targets = @(Get-Content 'inner-signing-list.txt') foreach ($requiredTarget in @('Orca.exe', 'resources\bin\orca.exe', 'resources\bin\orca-agent-hook.exe')) { if ($targets -notcontains $requiredTarget) { $targets += $requiredTarget } @@ -2193,17 +1924,9 @@ jobs: if ($targets -notcontains 'resources\elevate.exe') { $targets += 'resources\elevate.exe' } - # Why the uninstaller is not in $targets: NSIS embeds it in its own - # compressed data section (`File /oname=${UNINSTALL_FILENAME}` in - # app-builder-lib templates/nsis/include/installer.nsh), not in the - # app 7z payload extracted above - the bundled 7za cannot see it. - # What the receipt proves and does not: the digest comparison is - # equal by construction (the hook digests the bytes it copied from - # this same file), so the real signal is that the receipt exists at - # all - the import leg ran, and these are the bytes it embedded. The - # signature check below is the part with teeth. The shipped-artifact - # check lives in windows-signing-rehearsal.yml, which installs the - # installer and inspects the uninstaller it drops on disk. + # The uninstaller lives in NSIS’s compressed data section, outside the app 7z payload. + # A receipt proves the sign-hook import ran, not the installed bytes; its digest equality is by construction. + # Check that input’s signature here; windows-signing-rehearsal installs/inspects the actual shipped uninstaller. if ($env:UNINSTALLER_SIGNING_COMPLETED -eq 'true') { $signedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\signed\orca-uninstaller.exe' $receipt = "$signedUninstaller.embedded-sha256" @@ -2246,10 +1969,7 @@ jobs: if ($failures.Count -gt 0) { $failures | ForEach-Object { Write-Host "::warning::$_" } $message = "Windows inner-binary evidence gate found $($failures.Count) problems." - # Why assigned before any I/O: a write that throws here would reach - # the catch with $policyFailure still null, so a required-mode - # signature failure would be re-reported as ERRORED and the per-file - # report overwritten — the exact masking the hoist exists to prevent. + # Set policyFailure before I/O: write errors must not turn FAILED into ERRORED and overwrite per-file evidence. if ($required) { $policyFailure = $message } else { @@ -2258,8 +1978,7 @@ jobs: Add-GateEvidence "VERDICT: FAILED — $message" Add-GateSummary "FAILED — $message" } else { - # $report, not $targets: the embedded uninstaller is reported but - # is not one of the extracted payload targets. + # Use report, not extracted targets: it also includes the embedded uninstaller. $ok = "All $($report.Count) checked binaries are signed by SignPath Foundation." Add-GateEvidence "VERDICT: PASSED — $ok" Add-GateSummary "PASSED — $ok" @@ -2329,28 +2048,14 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Verify release remains draft after artifact upload - # Why: electron-builder `--publish always` can create a public release - # as soon as this platform uploads. Re-draft immediately, then fail, so - # /releases/latest never keeps serving a missing Windows exe. - # Why bash: the Windows matrix defaults to pwsh, which does not expand - # "$TAG" into argv. + # Re-draft immediately if publish:always exposed a partial release, then fail so latest cannot keep missing Windows assets. Use bash for $TAG expansion. shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ needs.cut.outputs.tag }} run: node config/scripts/assert-github-release-is-draft.mjs "${{ needs.cut.outputs.tag }}" - # Why post-pack for Linux: electron-builder packs and uploads in one - # `--publish always` invocation. The previous step re-drafts if that - # upload flipped the GitHub release public; this telemetry check still - # blocks `publish-release` from undrafting a bad binary. - # - # Why this guards against: a misconfigured CI run where - # `ORCA_POSTHOG_WRITE_KEY` is unset or the tag fails to classify - # would otherwise produce a binary with `BUILD_IDENTITY = null` and - # `WRITE_KEY = null`, which silently disables transport - # (`IS_OFFICIAL_BUILD === false`) — the exact failure mode flagged - # in PR #1385's deferred follow-up. + # Check Linux telemetry after inseparable pack/upload, then block final undraft. Reject null identity/key transport from missing configuration (#1385); prior step re-drafts premature publication. - name: Verify telemetry constants present in app.asar run: node config/scripts/verify-telemetry-constants.mjs @@ -2371,9 +2076,7 @@ jobs: needs.release-javascript.result == 'success' && (needs.orcad-template.result == 'success' || (needs.orcad-template.result == 'skipped' && needs.cut.outputs.ships_orcad_template == 'false')) - # Why: SignPath requires every job in this signing workflow to be - # GitHub-hosted. The actual mac build runs in release-mac-build.yml so - # Blacksmith stays outside Windows artifact provenance. + # Keep every SignPath workflow job GitHub-hosted; separate release-mac-build allows Blacksmith outside Windows provenance. runs-on: ubuntu-latest permissions: actions: write @@ -2496,22 +2199,14 @@ jobs: fi - name: Verify release assets complete - # Why: publish-release is the only intended draft -> published - # transition. Refuse to un-draft until every updater manifest and - # referenced installer asset is present on GitHub. + # Only undraft after every updater manifest and referenced installer exists. env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ needs.cut.outputs.tag }} run: node config/scripts/verify-release-required-assets.mjs "$TAG" - name: Publish release - # Why: derive `--prerelease` from the tag shape (not from whatever - # electron-builder left the release flagged as). On 2026-04-27, - # electron-builder's publish step flipped `prerelease` back to - # `false` on -rc.N releases, which caused an RC to be marked as - # GitHub's "latest" release and broke release-cut.yml's math. - # Re-asserting here means the final release state is determined - # by the tag — a ground truth electron-builder can't rewrite. + # Reassert prerelease from tag: on 2026-04-27 builder flipped RC to false/latest and broke release math; builder output is not authoritative. env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ needs.cut.outputs.tag }} diff --git a/.github/workflows/release-mac-build.yml b/.github/workflows/release-mac-build.yml index f709e42d22a..b6a752f6177 100644 --- a/.github/workflows/release-mac-build.yml +++ b/.github/workflows/release-mac-build.yml @@ -47,9 +47,7 @@ jobs: build-mac: needs: qualification-template if: ${{ !cancelled() && (!inputs.qualification_only || needs.qualification-template.result == 'success') && github.repository == 'stablyai/orca' }} - # Why: this workflow is outside the SignPath signing run, so Blacksmith - # cannot enter Windows artifact provenance while mac notarization gets the - # faster runner. + # Keep Blacksmith mac notarization outside SignPath’s GitHub-hosted Windows provenance. runs-on: blacksmith-6vcpu-macos-15 timeout-minutes: 60 # actions: read fetches the release-cut run's artifacts. @@ -97,8 +95,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Why: packaging bundles the pinned Node into the macOS terminal host. Keyed on the pin alone - # with no restore-keys, so a pin bump starts empty instead of carrying old archives forward. + # Cache bundled host Node by exact pin, without restore keys. - name: Locate the pinned Node runtime cache shell: bash run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV" @@ -109,8 +106,7 @@ jobs: path: ~/Library/Caches/orca-node-runtime key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }} - # Cache the Electron binary + electron-builder tool downloads (notarytool, - # winCodeSign, nsis, squirrel, AppImage). Saves ~30-90s per job, incl. mac. + # Cache Electron/builder tool downloads to save ~30–90s. - name: Cache electron-builder downloads uses: actions/cache@v5 with: @@ -121,12 +117,7 @@ jobs: restore-keys: | electron-builder-mac- - # Why: pnpm install triggers electron's postinstall, which downloads the - # Electron binary from GitHub release assets. GitHub's download CDN - # occasionally returns 504s that fail the whole release. Retry on - # failure so transient network errors don't require a manual re-run. - # Why both CPUs: the mac config packages x64 and arm64 from this arm64 - # runner, so the install must carry both variants of the native optional deps. + # Retry transient Electron CDN 504s; install both x64/arm64 optional natives for dual macOS packages. - name: Install dependencies uses: nick-fields/retry@v4 with: @@ -135,8 +126,7 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 - # Why here: electron-builder's beforePack requires out/mobile-web, and the bundle - # build resolves React Native and Expo from mobile/node_modules. + # Build required out/mobile-web using mobile’s React Native/Expo install. - uses: ./.github/actions/install-mobile-dependencies - name: Verify macOS signing environment @@ -148,14 +138,11 @@ jobs: APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} - # Why: `plutil -lint` accepts duplicate plist keys, but `codesign` - # rejects duplicate entitlements after the expensive app build. + # Check duplicate entitlements before build: plutil lint accepts duplicates but codesign rejects them. - name: Verify macOS entitlements run: pnpm verify:macos-entitlements - # Why: telemetry's transport gate (`src/main/telemetry/client.ts:IS_OFFICIAL_BUILD`) - # requires the build identity to be the literal string `stable` or `rc`, - # substituted by electron-vite's `define` block at build time. + # Set literal stable/rc telemetry identity through Vite define. - name: Classify release tag for telemetry build identity id: tag-classify shell: bash @@ -163,9 +150,7 @@ jobs: TAG: ${{ inputs.tag }} run: | set -euo pipefail - # Why the optional trailing identifier: suffixed side-branch RCs - # (vX.Y.Z-rc.N.perf) are rc-channel prerelease builds — same telemetry - # identity as plain RCs. + # Suffixed vX.Y.Z-rc.N.perf builds keep the same rc-channel telemetry identity. if [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+(\.[0-9A-Za-z]+)?$ ]]; then identity=rc elif [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -216,16 +201,13 @@ jobs: - name: Gate runtime file-watcher process isolation run: | - # Why: #8212 is a native-process crash contract. Prove both the Node - # host and the exact Electron runtime survive SIGSEGV before packaging. + # #8212: require Node and the exact Electron runtime to survive native SIGSEGV before packaging. node config/scripts/runtime-file-watcher-fault-harness.mjs ELECTRON_RUN_AS_NODE=1 pnpm exec electron config/scripts/runtime-file-watcher-fault-harness.mjs - name: Abort if the parent release-cut run was cancelled if: ${{ !inputs.qualification_only }} - # Why: this workflow is dispatched separately, so cancelling release-cut - # does not stop mac `--publish always`. A cancelled parent left v1.4.206 - # public with only a partial mac upload. + # Watch parent cancellation: separately dispatched mac upload otherwise continues (v1.4.206 went public with partial assets). env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PARENT_RUN: ${{ inputs.release_run_id }} @@ -256,18 +238,14 @@ jobs: - name: Verify release remains draft after artifact upload if: ${{ !inputs.qualification_only }} - # Why: re-draft immediately if electron-builder flipped the GitHub - # release public, then fail. Checking without restoring leaves - # /releases/latest serving a missing Windows exe. + # Re-draft premature builder publication before failing; latest must not keep an incomplete Windows release. shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ inputs.tag }} run: node config/scripts/assert-github-release-is-draft.mjs "${{ inputs.tag }}" - # Why post-publish for macOS: electron-builder packs and uploads in a - # single `--publish always` invocation, so there is no cheap insertion - # point between pack and upload without splitting those steps. + # Check after publish because builder’s single pack/upload invocation has no cheap interposition point. - name: Verify telemetry constants present in app.asar run: node config/scripts/verify-telemetry-constants.mjs diff --git a/.github/workflows/terminal-ime-e2e.yml b/.github/workflows/terminal-ime-e2e.yml index eb10051181e..58bc021688a 100644 --- a/.github/workflows/terminal-ime-e2e.yml +++ b/.github/workflows/terminal-ime-e2e.yml @@ -1,8 +1,7 @@ name: Terminal IME E2E on: - # Why workflow_call and not pull_request: pr.yml owns the path filter that decides when an IME - # change is worth a real ibus session. A pull_request trigger here would run it on every PR. + # Let pr.yml own costly real-IBus path routing; a separate pull_request trigger would run every PR. workflow_call: workflow_dispatch: inputs: @@ -23,7 +22,7 @@ env: jobs: linux-x11: name: Linux X11 terminal IME - runs-on: blacksmith-4vcpu-ubuntu-2204 + runs-on: ubuntu-22.04 timeout-minutes: 25 steps: @@ -63,9 +62,7 @@ jobs: --workers=1 - name: Run native IBus Hangul exact-byte tests - # Why not the default success(): the synthetic step above runs first, so its failure used - # to skip this one entirely — the real-IME half reported nothing on exactly the PRs that - # broke IME code. The two signals are independent and both belong in the log. + # Run native half despite synthetic failure: independent signals must both appear, rather than default success() hiding native evidence. if: '!cancelled()' env: SKIP_BUILD: '1' @@ -82,7 +79,7 @@ jobs: linux-wayland: name: Linux Wayland Hangul terminating digit - runs-on: blacksmith-4vcpu-ubuntu-2204 + runs-on: ubuntu-22.04 timeout-minutes: 25 steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/unit-plan.yml b/.github/workflows/unit-plan.yml index 1bd32f72ecc..fb3836ac6ff 100644 --- a/.github/workflows/unit-plan.yml +++ b/.github/workflows/unit-plan.yml @@ -1,11 +1,6 @@ name: Unit plan -# Why its own workflow: the caller's `needs` gate the whole called workflow, so while this lived -# inside unit-tests.yml it waited on static analysis and typecheck before it could even start — -# and the shards then waited on it. Planning needs neither (its inputs are the checkout, a git -# diff against HEAD^1, the import graph, and the checked-in timing baseline), so hoisting it out -# lets it overlap the gate instead of queueing behind it. Measured: a median 93s off the shard -# matrix's start. +# Plan independently of static/type gates so checkout/diff/import-graph/timing work overlaps them; moving it out of unit-tests advanced shards by median 93s. on: workflow_call: inputs: diff --git a/.github/workflows/unit-selection-evidence.yml b/.github/workflows/unit-selection-evidence.yml index f49da2f58ec..e9e1104aad6 100644 --- a/.github/workflows/unit-selection-evidence.yml +++ b/.github/workflows/unit-selection-evidence.yml @@ -1,10 +1,6 @@ name: Unit selection evidence -# Why its own workflow: this job is advisory -- `continue-on-error` on both the job and its -# comparison step, so it can never fail a PR. But a caller's `needs: test` waits for every job in -# the called workflow, so while it lived in unit-tests.yml it held `verify` for ~36s after the -# last shard finished. Called as a sibling instead, it still runs on every PR and still uploads -# its review artifact; it just no longer sits on the critical path. +# Call advisory evidence beside unit-tests, preserving artifacts without delaying verify ~36s after shards; job/comparison remain continue-on-error. on: workflow_call: diff --git a/.github/workflows/win-crash-survival-e2e.yml b/.github/workflows/win-crash-survival-e2e.yml index eabb84bb955..a18a00c6489 100644 --- a/.github/workflows/win-crash-survival-e2e.yml +++ b/.github/workflows/win-crash-survival-e2e.yml @@ -1,13 +1,6 @@ name: Windows Crash-Survival E2E -# Why: proves the detached daemon-host relocation makes terminal sessions SURVIVE -# a CRASH of Orca's main process (GitHub #7742), the companion guarantee to the -# update-survival harness. Builds an (unsigned) installer FROM THIS BRANCH, -# silent-installs it, opens a terminal, force-kills ONLY the app main (no -# tree-kill), and asserts the daemon + shell stay alive, a relaunch adopts the -# same daemon and shell, and post-crash input causes no pwsh FailFast. Targeted -# manual runs keep it as a durable regression proof. A CI runner is the only -# safe place to install — see the relocation post-mortem. +# #7742: build/install unsigned branch artifact on disposable CI, crash only main and require same daemon/shell survival, adoption and input without FailFast. Companion update proof tests installer survival; see relocation post-mortem. on: workflow_dispatch: diff --git a/.github/workflows/win-orcad-serve-switch-e2e.yml b/.github/workflows/win-orcad-serve-switch-e2e.yml index f7227d7ff19..00fba9c7637 100644 --- a/.github/workflows/win-orcad-serve-switch-e2e.yml +++ b/.github/workflows/win-orcad-serve-switch-e2e.yml @@ -1,10 +1,6 @@ name: Windows packaged orcad serve switch E2E -# Why: proves D7 on a real Windows install. The installed desktop app forks its terminal daemon -# from the relocated %LOCALAPPDATA%\Orca\daemon-host; `orca serve` then serves the same profile -# on orcad and must adopt that daemon, and the desktop must reattach the terminal afterwards and -# still write its settings under orcad's data-root ACL. Unpackaged e2e cannot exercise the -# relocation, which only packaged builds do. A CI runner is the only safe place to install. +# D7 packaged Windows proof: relocated desktop daemon survives orcad serve adoption and desktop reattach, with settings writes after data-root ACL. Run install only on disposable CI; unpackaged E2E cannot exercise relocation. on: pull_request: diff --git a/.github/workflows/win-update-e2e.yml b/.github/workflows/win-update-e2e.yml index 48738213729..d0652121147 100644 --- a/.github/workflows/win-update-e2e.yml +++ b/.github/workflows/win-update-e2e.yml @@ -1,20 +1,8 @@ name: Windows Update-Survival E2E -# Why: proves what happens to terminal sessions across a real Windows app update -# by installing one released version, updating to another, and asserting on the -# packaged artifacts (daemon survival, terminal interactivity, zero console -# flashes). This MUST run on a disposable CI Windows: electron-builder's oneClick -# installer uninstalls the registry-registered copy of the app before every -# install, so running the harness on a machine with a real Orca install would -# delete it. A fresh runner has no Orca registered, so it is the only safe home -# for install-based testing. Manual dispatch only. +# Install/update two released Windows builds and prove daemon/input/no-flash behavior on disposable CI only. oneClick uninstalls registry-selected app, so a real workstation install could be deleted. -# Why: workflow_dispatch only works once a workflow is on the default branch. -# To exercise this from the unmerged feature branch WITHOUT touching main, a -# push trigger scoped to that exact branch runs the workflow from the branch's -# own tree. The paths filter keeps it to harness/workflow edits so ordinary -# commits don't spend 30 min of Windows runner time. Push runs have no inputs, -# so every parameter falls back to a default below. +# Feature-branch push enables pre-merge workflow testing before dispatch exists on default; path-filter it to avoid 30min runs on unrelated changes. Push inputs use defaults. on: push: branches: @@ -113,10 +101,7 @@ jobs: "FROM_EXE=$($from.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append "TO_EXE=$($to.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - # Why: no --install-dir here. A CI runner has no real Orca registered, so - # the harness installs to the default per-user location and owns it — the - # isolated-install machinery exists only to (imperfectly) protect a real - # developer install, which does not apply on a throwaway runner. + # Own default per-user install on disposable CI; no real install needs the imperfect isolated-mode protection. - name: Run update-survival harness id: harness shell: pwsh diff --git a/.github/workflows/win-update-survival-e2e.yml b/.github/workflows/win-update-survival-e2e.yml index a52959327de..eab49d27f47 100644 --- a/.github/workflows/win-update-survival-e2e.yml +++ b/.github/workflows/win-update-survival-e2e.yml @@ -1,13 +1,6 @@ name: Windows Update-Survival E2E (branch build) -# Why: proves the Phase 1 daemon-host relocation actually makes terminal -# sessions SURVIVE a Windows update. Builds an (unsigned) installer FROM THIS -# BRANCH, then runs the win-update-e2e harness with --expect survival: install, -# open a terminal, silently update over it, relaunch, and assert the daemon PID -# is unchanged and the pre-update terminal is still interactive with no console -# flashing. Runs from the feature branch via push (workflow_dispatch only works -# on the default branch); main is never touched. A CI runner is the only safe -# place to install/update — see the relocation post-mortem. +# Build/install/update unsigned branch artifact with --expect survival; require same daemon and interactive original terminal without flashes. Feature push tests pre-default-branch workflow; run only on disposable CI (relocation post-mortem). on: push: @@ -78,10 +71,7 @@ jobs: with: install: false - # Why: cache the built installer keyed on the inputs that affect it, so a - # harness-only edit skips the ~20 min electron-builder build. The daemon - # relocation code lives under src/, so changing it correctly rebuilds. - # Hash before installation creates native build artifacts under native/. + # Cache installer by product inputs so harness-only changes skip ~20min build; hash before native install generates artifacts. - name: Cache branch installer id: cache-installer uses: actions/cache/restore@v4 diff --git a/.github/workflows/windows-signing-rehearsal.yml b/.github/workflows/windows-signing-rehearsal.yml index ceadbe3d14f..a42a93504b6 100644 --- a/.github/workflows/windows-signing-rehearsal.yml +++ b/.github/workflows/windows-signing-rehearsal.yml @@ -1,18 +1,4 @@ -# Windows inner-binary signing rehearsal. -# -# Why: SignPath cannot deep-sign inside NSIS installers, so shipping signed -# inner binaries (Orca.exe, node-pty *.node, DLLs — see issue #7785) requires -# a two-request flow: sign the unpacked PE files first, then build the NSIS -# installer from the signed tree, then sign the installer. The NSIS uninstaller -# rides that same first request — it is captured through electron-builder's sign -# hook and swapped back in during the rebuild — so it adds no third approval. -# This workflow rehearses that entire flow from a branch, end to end, without -# publishing anything — so the release pipeline on main is never at risk. -# -# Runs only via manual dispatch. Use the test-signing policy for iteration -# (auto-approved test certificate) and release-signing to rehearse the -# production flow (requires manual SignPath approvals). Rehearsed successfully -# with test-signing (run 28987534795) and release-signing (run 28988432001). +# Rehearse #7785 inner-PE/uninstaller signing then rebuilt installer signing without publication. SignPath cannot deep-sign NSIS; captured uninstaller shares first request. Manual test-signing auto-approves; release-signing needs production review. Proven runs 28987534795/28988432001. name: Windows signing rehearsal on: @@ -93,16 +79,11 @@ jobs: env: NODE_OPTIONS: --max-old-space-size=4096 - # Why a full --win build and not --dir: the NSIS uninstaller only exists - # inside the installer build, and it is the file the MDE update cluster - # flags. --dir would never produce it, so the rehearsal would not rehearse - # the uninstaller leg at all. This mirrors release-cut's first Windows pass. + # Build full --win: --dir never creates the NSIS uninstaller needed for MDE evidence. - name: Package Windows app and export the NSIS uninstaller shell: pwsh env: - # runner.temp, never the workspace: the all-negation `files` list in - # config/electron-builder.config.cjs packs whatever is left in the - # checkout root into app.asar. + # Keep capture under runner.temp; all-negation files would include checkout leftovers in app.asar. ORCA_WIN_UNINSTALLER_EXPORT_PATH: ${{ runner.temp }}\uninstaller-signing\unsigned\orca-uninstaller.exe run: | node config/scripts/ensure-native-runtime.mjs --runtime=electron @@ -116,9 +97,7 @@ jobs: throw "The sign hook did not export the NSIS uninstaller to $env:ORCA_WIN_UNINSTALLER_EXPORT_PATH" } - # Why: only unsigned PE files go to SignPath. Files that already carry a - # valid signature (Microsoft's OpenConsole.exe, signed vendor DLLs) must - # keep their original signer, so they are excluded from the request. + # Sign only unsigned PE files; retain vendor signatures such as Microsoft OpenConsole. - name: Stage unsigned PE files for inner signing id: stage-inner shell: pwsh @@ -156,11 +135,8 @@ jobs: Write-Host "Skipped $($skipped.Count) already-signed files:" $skipped | ForEach-Object { Write-Host " $_" } - # Why kept out of inner-signing-list.txt: that list drives the copy-back - # into dist/win-unpacked, and the uninstaller does not live there — it is - # re-injected through the electron-builder sign hook during the rebuild. - # No catch here, unlike the release job: the rehearsal exists to prove - # the flow, so a staging failure must fail it loudly. + # The uninstaller is re-injected by the sign hook, outside win-unpacked’s copy-back list. + # Fail staging loudly: rehearsal proves the flow and needs no release-job recovery catch. $exportedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\unsigned\orca-uninstaller.exe' $uninstallerStagePath = Join-Path $stage.FullName 'uninstaller\orca-uninstaller.exe' New-Item -ItemType Directory -Force -Path (Split-Path $uninstallerStagePath) | Out-Null @@ -206,9 +182,7 @@ jobs: New-Item -ItemType Directory -Path signed-inner -Force Expand-Archive -Path signed-inner.zip -DestinationPath signed-inner -Force - # Why: copy back strictly by the staged list so a layout mismatch in the - # returned artifact fails loudly instead of silently shipping a mix of - # signed and unsigned binaries. + # Copy back strictly by staged list so mismatches fail instead of shipping mixed signing. - name: Restore signed inner binaries into unpacked app shell: pwsh env: @@ -343,10 +317,7 @@ jobs: $latestYml = $latestYml -replace '(?m)^ blockMapSize: \d+$', " blockMapSize: $($blockmap.Length)" Set-Content -Path 'dist/latest.yml' -Value $latestYml -NoNewline - # Why: this is the pass/fail heart of the rehearsal — the same evidence - # check that will later gate releases (see PR #7170). Test certificates - # do not chain to a trusted root, so Status=Valid is only required for - # release-signing runs; test-signing runs require a present signature. + # Use production evidence gate (#7170); require trusted Status=Valid only for release signing, signature presence for test certs. - name: Verify signatures end to end shell: pwsh env: @@ -357,10 +328,7 @@ jobs: $advisories = New-Object System.Collections.Generic.List[string] $requireValid = $env:SIGNING_POLICY -eq 'release-signing' - # -Advisory records a problem without failing the run. It exists for - # exactly one file (resources\elevate.exe, below) and must not be - # widened casually: the point of this workflow is to fail when signing - # is broken. + # Advisory is only for resources\elevate.exe; all other signing failures must fail this rehearsal. function Test-Signature([string]$label, [string]$path, [switch]$Advisory) { $signature = Get-AuthenticodeSignature -FilePath $path $subject = if ($null -eq $signature.SignerCertificate) { '' } else { $signature.SignerCertificate.Subject } @@ -386,9 +354,7 @@ jobs: Test-Signature 'installer: orca-windows-setup.exe' 'dist/orca-windows-setup.exe' - # Extract the signed installer and verify the files a user actually - # gets on disk — including the exact file from issue #7785. - # electron-builder 26.9+ removed the bundled 7zip-bin package (#6487). + # Extract/verify shipped bytes, including #7785. Builder 26.9+ removed bundled 7zip-bin (#6487). $7zaOutput = node config/scripts/resolve-7za-path.mjs $7zaExitCode = $LASTEXITCODE if ($7zaExitCode -ne 0) { @@ -402,13 +368,8 @@ jobs: & $7za x 'dist/orca-windows-setup.exe' '-oextracted-app' -y | Out-Null $root = Resolve-Path 'extracted-app' - # The receipt only proves the import leg ran; it cannot prove what NSIS - # embedded, because the uninstaller lives in a compressed NSIS data - # section rather than the app 7z payload above and the bundled 7za has - # no NSIS handler. So the rehearsal - unlike the release job, which - # must not mutate the runner it publishes from - goes all the way: it - # installs the installer silently and inspects the uninstaller the - # installer actually wrote to disk. That is the file MDE flags. + # The receipt proves import, not NSIS embedding: the uninstaller lies outside the app 7z payload. + # Unlike publishing release jobs, rehearsal can install silently and inspect the actual MDE-flagged file. $signedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\signed\orca-uninstaller.exe' $receipt = "$signedUninstaller.embedded-sha256" if (-not (Test-Path -LiteralPath $receipt)) { @@ -417,10 +378,7 @@ jobs: Test-Signature 'relayed: orca-uninstaller.exe' $signedUninstaller } - # Why a full 7-Zip attempt first: it is non-invasive. The runner image - # ships the complete 7z.exe, which - unlike the reduced 7za - has an - # NSIS handler. If it cannot read the section either, fall back to a - # real silent install. + # Try full 7z.exe’s NSIS handler non-invasively first; reduced 7za lacks it. Fall back to a real silent install. $installedUninstaller = $null $installedVia = $null $expectedDigest = if (Test-Path -LiteralPath $receipt) { (Get-Content -LiteralPath $receipt -Raw).Trim() } else { $null } @@ -430,14 +388,8 @@ jobs: & $full7z x -tnsis 'dist/orca-windows-setup.exe' '-onsis-extract' -y 2>&1 | Out-Null $installedUninstaller = Get-ChildItem -Path nsis-extract -Recurse -File -Filter 'Uninstall*.exe' -ErrorAction SilentlyContinue | Select-Object -First 1 - # Why the digest guard before trusting this route: 7-Zip's NSIS - # handler emits partial or garbled output on some NSIS builds, and a - # truncated extract would score NotSigned and fail the rehearsal as - # "the shipped uninstaller is unsigned" when nothing is wrong. Only - # trust it when it reproduces the bytes the relay embedded; otherwise - # fall through to the install route, which is ground truth. A name - # miss (the handler labelling the entry by its source name) falls - # through the same way. + # NSIS extraction can be partial/garbled: trust 7z only when its digest matches embedded bytes, + # otherwise use ground-truth installation. Missing/misnamed entries also fall through. if ($null -ne $installedUninstaller -and $null -ne $expectedDigest -and (Get-FileHash -LiteralPath $installedUninstaller.FullName -Algorithm SHA256).Hash.ToLowerInvariant() -ne $expectedDigest) { Write-Host "7-Zip's NSIS output did not match the relayed digest; falling back to a silent install." @@ -452,19 +404,13 @@ jobs: } if ($null -eq $installedUninstaller) { - # Nothing here is published, so mutating this runner is free. - # Why -PassThru and a bounded wait rather than -Wait: a bare -Wait on - # an installer that ever prompts hangs to the job's 360-minute cap. + # No artifacts are published, so installation is safe here. Bound -PassThru waits; -Wait could hang on a prompt for 360 minutes. $installerProcess = Start-Process -FilePath (Resolve-Path 'dist/orca-windows-setup.exe') -ArgumentList '/S' -PassThru if (-not $installerProcess.WaitForExit(300000)) { $installerProcess | Stop-Process -Force -ErrorAction SilentlyContinue $failures.Add('the silent install did not exit within 5 minutes; it is likely prompting') } - # Why a poll rather than one Stop-Process: the oneClick installer - # launches the app as it finishes, so Orca.exe can appear *after* the - # installer process exits. A single silenced Stop-Process would miss - # it and leave Orca plus orca-terminal-daemon.exe holding handles - # under %LOCALAPPDATA%\Programs for the rest of the job. + # Poll app shutdown: oneClick can launch Orca after installer exit, leaving app/daemon handles in LOCALAPPDATA\Programs. for ($attempt = 0; $attempt -lt 20; $attempt++) { $running = @(Get-Process -Name 'Orca' -ErrorAction SilentlyContinue) if ($running.Count -gt 0) { @@ -484,12 +430,8 @@ jobs: if ($null -eq $installedUninstaller) { $failures.Add('could not obtain the uninstaller the installer ships; neither 7-Zip nor a silent install produced it') } else { - # Why this digest comparison is the point of the whole rehearsal: - # unlike the release job's, it hashes a file NSIS itself wrote out - # rather than the file the hook copied, so it is the only check that - # proves the shipped installer embedded the SignPath-signed bytes. On - # the 7-Zip route the guard above already forced equality; on the - # install route this is the first time it is tested. + # Hash NSIS-written bytes to prove the installer embedded SignPath-signed input; the release hook’s + # input-to-input comparison cannot prove this. The 7z route already requires digest equality. if ($null -ne $expectedDigest) { $shippedDigest = (Get-FileHash -LiteralPath $installedUninstaller.FullName -Algorithm SHA256).Hash.ToLowerInvariant() if ($shippedDigest -ne $expectedDigest) { @@ -509,34 +451,11 @@ jobs: $failures.Add("missing from installer payload: $relative") continue } - # Why elevate.exe alone is advisory: app-builder-lib re-copies the - # pristine cached elevate.exe over resources\elevate.exe on EVERY nsis - # pack - AppPackageHelper.packArch calls elevateHelper.copy() before - # buildAppPackage (nsisUtil.js), and CopyElevateHelper.copy does - # `copyFile(elevatePath, outFile, false)` then `signIf(outFile)`, which - # signs nothing because this build configures no certificate. So the - # signed copy restored into win-unpacked is clobbered by the rebuild. - # This predates the uninstaller relay and is not caused by it: with no - # `sign` hook, signIf already returned false at "no signing info - # identified" (windowsSignToolManager.js), so no signtool call was - # displaced. release-cut.yml mitigates it separately by pre-seeding the - # electron-builder cache ("Replace cached elevate.exe with the signed - # copy"); this workflow has no such step, which is why the clobber is - # visible here and not there. Mirroring that step here would not help: - # it only swaps when the copy is already Valid and SignPath-signed, so - # it no-ops under the test certificate. - # - # DO NOT relax that Valid + SignPath-signed guard to make this - # rehearsal go green. This workflow and release-cut.yml share the - # cache key `electron-builder-win-`, and that guard is - # the only thing stopping a test certificate from being seeded into - # the cache a real release restores from. Shipping users a binary - # signed by "Test certificate for 'Orca agent ide [OSS]'" is worse - # than shipping it unsigned. - # - # Fixing elevate.exe belongs in its own PR - it is a UAC elevation - # helper, and it deserves more scrutiny than a footnote in an - # uninstaller change. + # Only elevate.exe is advisory: NSIS pack recopies pristine cache bytes over signed win-unpacked output. + # With no certificate, CopyElevateHelper.signIf signs nothing; this predates the uninstaller relay. + # release-cut pre-seeds only Valid, SignPath-signed cache bytes; test certificates deliberately fail that guard. + # DO NOT relax it: rehearsal shares electron-builder-win- with real releases and would poison + # their cache with “Test certificate for Orca agent ide [OSS]”. Fix the UAC helper separately with proper scrutiny. if ($relative -eq 'resources\elevate.exe') { Test-Signature "installed: $relative" $path -Advisory } else { diff --git a/Casks/orca.rb b/Casks/orca.rb index 1abc9b3a912..873b6fbc492 100644 --- a/Casks/orca.rb +++ b/Casks/orca.rb @@ -15,27 +15,17 @@ cask "orca" do strategy :github_latest end - # Why: electron-updater (src/main/updater.ts) handles in-place updates by - # writing a new Orca.app into /Applications. Marking the cask auto_updates - # tells Homebrew not to compete with the in-app updater — `brew upgrade` - # becomes a no-op unless the user passes --greedy, and brew's version - # metadata stays aligned with whatever the app has swapped itself to. + # Let the in-app updater own updates; Homebrew skips auto_updates unless --greedy. auto_updates true conflicts_with cask: "orca@rc" depends_on macos: :big_sur app "Orca.app" - # Why: expose the bundled `orca` CLI on PATH at install time (Homebrew symlinks - # this into its already-on-PATH bin dir). Without it, the CLI is only registered - # by the in-app "Install CLI" action, which a headless host can never trigger — - # so `orca serve` on a server would be unreachable from the shell. The shim - # resolves the real app by walking symlinks, so the Homebrew symlink works. + # Expose the symlink-resolving bundled CLI on PATH so headless hosts can use orca serve without the in-app installer. binary "#{appdir}/Orca.app/Contents/Resources/bin/orca" - # Why: Orca writes user data under ~/.orca (worktrees, agent state) and - # Electron's standard userData directories. Zap removes everything the app - # creates during normal use so `brew uninstall --zap` is a clean slate. + # Zap app-created ~/.orca and Electron userData for a clean uninstall. zap trash: [ "~/.orca", "~/Library/Application Support/Orca", diff --git a/Casks/orca@rc.rb b/Casks/orca@rc.rb index 08eaeeca459..35644adaeef 100644 --- a/Casks/orca@rc.rb +++ b/Casks/orca@rc.rb @@ -26,24 +26,17 @@ cask "orca@rc" do end end - # Why: RC installs should follow Orca's prerelease-aware updater instead of - # waiting for Homebrew metadata churn between frequent release candidates. + # Use Orca’s prerelease-aware updater between frequent RCs. auto_updates true conflicts_with cask: "orca" depends_on macos: :big_sur app "Orca.app" - # Why: expose the bundled `orca` CLI on PATH at install time (Homebrew symlinks - # this into its already-on-PATH bin dir). Without it, the CLI is only registered - # by the in-app "Install CLI" action, which a headless host can never trigger — - # so `orca serve` on a server would be unreachable from the shell. The shim - # resolves the real app by walking symlinks, so the Homebrew symlink works. + # Expose the symlink-resolving bundled CLI on PATH so headless hosts can use orca serve without the in-app installer. binary "#{appdir}/Orca.app/Contents/Resources/bin/orca" - # Why: Orca writes user data under ~/.orca (worktrees, agent state) and - # Electron's standard userData directories. Zap removes everything the app - # creates during normal use so `brew uninstall --zap` is a clean slate. + # Zap app-created ~/.orca and Electron userData for a clean uninstall. zap trash: [ "~/.orca", "~/Library/Application Support/Orca", diff --git a/cloud/.gitleaks.toml b/cloud/.gitleaks.toml index 0bb1f966fae..33fc17cf805 100644 --- a/cloud/.gitleaks.toml +++ b/cloud/.gitleaks.toml @@ -6,16 +6,14 @@ description = "Explicit Relay test signing key" regexTarget = "secret" regexes = ['''^test-assignment-key-with-at-least-32-bytes$'''] -# The Cloud SQL rollout lease records `owner/repo/run_id` as the holder of a lease. The action's -# unit tests build fixture holders from that shape, which the generic key rule reads as a secret. +# Fixture lease holders use owner/repo/run_id, which the generic key rule mistakes for a secret. [[allowlists]] description = "Cloud SQL rollout lease holder keys in the action's unit tests" regexTarget = "secret" paths = ['''\.github/actions/cloud-sql-rollout-lease/[a-z-]+\.test\.mjs$'''] regexes = ['''^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/[0-9]+$'''] -# RFC 6455 §1.3 example handshake nonce ("the sample nonce" in base64), sent by the raw-socket -# upgrade tests; the generic key rule reads any base64 header value as a secret. +# Raw-socket tests use RFC 6455 §1.3’s example base64 nonce, triggering the generic key rule. [[allowlists]] description = "RFC 6455 example Sec-WebSocket-Key in upgrade tests" regexTarget = "secret" diff --git a/cloud/apps/push/src/apns-http2-transport.ts b/cloud/apps/push/src/apns-http2-transport.ts index 167b4d14e38..d6cbc9b0b13 100644 --- a/cloud/apps/push/src/apns-http2-transport.ts +++ b/cloud/apps/push/src/apns-http2-transport.ts @@ -11,8 +11,7 @@ export type ApnsRequest = { export type { ApnsResponse } export type ApnsTransport = (request: ApnsRequest) => Promise -// APNs requires HTTP/2 and rewards a long-lived session per host, so sessions -// are cached and only dropped when the socket itself goes away. +// Cache APNs HTTP/2 sessions per host until socket loss; long-lived connections avoid reconnect overhead. export function createApnsHttp2Transport(): ApnsTransport & { close(): void } { const sessions = new Map() diff --git a/cloud/apps/push/src/apns-stream-response.ts b/cloud/apps/push/src/apns-stream-response.ts index 9fa189304e8..8ea0c1b1a21 100644 --- a/cloud/apps/push/src/apns-stream-response.ts +++ b/cloud/apps/push/src/apns-stream-response.ts @@ -4,8 +4,7 @@ import { constants } from 'node:http2' export type ApnsResponse = { status: number; body: string; retryAfterMs?: number } -// The subset of ClientHttp2Stream this module drives, so a fake emitter can -// stand in for a real APNs stream in tests. +// Stream subset permits fake emitters in APNs tests. export type ApnsResponseStream = EventEmitter & { setTimeout(ms: number, callback: () => void): void destroy(error?: Error): void @@ -45,8 +44,7 @@ export function readApnsStreamResponse( }) ) ) - // A peer reset with NGHTTP2_NO_ERROR emits neither 'end' nor 'error', which - // would leave the worker's delivery pending for the life of the process. + // NGHTTP2_NO_ERROR resets emit neither end nor error; close must settle or delivery hangs for the process lifetime. stream.on('close', () => settle(() => reject(new Error('apns_stream_closed')))) stream.end(body) }) diff --git a/cloud/apps/push/src/client-ip-rate-limit.ts b/cloud/apps/push/src/client-ip-rate-limit.ts index 18518a2d316..6ac3e4d3c15 100644 --- a/cloud/apps/push/src/client-ip-rate-limit.ts +++ b/cloud/apps/push/src/client-ip-rate-limit.ts @@ -14,13 +14,7 @@ export type ClientIpRateLimiterOptions = { type Bucket = { tokens: number; updatedAt: number } -// Read x-forwarded-for from the right. Cloud Run appends the connecting peer, -// so the last value is the only one it wrote; everything to its left is -// whatever the caller sent and can be a fresh forgery on every request. -// trustedProxyHops is how many appenders sit between Cloud Run and the client -// (0 today, 1 once a load balancer fronts it). A header too short for that -// depth is not trusted at all and falls through to the shared bucket, which -// throttles rather than opens. +// Read from the right: Cloud Run appends the peer, while callers can forge earlier entries. Skip trusted proxy hops (0 direct, 1 behind LB); short chains share the throttled fallback bucket. export function readClientIp(context: Context, trustedProxyHops = 0): string { const hops = context.req diff --git a/cloud/apps/push/src/desktop-host-proof-interop.test.ts b/cloud/apps/push/src/desktop-host-proof-interop.test.ts index 654423b0de8..39ba0318d4c 100644 --- a/cloud/apps/push/src/desktop-host-proof-interop.test.ts +++ b/cloud/apps/push/src/desktop-host-proof-interop.test.ts @@ -6,9 +6,7 @@ import { PushHostChallengeStore } from './host-challenge-store.js' import { deriveHostFingerprint } from './host-fingerprint.js' import { openInMemoryPushDatabase } from './push-database.js' -// Why: the desktop answers challenges in a workspace this one cannot import. -// Both sides replay the same checked-in vector, so a transcript drift on -// either side fails in that side's own suite. +// Desktop is a separate workspace; replay one checked-in vector in both suites to catch transcript drift. describe('desktop host proof interop', () => { it('the checked-in vector answers to the same proof the fixture host computes', () => { const secretKey = new Uint8Array(Buffer.from(vector.hostSecretKeyB64, 'base64')) diff --git a/cloud/apps/push/src/device-registry-store.ts b/cloud/apps/push/src/device-registry-store.ts index 1729f7d1b82..a4061189e35 100644 --- a/cloud/apps/push/src/device-registry-store.ts +++ b/cloud/apps/push/src/device-registry-store.ts @@ -52,8 +52,7 @@ export class PushDeviceRegistryStore { private readonly now: () => number = Date.now ) {} - // The registration id is stable for a (host, device) pair so a re-registered - // phone keeps the id the desktop already persisted; only the token rotates. + // Stable host/device registration IDs preserve desktop references across phone re-registration; only tokens rotate. async upsert(input: PushDeviceUpsert): Promise { const now = this.now() return await this.database.transaction(async (transaction) => { diff --git a/cloud/apps/push/src/durable-push-store.ts b/cloud/apps/push/src/durable-push-store.ts index eee2e5e2666..1bcdda0bdc7 100644 --- a/cloud/apps/push/src/durable-push-store.ts +++ b/cloud/apps/push/src/durable-push-store.ts @@ -109,8 +109,7 @@ export class DurablePushStore { async claim(canClaim?: () => boolean): Promise { return this.claims.transaction(async (tx) => { if (canClaim && !canClaim()) return null - // The previous revision's claim holds this key exclusively; sharing it makes that claim snapshot - // only after our leases commit. Drop one release after every worker runs this revision. + // Share the previous revision’s exclusive claim key so it snapshots after our leases commit; drop one release after all workers upgrade. if (!(await tx.tryLockSharedScope('push-worker-claim'))) return null const now = this.now() // SQLite already serializes the whole transaction. @@ -134,8 +133,7 @@ export class DurablePushStore { }) } - // The row lock alone lets a second claimer skip this device's head and lease the row behind it. - // Under the device lock, a fresh read sees every committed lease and every uncommitted head. + // Row locks alone allow leasing behind a skipped device head; device locking plus reread sees committed leases and uncommitted heads. private async ownsDeviceHead(tx: PushDatabase, row: SqlRow, now: number): Promise { const registrationId = String(row.registration_id) if (!(await tx.tryLockScope(`push-device:${registrationId}`))) return false diff --git a/cloud/apps/push/src/fcm-access-token.ts b/cloud/apps/push/src/fcm-access-token.ts index 542e0e8d0ed..9ac8f755d0d 100644 --- a/cloud/apps/push/src/fcm-access-token.ts +++ b/cloud/apps/push/src/fcm-access-token.ts @@ -1,9 +1,7 @@ import { GoogleAuth } from 'google-auth-library' import { FCM_SCOPE } from './fcm-client.js' -// Resolves the runtime service account credential from the GCE metadata server -// in Cloud Run and from GOOGLE_APPLICATION_CREDENTIALS locally; the library -// caches and refreshes the token itself. +// GoogleAuth handles metadata credentials on Cloud Run and GOOGLE_APPLICATION_CREDENTIALS locally, including cache/refresh. export function createFcmAccessTokenProvider(): () => Promise { const auth = new GoogleAuth({ scopes: [FCM_SCOPE] }) return async () => { diff --git a/cloud/apps/push/src/host-challenge-answering.test-fixture.ts b/cloud/apps/push/src/host-challenge-answering.test-fixture.ts index a1ece237ff5..ac5320cd1ab 100644 --- a/cloud/apps/push/src/host-challenge-answering.test-fixture.ts +++ b/cloud/apps/push/src/host-challenge-answering.test-fixture.ts @@ -9,8 +9,7 @@ import nacl from 'tweetnacl' import { decodeCanonicalBase64 } from './canonical-base64.js' import { deriveHostFingerprint } from './host-fingerprint.js' -// The desktop side of the push challenge, written the way the shipped host -// will answer it, so the gateway is exercised against a real box-opening peer. +// Model shipped desktop challenge answering so gateway tests use a real box-opening peer. const textEncoder = new TextEncoder() const textDecoder = new TextDecoder() diff --git a/cloud/apps/push/src/host-session-store.ts b/cloud/apps/push/src/host-session-store.ts index 899bacabcc8..5ee1f2aa6a0 100644 --- a/cloud/apps/push/src/host-session-store.ts +++ b/cloud/apps/push/src/host-session-store.ts @@ -27,9 +27,7 @@ export class PushHostSessionStore { const createdAt = this.now() const expiresAt = createdAt + PUSH_LIMITS.sessionTtlMs await this.database.transaction(async (transaction) => { - // Why: a desktop holds one session at a time and only re-proves once it is - // gone, so an earlier row is dead weight. It also bounds the table to one - // row per host however many proofs a self-minted identity answers. + // One active session per desktop bounds rows even for self-minted hosts that flood proofs. await transaction.lockQuotaScope(`orca-push-session:${hostFingerprint}`) await transaction.query('DELETE FROM push_sessions WHERE host_fingerprint = ?', [ hostFingerprint diff --git a/cloud/apps/push/src/push-background.ts b/cloud/apps/push/src/push-background.ts index 9fad4700e7b..25b98d0d467 100644 --- a/cloud/apps/push/src/push-background.ts +++ b/cloud/apps/push/src/push-background.ts @@ -6,8 +6,7 @@ const CHALLENGE_PRUNE_INTERVAL_MS = 60_000 const SESSION_PRUNE_INTERVAL_MS = 10 * 60_000 const DELIVERY_PRUNE_INTERVAL_MS = 60_000 -// Chained rather than periodic, so a sweep spanning many bounded statements never overlaps itself and -// one that exhausted its batch budget resumes at once instead of idling out the rest of the interval. +// Chain sweeps to prevent overlap; budget exhaustion resumes immediately instead of waiting another interval. function prune(label: string, run: () => Promise, intervalMs: number): () => void { let timer: NodeJS.Timeout | undefined let overdue: NodeJS.Timeout | undefined diff --git a/cloud/apps/push/src/push-database-postgres-startup.test.ts b/cloud/apps/push/src/push-database-postgres-startup.test.ts index b6fd23071fc..5f5520aee1b 100644 --- a/cloud/apps/push/src/push-database-postgres-startup.test.ts +++ b/cloud/apps/push/src/push-database-postgres-startup.test.ts @@ -73,8 +73,7 @@ describe('PostgreSQL push gateway startup', () => { await database.close() }) - // Why: a CREATE INDEX on a grown table can outlive the 5s request deadline, - // and a schema that inherits it fails every startup at the same statement. + // Growing-table CREATE INDEX can exceed request deadlines; inheriting them makes startup fail forever at the same DDL. it('applies the schema on an untimed pool that is gone before the serving pool opens', async () => { const database = await openPushDatabase({ databaseUrl: 'postgresql://push@localhost:55440/orca_push', diff --git a/cloud/apps/push/src/push-database.ts b/cloud/apps/push/src/push-database.ts index 9957a7c28b5..41b6cf9696b 100644 --- a/cloud/apps/push/src/push-database.ts +++ b/cloud/apps/push/src/push-database.ts @@ -55,8 +55,7 @@ class SqliteTransaction implements PushDatabase { return await operation(this) } - // BEGIN IMMEDIATE already holds the single writer lock for the whole - // transaction, so there is nothing narrower left to take. + // BEGIN IMMEDIATE already holds SQLite’s single writer lock. async lockQuotaScope(): Promise {} async tryLockScope(): Promise { @@ -71,8 +70,7 @@ class SqliteTransaction implements PushDatabase { } class SqliteDatabase extends SqliteTransaction { - // node:sqlite is synchronous and has no nested transactions, so overlapping - // callers are serialized behind one tail promise instead of racing BEGIN. + // Serialize callers: synchronous node:sqlite has no nested transactions, so overlapping BEGIN would race. private tail: Promise = Promise.resolve() override async query(sql: string, params: unknown[] = []): Promise { @@ -147,8 +145,7 @@ class PostgresTransaction implements PushDatabase { function retryablePostgresTransactionError(error: unknown): boolean { const code = String((error as { code?: unknown }).code) - // 57014 is the pool statement_timeout firing. It aborts the transaction the - // same way a lock timeout does, so it takes the bounded retry path too. + // 57014 statement timeout aborts transactions like lock timeout; use the same bounded retry. return code === '40P01' || code === '40001' || code === '55P03' || code === '57014' } @@ -205,8 +202,7 @@ class PostgresDatabase implements PushDatabase { throw new Error('postgres_transaction_retry_exhausted') } - // An advisory transaction lock taken outside a transaction is released by the - // implicit commit before the caller reads anything, which protects nothing. + // Outside an explicit transaction, implicit commit releases the advisory lock before any protected read. async lockQuotaScope(): Promise { throw new Error('lock_quota_scope_requires_transaction') } @@ -228,11 +224,7 @@ async function applySchema(database: PushDatabase): Promise { for (const statement of pushSchemaStatements()) await database.query(statement) } -// Why: DDL is not a request. A CREATE INDEX on a grown table can legitimately -// outlive the request statement_timeout, and inheriting it would fail every -// startup at the same statement instead of finishing once. One connection of -// its own, closed before the serving pool opens, keeps the untimed session off -// the request path entirely. +// DDL may outlast request statement_timeout; apply once on a separate untimed connection, closed before serving, or every startup repeats the failure. async function applySchemaOnUntimedPool( databaseUrl: string, applicationName: string | undefined @@ -251,8 +243,7 @@ async function applySchemaOnUntimedPool( try { await applyPostgresSchema(pushSchemaStatements(), (statement) => database.query(statement), { eventPrefix: 'orca_push_postgres_schema', - // Push has no catalog pre-check, so a lock timeout here says nothing about whether the - // object already exists and the old bounded retry is still the right answer. + // No catalog pre-check: lock timeout cannot prove DDL already exists, so retain bounded retry. retryLockTimeout: true }) } finally { @@ -262,8 +253,7 @@ async function applySchemaOnUntimedPool( export function absorbPostgresIdleClientErrors(pool: Pick): void { pool.on('error', () => { - // node-postgres removes failed idle clients itself; an unhandled 'error' - // would crash the service and turn a SQL blip into a restart loop. + // Handle idle pg errors after the pool removes failed clients; otherwise transient SQL faults crash-loop the service. console.warn('[orca-push] idle PostgreSQL client failed') }) } diff --git a/cloud/apps/push/src/push-server.ts b/cloud/apps/push/src/push-server.ts index 17c72b14b16..a174a3994f1 100644 --- a/cloud/apps/push/src/push-server.ts +++ b/cloud/apps/push/src/push-server.ts @@ -44,8 +44,7 @@ export function readBearer(header: string | undefined): string | null { return scheme?.toLowerCase() === 'bearer' && token.length > 0 ? token : null } -// Hono's body limit, not a Content-Length check: a chunked body declares no -// length, and req.json() would buffer all of it before any handler ran. +// Enforce Hono’s body limit: Content-Length misses chunked bodies that req.json would fully buffer. const limitBody = bodyLimit({ maxSize: PUSH_LIMITS.maxHttpBodyBytes, onError: (context) => context.json({ error: 'request_too_large' }, 413) @@ -119,8 +118,7 @@ export function createPushServer( const app = new Hono<{ Variables: PushVariables }>() const requestDrain = new PushRequestDrain() app.use('*', requestDrain.middleware) - // Hono's default handler prints the whole error, and a pg error carries the - // offending row in `detail`. Only the error's name may reach the logs. + // Default Hono logging exposes pg row detail; log only error names. app.onError((error, context) => { observability.record('request_error') console.warn( diff --git a/cloud/apps/push/src/push-worker-concurrency.ts b/cloud/apps/push/src/push-worker-concurrency.ts index daec5659d21..10b637c45ff 100644 --- a/cloud/apps/push/src/push-worker-concurrency.ts +++ b/cloud/apps/push/src/push-worker-concurrency.ts @@ -1,3 +1,2 @@ -// Twelve drains lift the ~30/s ceiling four drains hit at ~120 ms per item; each drain holds one -// delivery in flight, so this is the worker's concurrency, not its database draw. +// Twelve concurrent deliveries exceed the ~30/s limit of four ~120ms drains; database concurrency is bounded separately. export const WORKER_DRAINS = 12 diff --git a/cloud/apps/relay/src/admin-route-transient-app.test.ts b/cloud/apps/relay/src/admin-route-transient-app.test.ts index e326d8f561e..5ee5cbb9424 100644 --- a/cloud/apps/relay/src/admin-route-transient-app.test.ts +++ b/cloud/apps/relay/src/admin-route-transient-app.test.ts @@ -17,8 +17,7 @@ vi.mock('./relay-token-verifier.js', () => ({ import { createRelayApp } from './app.js' -// The message the pool raises when its own dial outruns connectionTimeoutMillis. -// This is the shape that failed a rollout wave as an HTTP 404. +// Actual pg-pool timeout shape that a rollout incorrectly surfaced as 404. const poolTimeout = () => new Error('Connection terminated due to connection timeout') function adminRequest(body: unknown): RequestInit { diff --git a/cloud/apps/relay/src/app.ts b/cloud/apps/relay/src/app.ts index e0f2da12dc0..1604f16dc3f 100644 --- a/cloud/apps/relay/src/app.ts +++ b/cloud/apps/relay/src/app.ts @@ -97,8 +97,7 @@ const ASSIGNMENT_REJECTION_LOG_WINDOW_MS = 10_000 // A release holds the row for about one lock timeout, so one second is enough. const ASSIGNMENT_ROW_BUSY_RETRY_AFTER_SECONDS = 1 const REGION_CATALOG_CACHE_MS = 30_000 -// A drain that outlives the roll step it belongs to is an outage, not a pacing win. The roll -// step's timeout scales with the window, up to the same-cap wave's slowest pace. +// Drain pace must fit the roll-step timeout, including its slowest same-cap wave. const DRAIN_PACE_WINDOW_MAX_MS = 20 * 60 * 1_000 type AdmissionRejectionLogEntry = { @@ -262,9 +261,7 @@ export function createRelayApp( context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds)) return context.json({ error: 'assignments_temporarily_unavailable' }, 503) } - // Reconnecting hosts declare themselves and are verified against the durable - // assignment inside this bounded lane, so a placement backlog can never - // starve session recovery. Unhinted traffic never touches this lane. + // Verify hinted reconnects against durable assignments in a separate bounded lane so placement backlogs cannot starve recovery. const stickyRetryAfterSeconds = config.publicStickyRetryAfterSeconds ?? 2 const stickyAssignmentAdmission = new RelayPublicAssignmentAdmission({ maxConcurrent: config.publicStickyConcurrency ?? 1, @@ -310,10 +307,7 @@ export function createRelayApp( context.header('Retry-After', String(retryAfterSeconds)) return context.json({ error: 'assignments_temporarily_unavailable' }, 503) } - // An admin route that collapses every failure into one status cannot tell a - // real conflict from a database that was briefly out of reach, and the rollout - // tooling retries on 503 only. Transient failures get the answer the public - // routes already give; everything else keeps the route's own mapping. + // Rollout tooling retries only 503; classify transient admin DB failures like public routes while preserving other route mappings. const rejectAdminOperation = ( context: Context, error: unknown, @@ -327,9 +321,7 @@ export function createRelayApp( context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds)) return context.json({ error: 'database_temporarily_unavailable' }, 503) } - // Aggregate counters cannot separate a handful of pathological hosts from a broad - // population, so every admission rejection names its host and reason. Keyed on - // route:lane:reason rather than host, the log stays bounded under load. + // Log host/reason to distinguish isolated failures; bound windows by route:lane:reason to stay bounded across hosts. const rejectionLogWindow = new AssignmentRejectionLogWindow({ windowMs: ASSIGNMENT_REJECTION_LOG_WINDOW_MS, onWindowClosed: ({ suppressed, sample }) => logAssignmentRejection({ ...sample, suppressed }) @@ -383,9 +375,7 @@ export function createRelayApp( demote: operations.cellDemote }) - // Not /healthz: Google Front End reserves that path before the container. - // The drain pace cap is read before a roll isolates a cell, so a slower pace than the cell - // accepts stops with the cell untouched. + // Google Front End reserves /healthz. Report drain cap before isolation so an unsupported pace refuses without touching the cell. app.get('/health', (context) => context.json({ ok: true, @@ -689,10 +679,7 @@ export function createRelayApp( admission.release() } recordSelection(assignment, assignment.placed === true) - // Grant-side counterpart of the rejection log: reconnect grants are rare - // enough to log and make "which cell is this host on" answerable. The - // placement-lane ones matter most — they are the only record that a host - // whose sticky lane failed verification landed anywhere at all. + // Reconnect grant logs locate hosts; placement-lane grants prove where failed sticky verifications eventually landed. if (body.data.reconnect) { console.warn( `[orca-relay] assignment granted lane=${lane} hinted=true` + @@ -1264,8 +1251,7 @@ export function createRelayApp( return rejectAdminOperation(context, error, 409) } }) - // Step 5: the flag workflow's record of a cell's admitMode, which every sweep reads. Without - // `admitMode` it only reads; reserve is refused while a drain, migration or rehome is open. + // Read without admitMode; writes feed every sweep and refuse reserve during open drain/migration/rehome flows. app.post('/v1/admin/cell-admit-mode', async (context) => { const bearer = readBearer(context.req.header('authorization')) if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) @@ -1856,9 +1842,7 @@ const RegionalRehomeSafetySchema = z databasePoolWaiting: z.number().int().nonnegative().max(100), databasePoolWaitersMax: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), databasePoolWaitMsMax: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), - // Cells spread the full pool-pressure counts into the payload; rejecting - // the extra gauges 400'd every rehome-status heartbeat since 2026-08-15, - // so no cell ever recorded rehome protocol 1. + // Accept cell pool gauges or strict validation rejects every rehome-status heartbeat, preventing protocol-1 registration. databasePoolTotal: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(), databasePoolIdle: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(), databasePoolOldestWaitMs: z @@ -2071,13 +2055,7 @@ const AdminAdmissionSelectorApplySchema = z expectedGeneration: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), expectedMembershipSha256: z.string().regex(/^[a-f0-9]{64}$/).optional(), membership: AdmissionSelectorMembershipSchema, - // Optional, so an older caller reaching an updated director is unchanged: - // the cell goes unmarked and its hosts stay pinned, today's behaviour. The - // other direction is NOT ignored — the schema below is .strict(), so an - // updated caller reaching an older director is a 400. That fails closed, - // before the isolate step sets MUTATION_STARTED and before anything is - // written, but it is a deploy ordering constraint: the director ships - // first, then any workflow run that uses the updated script. + // Deploy director before callers: old callers omit this safely, but old strict directors reject it with 400 before isolation/MUTATION_STARTED, leaving hosts pinned. rollIsolatedCells: z.array(CellIdSchema).max(256).optional() }) .strict() @@ -2417,8 +2395,7 @@ function operationError(error: unknown): string { } -// `suppressed` is present only on a window-closing line, and counts the rejections -// that line stands for beyond the one already logged when the window opened. +// Only window-close logs carry suppressed, excluding the rejection already logged at window open. function logAssignmentRejection(input: { route: 'assign' | 'assign-verify' | 'resolve' lane: AssignmentAdmissionLane | 'none' diff --git a/cloud/apps/relay/src/assignment-dead-reserve-cell-postgres.test.ts b/cloud/apps/relay/src/assignment-dead-reserve-cell-postgres.test.ts index 00e884c7b6f..e64ccada30a 100644 --- a/cloud/apps/relay/src/assignment-dead-reserve-cell-postgres.test.ts +++ b/cloud/apps/relay/src/assignment-dead-reserve-cell-postgres.test.ts @@ -5,8 +5,7 @@ import type { RelayCellConfig } from './config.js' import { openRelayDatabase, type RelayDatabase } from './database.js' import { ReserveLedgerWriter } from './reserve-ledger-writer.js' -// A host that lost its control leaves a capped reserve cell this director proves dead (stale -// heartbeat + its own polls unanswered) without an operator fence; every live-cell case keeps it. +// Only host-requested moves may leave a reserve cell proven dead by heartbeat and poll silence without a fence; live cells retain hosts. const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip diff --git a/cloud/apps/relay/src/assignment-isolated-cell-replacement-postgres.test.ts b/cloud/apps/relay/src/assignment-isolated-cell-replacement-postgres.test.ts index b7deda35921..e10b05fe9af 100644 --- a/cloud/apps/relay/src/assignment-isolated-cell-replacement-postgres.test.ts +++ b/cloud/apps/relay/src/assignment-isolated-cell-replacement-postgres.test.ts @@ -273,11 +273,7 @@ describePostgres('PostgreSQL re-placement off a cell isolated for a roll', () => await applySelector(Object.fromEntries(CELLS.map((cell) => [cell.id, 'general']))) } - // The selector is fleet-wide and this database is shared with every other - // Postgres file in the project, all of which write admission through the - // generation-0 helpers. Advancing the generation and leaving it advanced - // would fail every one of them with admission_selector_boundary_active, so - // this file puts the boundary back exactly as it found it. + // Shared PostgreSQL suites use generation-0 helpers; restore the exact selector boundary or later files fail admission_selector_boundary_active. async function resetSelectorBoundary(): Promise { await databases[0]!.query( `UPDATE relay_admission_selectors SET generation = 0, attempt_id = NULL @@ -329,9 +325,7 @@ describePostgres('PostgreSQL re-placement off a cell isolated for a roll', () => afterAll(async () => { if (databases[0]) { await deleteHostRows() - // Order matters: drop the cells first, then rebuild the selector's - // membership from what is left, or it names rows that no longer exist and - // every later read fails admission_selector_membership_drift. + // Drop cell rows before rebuilding membership or later reads fail admission_selector_membership_drift. for (const cell of CELLS) { for (const table of [ 'relay_cell_connection_snapshots', @@ -497,9 +491,7 @@ describePostgres('PostgreSQL re-placement off a cell isolated for a roll', () => ) expect(granted.length).toBeGreaterThan(0) - // However many dials were granted, only one re-placement may have - // committed: the epoch advances by exactly one and the target gains exactly - // one unit. The source keeps its lease until the source releases it. + // Concurrent dials may grant repeatedly but commit one move: one epoch/unit on target, preserving source leases until release. const settled = await stores[0]!.resolve(identity) expect(settled?.assignmentEpoch).toBe(first.assignmentEpoch + 1) expect(settled?.cellId).not.toBe(ISOLATED.id) diff --git a/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts b/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts index 8abe1b3678d..96df81c71b5 100644 --- a/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts +++ b/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts @@ -17,14 +17,7 @@ import { type SqlRow } from './database.js' -// `migration-only` alone says nothing about a roll: it is the admission class -// (cloud/docs/orca-relay-operations.md:229-233) that evacuation targets, Asia -// `--mode rollback`, a failed wave's re-isolate and newly registered cells all -// occupy durably while holding hosts. The same-cap roll's isolate step is the -// only writer of the roll stamp, via `rollIsolatedCells` on the selector apply -// (cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs isolate mode); -// restore moves the cell to 'general', which clears the stamp in the same -// statement that writes the state. +// migration-only also holds evacuation targets, Asia rollback and failed-wave/new cells; only same-cap isolate writes rollIsolatedCells, and restore clears its stamp atomically (orca-relay-operations.md). const HEARTBEAT_TTL_MS = 45_000 const START_MS = 100 const IDENTITY = { userId: 'user-a', relayHostId: 'host000000000001' } @@ -307,9 +300,7 @@ describe('re-placing a host off a cell isolated for a roll', () => { }) it('keeps a host pinned to a migration-only cell with no roll stamp', async () => { - // The guard for every non-roll flow that parks a loaded cell: an Asia - // `--mode rollback`, an evacuation target awaiting promotion, a failed - // wave's re-isolate. Moving these hosts would undo the operator's intent. + // Unstamped non-roll parking must retain hosts or undo evacuation/Asia rollback/operator intent. const { store, park, rollIsolatedAt } = await setup() const first = await store.assign(IDENTITY, 'us-central1') await park(first.cellId, 'migration-only') @@ -344,10 +335,7 @@ describe('re-placing a host off a cell isolated for a roll', () => { { label: 'just inside the bound', age: 2 * 60 * 60_000 - 1, moves: true }, { label: 'just outside the bound', age: 2 * 60 * 60_000 + 1, moves: false } ])('treats a stamp $label as $moves', async ({ age, moves }) => { - // Why the bound exists: a roll isolates and restores inside ~15 minutes, so - // an older stamp is a failed wave waiting on an operator, or an orphan left - // by a director rollback whose restore predates the clearing clause. Both - // mean a possibly healthy cell, and the safe answer is the pre-existing one. + // Past a normal ~15-minute roll, stamps may be failed-wave or old-restore orphans; keep possibly healthy hosts pinned for operator recovery. const { store, heartbeat, isolateForRoll, setNow } = await setup() const first = await store.assign(IDENTITY, 'us-central1') await isolateForRoll(first.cellId) @@ -421,9 +409,7 @@ describe('re-placing a host off a cell isolated for a roll', () => { }) it('keeps a host pinned on an unstamped cell whose migration has completed', async () => { - // The terminal state of a successful evacuation: the host's row points at - // the target, the migration is completed, and the cell waits migration-only - // for a separate promote dispatch. Re-placing here undoes the evacuation. + // Completed evacuations await separate promotion in migration-only; re-placement would undo them. const { store, database, park } = await setup() const first = await store.assign(IDENTITY, 'us-central1') await insertMigration(database, { @@ -474,9 +460,7 @@ describe('re-placing a host off a cell isolated for a roll', () => { }) it('keeps the pin while a migration row is open but its lease has lapsed', async () => { - // Why: the durable relay_assignment_migrations row outlives the 15-minute - // lease the counter tracks, and rollBackStalledRegionalRehomes refuses to - // unwind it while the source is not general. Re-placing would strand it. + // Migration rows outlive their 15-minute counters and cannot roll back from non-general sources; re-placement would strand them. const { store, database, isolateForRoll } = await setup() const first = await store.assign(IDENTITY, 'us-central1') await insertMigration(database, { @@ -723,9 +707,7 @@ describe('re-placing a host off a cell isolated for a roll', () => { }) it('emits nothing when the placement rolls back after the decision', async () => { - // Why: the decision and the writes share one transaction. A line already on - // stdout cannot be rolled back with it, so an event written where it is - // decided would have the canary counting moves that never happened. + // Log only committed moves; stdout cannot roll back with a failed lease/reservation transaction. const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) const { store, counter, isolateForRoll } = await setup() const first = await store.assign(IDENTITY, 'us-central1') @@ -739,15 +721,13 @@ describe('re-placing a host off a cell isolated for a roll', () => { ) expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([]) - // The precondition for reading that absence: the move really was rolled - // back, so the event would have been a lie rather than merely early. + // Prove the move rolled back so the missing event is required, not merely delayed. expect(await store.resolve(IDENTITY)).toMatchObject({ cellId: first.cellId, assignmentEpoch: first.assignmentEpoch }) - // Control: the same dial with nothing injected does emit, so the assertion - // above is measuring the rollback and not a broken harness. + // Successful control proves event absence above comes from rollback, not a broken harness. warn.mockClear() const moved = await store.assign(IDENTITY, 'us-central1') expect(moved.cellId).not.toBe(first.cellId) diff --git a/cloud/apps/relay/src/assignment-reserve-mode-postgres.test.ts b/cloud/apps/relay/src/assignment-reserve-mode-postgres.test.ts index 442f63744c7..3c8e7142d20 100644 --- a/cloud/apps/relay/src/assignment-reserve-mode-postgres.test.ts +++ b/cloud/apps/relay/src/assignment-reserve-mode-postgres.test.ts @@ -5,8 +5,7 @@ import { CELL_ADMIT_EFFECTIVE_FRESH_MS, RelayAssignmentStore } from './assignmen import type { RelayCellConfig } from './config.js' import { openRelayDatabase, type RelayDatabase } from './database.js' -// The step-5 census on real PostgreSQL: every new statement (the admit-mode table, the -// open-flow check under FOR UPDATE, the live-reserve pin) must run there, not only on SQLite. +// Exercise admit-mode SQL, locked open-flow checks and live-reserve pins on PostgreSQL, not SQLite alone. const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip diff --git a/cloud/apps/relay/src/assignment-roll-replaced-cell.test.ts b/cloud/apps/relay/src/assignment-roll-replaced-cell.test.ts index 02faa564c15..dfb06d4c748 100644 --- a/cloud/apps/relay/src/assignment-roll-replaced-cell.test.ts +++ b/cloud/apps/relay/src/assignment-roll-replaced-cell.test.ts @@ -9,9 +9,7 @@ import { import type { RelayCellConfig } from './config.js' import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js' -// Production 2026-10-24, production-gce-c15: a same-cap roll isolates the cell, drains it, -// then replaces its one VM. A host that was offline through the drain still names the cell, -// and redialled in the ~45 s before the new process was ready. +// 2026-10-24, production-gce-c15: offline hosts redialed during the ~45s gap between draining a same-cap roll and replacement readiness. const HEARTBEAT_TTL_MS = 45_000 const START_MS = 100 const IDENTITY = { userId: 'user-a', relayHostId: 'host000000000001' } diff --git a/cloud/apps/relay/src/assignment-row-lock-order-postgres.test.ts b/cloud/apps/relay/src/assignment-row-lock-order-postgres.test.ts index 4b3893654f3..5d76473dd01 100644 --- a/cloud/apps/relay/src/assignment-row-lock-order-postgres.test.ts +++ b/cloud/apps/relay/src/assignment-row-lock-order-postgres.test.ts @@ -26,11 +26,7 @@ const hosts = ['rowlockhost00001', 'rowlockhost00002', 'rowlockhost00003', 'rowl (relayHostId) => ({ userId, relayHostId }) ) -// Why a deadlock counter and not "it eventually succeeded": the transaction -// wrapper retries 40P01 three times, so a cycle that fires on every wave still -// reports success to the caller while burning the retry budget that turns into -// a 503 under load. PostgreSQL counts every detected cycle in pg_stat_database, -// which sees through the retry. +// pg_stat_database exposes cycles hidden by three 40P01 retries; eventual success alone misses deadlocks that exhaust retries under load. describePostgres('PostgreSQL row lock order', () => { const databases: RelayDatabase[] = [] diff --git a/cloud/apps/relay/src/assignment-store.test.ts b/cloud/apps/relay/src/assignment-store.test.ts index b5ed192760c..472c66667ad 100644 --- a/cloud/apps/relay/src/assignment-store.test.ts +++ b/cloud/apps/relay/src/assignment-store.test.ts @@ -1512,12 +1512,7 @@ describe('RelayAssignmentStore', () => { }) it('moves a stranded host off an existing-only cell that stopped serving it', async () => { - // Why: C3's decommission created an existing-only cell that rejects - // attaches; the #194 pin then loops returning hosts forever while each - // grant refreshes their own activity (issue #225). C3 has no - // connection-limits row and expired leases are cleaned within a - // maintenance cycle, so the only durable evidence is the assignment row: - // a recent grant with no live real activity behind it. + // C3 rejected attaches on an existing-only cell; repeated grants defeated dormancy (#194, #225). Use recent failed grants on assignment rows: no cap reservations exist, and expired leases are pruned. let now = 100 const store = await setup(() => now) const identity = { userId: 'user-a', relayHostId: 'host000000000001' } @@ -1562,11 +1557,7 @@ describe('RelayAssignmentStore', () => { }) it('keeps migration-only cells pinned inside the stranded window', async () => { - // Why: migration-only is an admission class, not a drain signal. Evacuation - // targets, an Asia `--mode rollback` and a failed wave's re-isolate all park - // loaded cells there durably, and moving those hosts would undo the - // evacuation or scatter the region. Only the same-cap roll's isolate stamp - // releases the pin, and nothing here sets it. + // migration-only durably holds evacuation/Asia rollback/failed-wave hosts; only same-cap roll stamps authorize re-placement. let now = 100 const store = await setup(() => now) const identity = { userId: 'user-a', relayHostId: 'host000000000001' } @@ -2648,8 +2639,7 @@ describe('RelayAssignmentStore', () => { expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 0 }) }) - // Why: completion needs an enabled target and expiry rollback needs an unregistered one, so a - // target disabled after it registered satisfies neither and the migration never leaves the table. + // Disabled registered targets satisfy neither completion nor expiry rollback, stranding migration rows. async function wedgeRegisteredMigration( now: () => number, cells: RelayCellConfig[], @@ -2715,8 +2705,7 @@ describe('RelayAssignmentStore', () => { // A disabled target can never satisfy completion, so waiting cannot help. expect(await store.completeReadyEvacuations()).toBe(0) expect(await store.abortExpiredEvacuations()).toBe(1) - // Rollback lands on the disabled source, so the host resolves to nothing and is placed afresh - // by evacuateDeadCells; the point is that the row is retired rather than reaped every tick. + // Rollback to disabled source resolves empty for evacuateDeadCells to re-place; retire the row rather than revisit every tick. expect(await store.resolve(identity)).toBeNull() expect(await store.abortExpiredEvacuations()).toBe(0) }) diff --git a/cloud/apps/relay/src/assignment-store.ts b/cloud/apps/relay/src/assignment-store.ts index 9a323f3d5db..42dd73c273b 100644 --- a/cloud/apps/relay/src/assignment-store.ts +++ b/cloud/apps/relay/src/assignment-store.ts @@ -173,13 +173,10 @@ export class RelayCellInReserveModeError extends Error { } } -// Step 5: the flag workflow records each cell's admitMode here (reserve before the cell's -// switch file, db only after the cell has re-registered its controls). Every per-host sweep -// and reader skips reserve cells, and does nothing while the set is unknown. +// Record reserve before the cell switch, db after control re-registration; per-host sweeps/readers skip reserve and stop while the set is unknown. const RESERVE_MODE_CACHE_MS = 5_000 const RESERVE_MODE_STALE_MS = 60_000 -// After a failed read, the last good one answers this long without a query: during a stall -// every assign would otherwise spend a read's timeout before its own transaction. +// Back off failed reserve-set reads using the last good answer or every stalled assign adds another timeout before BEGIN. const RESERVE_MODE_FAILURE_BACKOFF_MS = 5_000 // A cell refreshes its admit-effective row every 15 s; three missed refreshes and it counts for nothing. export const CELL_ADMIT_EFFECTIVE_FRESH_MS = 45_000 @@ -453,31 +450,12 @@ const CELL_RESERVATION_UPDATE = `UPDATE relay_cells SET reserved_requests = // About one release round trip from Asia, with margin; see assignStickyOnce. const STICKY_ASSIGNMENT_ROW_WAIT_MS = 1_000 -// THE ROW LOCK ORDER. Every transaction that takes more than one of these -// takes them in this order, whichever role it runs on: -// -// 1. relay_assignments (the host's row) -// 2. relay_assignment_migrations -// relay_assignment_activity_leases -// 3. relay_control_connection_reservations (lockControlConnectionReservations) -// 4. relay_cells (lockCellInventory / lockCellRows / the -// conditional reservation UPDATE) -// -// relay_cells is last because it is the only row shared by every host on a -// cell: a transaction that takes it early holds it across every round trip -// that follows, and on a cell far from PostgreSQL that is what turns accepts -// into a queue. Everything above it is per-host, so holding it longer costs -// only that host. Paths that read the inventory to make a placement decision -// cannot defer relay_cells, so they lock the host's rows from tiers 1-3 up -// front instead, before the inventory, and re-check what they read afterwards. -// Tier 1 is what serialises two transactions on the same host; the tiers below -// it keep transactions on *different* hosts from cycling through relay_cells. -// Why: one global FOR UPDATE over a 23-row table serialises every director and -// cell. At the 1s pool lock_timeout each blocked waiter also holds a pooled -// client for a full second, so the queue converts contention into pool -// exhaustion. The lock is held to COMMIT and the assignment path runs many -// statements after taking it, and no hold-time telemetry existed before this -// change, so 500ms is a first value to tune once cellInventoryHoldMsMax lands. +// Lock order across all roles: relay_assignments → relay_assignment_migrations/activity_leases → +// relay_control_connection_reservations → relay_cells (including conditional reservation UPDATE). +// Per-host locks precede the cell shared by all hosts to prevent cross-host deadlocks and shorten +// cell hold time across distant-Postgres round trips. Inventory-first paths acquire tiers 1–3 +// before inventory and recheck afterward. Global FOR UPDATE queues also exhaust pooled clients; +// tune the 500ms request wait using cellInventoryHoldMsMax. export const CELL_INVENTORY_LOCK_TIMEOUT_MS = 500 // The same inventory lock is taken by live requests and by background sweeps, @@ -487,29 +465,14 @@ export type CellInventoryLockMode = | 'request' // Never queue: the caller handles database_lock_unavailable and moves on. | 'nowait' - // A sweep can enter here, so keep the pool default. Failing sooner would turn - // ordinary contention into a 55P03 the retry wrapper reports as terminal, which - // spends the incident gate's bounded exhausted-retry budget (300 per 5 min). + // Keep pool default for sweeps; an early 55P03 becomes terminal retry exhaustion and spends the incident gate’s 300/5min budget. | 'pool-default' -// Why: stranded detection (issue #225) needs a grant old enough that a real -// attach would have registered (the 90s activity lease covers dial + -// activation), yet recent enough to prove an active retry loop rather than -// ordinary dormancy — which stays governed by the 24h rule. +// Stranded grants must be old enough to attach (90s lease), yet recent enough to prove a retry loop; ordinary dormancy uses 24h (#225). const STRANDED_MIN_GRANT_AGE_MS = 60_000 const STRANDED_RECENT_ACTIVITY_MS = 15 * 60_000 -// Why: a roll's isolate step writes exactly this state -// (cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs:136) and its -// restore writes 'general' (same line). 'existing-only' is C3's decommission -// posture, where the cell still serves the hosts it already has, so it is -// deliberately not an isolation signal here. +// Same-cap isolate writes migration-only and restore writes general; existing-only still serves incumbents and cannot signal a roll. const ROLL_ISOLATED_ADMISSION: CellAdmissionState = 'migration-only' -// Why the stamp expires: a roll isolates and restores one cell inside ~15 -// minutes, so a stamp older than this is not a roll in progress. It is a failed -// or stalled wave whose failsafe re-isolated a possibly healthy cell and is -// waiting on an operator, or a director rollback whose restore wrote 'general' -// without the clause that clears the stamp, leaving an orphan that the next -// park would reactivate. Both want the same answer, and it is the pre-existing -// one: keep the pin and let the host retry its own cell. +// A stamp older than a ~15-minute roll may be a failed wave or old-restore orphan; keep the possibly healthy pin for operator recovery. const ROLL_ISOLATION_STAMP_MAX_AGE_MS = 2 * 60 * 60_000 const REGION_PREFERENCE_RETENTION_MS = 30 * 24 * 60 * 60_000 const REGIONAL_REHOME_UNREGISTERED_REFRESH_MS = 5 * 60_000 @@ -579,20 +542,14 @@ class AssignmentCapacityExhausted extends Error { } } -// Why a reason of its own: a host whose home cell is fenced-but-unattested is -// refused regardless of fleet headroom, so reporting it as capacity sends -// operators after capacity that was never short. Every cell boot and every -// readiness dip produces these. +// Fenced but unattested home cells refuse regardless of headroom; classify separately or operators chase capacity that was never short. export type RelayHomeCellUnavailableCause = | 'draining' | 'booting' | 'unheard' | 'not_ready' -// The host's own row is held, almost always by that host's release on its old -// cell, which sits queued on that cell's busy row. Waiting here would hold the -// director's single sticky slot for the length of that queue, so the dial is -// refused at once and the host redials after the release has settled. +// Own release often holds this host row while queued on the cell; refuse rather than monopolize sticky admission until that queue drains. export class RelayAssignmentRowBusyError extends Error { constructor() { super('relay_assignment_row_busy') @@ -608,19 +565,11 @@ export class RelayHomeCellUnavailableError extends Error { } } -// Debt holds connection headroom for a control that may still arrive shortly -// after its director-side timeout. Nothing legitimately arrives minutes late -// (attach deadline 10s, orphan grace 30s); unretired debt from hosts that -// never return otherwise starves connection headroom fleet-wide and turns -// every placement into relay_capacity_exhausted. +// Retain headroom for shortly late controls (10s attach/30s orphan grace), then retire debt or absent hosts exhaust fleet capacity. const LATE_ARRIVAL_DEBT_RETENTION_MS = 10 * 60 * 1_000 -// A released reservation is read by nothing: every reader filters it out by state, and the only -// statement that still touches it is the per-host lock, which just makes that lock set longer. A day -// is margin for forensics, not for reads. +// Readers ignore released reservations, but host locks still include them; one-day retention is forensic margin. export const RELEASED_CONTROL_RESERVATION_RETENTION_MS = 24 * 60 * 60 * 1_000 -// ~27 rows per page, so a statement deletes a few hundred rows at most. With ~9 director ticks a -// minute the row cap is ~5M rows a day: the 13.7M-row backlog drains over about three days, and a -// walk that finds nothing reads ~1 MB a tick. +// Bound each statement to a few hundred rows; ~9 ticks/min permits ~5M rows/day and empty scans read ~1MB/tick. const RELEASED_CONTROL_RESERVATION_REAP_BUDGET = { pagesPerStatement: 16, maxPagesPerTick: 128, @@ -663,10 +612,7 @@ export class RelayAssignmentStore { private readonly regionalRehomeDirectorRegion?: RelayRegion private readonly requireLiveCells: boolean private readonly heartbeatTtlMs: number - // Poisoned attempts never complete or abort and stay the oldest rows, so - // unquarantined they eventually fill the sweeps' LIMIT page and starve - // every healthy candidate. Process-local on purpose: it resets on deploy - // and each director relearns within a few ticks. + // Quarantine permanent oldest failures out of LIMIT pages so healthy candidates progress; process-local state relearns within ticks after deploy. private readonly regionalRehomeCandidateQuarantine = new Map< string, { failures: number; until: number } @@ -717,10 +663,7 @@ export class RelayAssignmentStore { this.migrationCellRegistrar = new RelayMigrationCellRegistrar(database, now) } - // Null when unreadable for a minute: callers then treat every cell as possibly reserve. - // Inside a transaction (any `database` but the pool) it answers from the last read and never - // runs a statement: a failed read there (57014) would abort the transaction, and the next - // statement's 25P02 is not retried. Callers read once before they begin. + // Read before BEGIN: failed in-transaction queries (57014) abort it and lead to unretried 25P02. Transactions use cache only; after a minute unreadable returns null so callers treat all cells as possibly reserve. async reserveModeCells( database: Pick = this.database ): Promise | null> { @@ -833,8 +776,7 @@ export class RelayAssignmentStore { return rows.length === 1 } - // A reserve cell still heartbeating holds its hosts in memory, ready or not (a database - // brownout drops ready): the same test as break glass, so neither re-places open sockets. + // Heartbeating reserve cells retain in-memory hosts even when DB brownouts clear ready; use break-glass liveness to avoid duplicate placement. private async liveReserveCell( cellId: string, database: RelayDatabase, @@ -1007,8 +949,7 @@ export class RelayAssignmentStore { identity: AssignmentIdentity, preferredRegion?: RelayRegion, placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION, - // evacuateDeadCells re-enters placement from a sweep; it must not take the - // bounded wait, whose 55P03 would surface as a terminal sweep failure. + // Sweep placement keeps pool default; a shorter bounded wait would surface 55P03 as terminal sweep failure. lockMode: CellInventoryLockMode = 'request', // Step 5: a fresh placement may first be booked on a reserve-mode cell, outside any // transaction; and a database mint stays above every epoch the director's map knew. @@ -1049,8 +990,7 @@ export class RelayAssignmentStore { } const booked = step5.placeFresh ? await bookReserve(step5.placeFresh) : null if (booked) return booked - // Only placement needs the global inventory critical section; queueing those - // attempts locally avoids turning true placement bursts into NOWAIT storms. + // Serialize placement’s global inventory section locally so bursts do not become NOWAIT storms. const placeOnDatabase = async (keepRegion: boolean) => await this.serializeAssignment( async () => @@ -1178,10 +1118,7 @@ export class RelayAssignmentStore { const now = this.now() await this.reserveModeCells() return await this.database.transaction(async (transaction) => { - // Why: the retry exists to take a cell row before the assignment row, the - // order placement uses. It only ever needs the one cell this host is - // pinned to, so read the pin unlocked and lock that row alone; taking all - // 23 queued every sticky refresh in the fleet behind every other one. + // Legacy-order retry locks only the unlocked pin’s cell before assignment; locking all cells would serialize fleet-wide sticky refreshes. const pinnedCellId = await this.pinnedCellId(transaction, identity) const lockedCells = !inventoryFirst || pinnedCellId === undefined @@ -1193,12 +1130,7 @@ export class RelayAssignmentStore { pinnedCellId === undefined ? undefined : await this.pinnedCellAdmission(transaction, pinnedCellId) - // Why wait at all: a calm host redialling after its own clean close meets - // its own release, and a refusal costs it the client's 5 s assign gate. On - // a live general cell that release is short, so a bounded wait is cheap. - // A roll, drain or dead cell keeps its release queued, so refuse at once. - // Never while holding the cell row: that reverses the lock order against - // this host's own release, which holds its row and wants the cell's. + // Wait briefly only for live general-cell releases to avoid the client’s 5s redial gate; never wait while holding cell first, which cycles against host-first release. const pinIsLiveGeneral = lockedCells === undefined && pinnedCellId !== undefined && @@ -1238,12 +1170,7 @@ export class RelayAssignmentStore { return null } - // Why: a null here means "fall through to placement", which is exactly - // what an isolated incumbent needs — and the only way out, because - // re-granting the pin refreshes the host's own activity and sustains the - // loop. The placement lane re-places it on a cell that will take it. - // Decided before the pinned cell row is touched: during a drain that row - // is the one the cell's releases keep busy. + // Fall through before touching the busy draining cell; re-granting its pin refreshes activity and sustains the retry loop. if ( await this.incumbentCellIsolatedForRoll( transaction, @@ -1334,18 +1261,7 @@ export class RelayAssignmentStore { }) } - // Why: PR #194 pins assignments on existing-only cells so capacity pressure - // cannot scatter existing hosts — assuming those cells still serve them. A - // decommissioning cell (C3) broke that assumption: existing-only AND - // rejecting attaches, so pinned hosts loop forever while each grant - // refreshes their own last_activity_at, keeping dormancy-based - // reassignment permanently out of reach (issue #225). "Stranded" therefore - // requires proof the cell is not serving this host: a recent grant - // (last_activity_at inside the window) that had ample time to attach and - // still produced no live real activity. The evidence must come from the - // assignment row itself — reservation rows do not exist for cells without - // connection limits (C3), and expired pending leases are cleaned within a - // maintenance cycle, so neither reliably survives until the next grant. + // Existing-only pins (#194) assume attaches still work; C3 decommission broke this (#225). Prove stranding from a recent grant old enough to attach but with no live activity: reservation rows may not exist and expired leases are pruned. private async assignmentStrandedOnUnservedCell( transaction: RelayDatabase, identity: AssignmentIdentity, @@ -1413,13 +1329,7 @@ export class RelayAssignmentStore { let retryScope: RetriedAssignmentInventoryScope = inventoryScope === 'none' ? 'general' : inventoryScope await this.reserveModeCells() - // Why the events ride back out rather than being written where they are - // decided: everything below runs in one transaction, and a reservation or - // lease write that fails after the decision rolls the placement back. A - // line already on stdout cannot be rolled back with it, so the canary would - // count re-placements that never happened. Returning them means only a - // committed attempt emits, and a transaction retry cannot leave a stale - // line behind either. + // Return events out of the transaction; stdout cannot roll back failed/retried placements and would inflate canary move counts. const outcome = await this.database.transaction(async (transaction) => { const events: string[] = [] // The retry paths below open with the inventory, so this path takes its @@ -1445,10 +1355,7 @@ export class RelayAssignmentStore { ) : undefined const existing = await this.assignmentRowOrBusy(transaction, identity) - // A dormant host holds no units, so its placement never writes its old - // cell and needs only the rows it could land on. - // A live reserve cell's hosts hold no activity rows here: idle there is not dormant. A - // dead one holds no duplicate, so its hosts are re-placed as any dormant host is. + // Dormant hosts hold no units and need only target locks. Live reserve cells keep activity in memory, so empty rows prove dormancy only once dead. const dormant = !existing || (mayNormallyReassign(activity(existing), now) && @@ -1469,10 +1376,7 @@ export class RelayAssignmentStore { let isolatedIncumbent: CellRow | undefined let isolatedTarget: CellRow | undefined let deadReserveSource: { cell: CellRow; verdict: DeadCellVerdict } | undefined - // Set when only the target rows are locked. The old cell's row is never - // written then, so the host must hold no units that no lease backs. - // Counters below the leases (drift the old reset left) are fine: the - // write below rebuilds them from the leases. + // Skipping source lock requires no units unbacked by leases; lower drifted counters are rebuilt from retained leases below. let sourceRowSkipped = false if (existing && !dormant && inventoryScope !== 'all') { // Unlocked on purpose: this only chooses which rows to lock. A restore @@ -1519,9 +1423,7 @@ export class RelayAssignmentStore { currentRow, await this.cellRegion(transaction, text(existing, 'cell_id')) ) - // Why: an existing-only cell that rejects attaches (C3's decommission - // posture) must not re-pin the hosts it refuses; the dead-cell fence - // path below does not apply either — the cell is live, just unwilling. + // Live existing-only cells rejecting attaches cannot re-pin hosts; dead-cell fencing does not apply to this C3 posture. strandedReassignment = await this.assignmentStrandedOnUnservedCell( transaction, identity, @@ -1532,12 +1434,7 @@ export class RelayAssignmentStore { strandedReassignment || !this.requireLiveCells || (await this.cellIsLive(transaction, current.cellId, now)) - // Why: the sticky lane sends an isolated incumbent here, and the gate - // below would hand the pin straight back — the cell is live and, being - // emptied, has more headroom than anyone. A cell that is *not* live is - // left to the dead-cell path below, whose fence is the only proof that - // an unreachable cell has stopped serving the sockets it still holds — - // unless a process started since the roll stamp proves the drained one gone. + // Drain headroom must not retain an isolated pin; unreachable cells need fencing unless a post-roll restart proves old sockets gone. if ( !strandedReassignment && (await this.incumbentCellIsolatedForRoll( @@ -1626,13 +1523,8 @@ export class RelayAssignmentStore { } connectionHeadroomReassignment = true } - // Why no fence for a live isolated cell: a fence proves a cell we cannot - // contact has stopped serving a host. This one is contactable and - // enforces the invariant itself — verifyCellAssignment reads - // relay_assignments live, so the epoch bump below makes it close any - // attach naming the old epoch with 4409. `isolatedIncumbent` is only set - // when the cell is live or restarted since the stamp, so a cell that - // stopped heartbeating while still holding sockets still takes this branch. + // Live isolated cells verify epochs (4409); a post-roll process restart proves the old sockets gone. + // Without either proof, the dead-cell path still requires fencing. if (!strandedReassignment && !connectionHeadroomReassignment && !isolatedIncumbent) { const fence = await this.deadCellFenceRequirement( transaction, @@ -1708,10 +1600,7 @@ export class RelayAssignmentStore { }) ) } - // Units come off the cell that holds them. A lease this placement deletes - // frees its units on its own cell; a lease it keeps stays counted until - // its own release or expiry. Only units no lease backs are charged to - // the old cell, which is locked whenever there are any. + // Freed leases charge their own cells; retained leases count until release/expiry. Only unbacked units charge the old cell, whose row must then be locked. const deletesLeases = forcedDeadReassignment || strandedReassignment const keptLeases = deletesLeases ? [] : activityLeases if (existing) { @@ -1720,10 +1609,7 @@ export class RelayAssignmentStore { await this.adjustCellReservation(transaction, text(existing, 'cell_id'), -unbackedUnits) } if (deletesLeases) { - // A fenced/dead incarnation cannot own drainable work, and a - // stranded host's only leases are the unclaimed grant artifacts of - // its own loop. Removing them prevents late expiry from - // decrementing the replacement. + // Dead/fenced incarnations have no drainable work; stranded leases are unclaimed loop artifacts. Delete them so late expiry cannot decrement replacements. for (const [cellId, units] of leaseUnitsByCell(activityLeases)) { await this.adjustCellReservation(transaction, cellId, -units) } @@ -3434,8 +3320,7 @@ export class RelayAssignmentStore { const cellIds = (await this.deadCellEvacuationCandidates(cutoff)).filter( (cellId) => !reserveCells.has(cellId) ) - // Why: without a candidate cell the host query below walks every assignment by primary key to - // return nothing (574 ms per call in production, from stale existing-only cells it can never act on). + // Without candidates, this scans all assignments for nothing (~574ms in production on stale existing-only cells). if (cellIds.length === 0) return 0 const rows = await this.database.query( `SELECT assignment.user_id, assignment.relay_host_id, assignment.cell_id @@ -4158,9 +4043,7 @@ export class RelayAssignmentStore { } const units = ACTIVITY_REQUEST_UNITS[input.kind] if (existing) { - // Why: a client-chosen activity id can move between cells, so lock the - // one or two rows this path touches in cell_id order, the same order - // placement takes the inventory in, and no cycle can form. + // Client activity IDs can change cells; lock both touched rows in inventory’s cell_id order to prevent cycles. await this.lockCellRows(transaction, [text(existing, 'cell_id'), input.cellId]) await this.removeActivityLease(transaction, identity, existing, now) await this.adjustCellReservationAtomically(transaction, input.cellId, units) @@ -4237,10 +4120,7 @@ export class RelayAssignmentStore { if (!control || Number(control.enabled) !== 1 || Number(control.not_before) > now) { return gated('control-closed') } - // The dispatch budget is durable and global, but until now only - // `commitIdleRegionalRehome` consulted it -- after the join had already run and - // the worker had already POSTed every candidate to its source cell. An absent - // row means the budget has never been spent, so it opens the gate. + // Check durable global dispatch budget before joining/POSTing candidates; absent worker state means no budget spent yet. const worker = (await this.database.query( `SELECT paused_until, next_dispatch_at FROM relay_region_rehome_worker_state WHERE worker_id = 'global'` @@ -4307,18 +4187,13 @@ export class RelayAssignmentStore { // either way, so no later candidate in this poll can get past it. let safetyPaused = false let targetDeferral: RegionalRehomeTargetDeferral | null = null - // Lock order: control, worker, host rows, then the target cell row alone, - // as the last statement. No fleet-wide lock: from a cell far from Postgres - // each statement costs a cross-region round trip, and the old inventory - // lock held every relay_cells row across ~20 of them. + // Lock control→worker→host→one target cell, last; fleet-wide locking previously held all cells across ~20 cross-region round trips. const result = await this.database.transaction(async (transaction): Promise => { safetyDisable = null safetyPaused = false targetDeferral = null const now = this.now() - // NOWAIT: these rows serialise the rate budget, the open-migration count - // and the hard-cap read, and a second commit must not queue behind one - // that is crossing the ocean. + // NOWAIT keeps global rate/migration/hard-cap budgets from queueing behind cross-region commits. const control = (await transaction.queryLocked( `SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`, [], @@ -4545,9 +4420,7 @@ export class RelayAssignmentStore { for (const [position, row] of ordered.entries()) outcomes[row.index] = parsed[position]! return outcomes } catch (error) { - // One statement means one contended assignment row can fail the whole - // batch, so a failure degrades to the per-host statements this replaced - // rather than costing every other host on the cell its renewal. + // One contended host can fail the batch; fall back to per-host writes so other leases still renew. console.warn( JSON.stringify({ event: 'orca_relay_control_renewal_batch_failed', @@ -4555,8 +4428,7 @@ export class RelayAssignmentStore { message: String((error as { message?: unknown }).message) }) ) - // No connection at all, or a lost reply: per-host statements would only queue up - // behind the same saturated pool or stalled server, multiplying the load. + // Skip fallback on connection loss/read timeout; per-host retries multiply load on the same stalled pool/server. if (isPostgresPoolConnectFailure(error) || isPostgresReadTimeout(error)) { return outcomes.fill('database_error') } @@ -4727,9 +4599,7 @@ export class RelayAssignmentStore { return (await this.activateControlOnce(identity, input, false)).activityId } - // Step 5 flip back: the same activation, but the cell-row delta is returned for the caller - // to fold into one relay_cells write per batch (commitCellReservationDelta), not committed - // here. A dozen re-registrations from Asia each holding that row's lock convoy on it. + // Flip-back returns deltas for one commitCellReservationDelta write per batch; separate cross-region re-registrations would convoy on the cell row. async activateControlDeferringCell( identity: AssignmentIdentity, input: ControlActivationInput @@ -4861,10 +4731,7 @@ export class RelayAssignmentStore { input.idleRegionalRehome && input.cellIncarnation ? 1 : 0 ] ) - // Last, and only if the count actually moved: the cell row is shared by - // every host on the cell, and this transaction spans a dozen round - // trips. Holding its write lock from the first of them capped a - // far-from-Postgres cell at a couple of accepts a second. + // Write shared cell row only last and only for nonzero delta; holding it across a dozen remote round trips capped accepts at a few per second. if (reservationDelta !== 0 && !deferCell) { await this.commitCellReservationAtomically( transaction, @@ -6321,9 +6188,7 @@ export class RelayAssignmentStore { return await this.readRegionalRehomeFleetSafety(this.database, this.now()) } - // The rehome fleet is every general cell that can be drained: those are the - // sources and, because a host must be movable back out again, the only legal - // targets. The region join stays so a cell with no region row is excluded. + // Both source and target must be general/drainable so hosts can move back out; retain region join to exclude unassigned cells. private async readRegionalRehomeFleetSafety( database: RelayDatabase, now: number @@ -6787,17 +6652,7 @@ export class RelayAssignmentStore { } } - // The rehome commit's only relay_cells lock: one row, NOWAIT, as the final - // statement. NOWAIT because the commit took the host's assignment row first, - // the reverse of placement's order; a wait here could deadlock with it. - // Enabled, admission and capacity are checked against the locked rows inside - // the statement, because an admission flip no longer serialises against this - // transaction any other way. Locking the admission row too makes a flip that - // committed after the statement's snapshot re-evaluate on its new version. - // Connection headroom is re-checked here too: the candidate read happened before this - // transaction's own reservation insert, and a placement may have reserved since. Every - // other reservation inserter holds this row first, so NOWAIT defers rather than races. - // The host's own reservation is excluded by key because the insert may have added none. + // Last statement locks one target row NOWAIT: host-first ordering must not wait against placement. Recheck enabled/admission/capacity and headroom under locks, including admission flips since snapshot; exclude own reservation by key because insert may add none. private async reserveRegionalRehomeTargetRow( database: RelayDatabase, input: { @@ -6894,9 +6749,7 @@ export class RelayAssignmentStore { RETURNING generation`, [now] ) - // The durable disable is otherwise invisible: nothing else records why - // claims stopped and inspection only shows enabled=false. Logged after - // the transaction commits so a rollback cannot fabricate the record. + // Return disable reason for logging after commit; inspection alone shows only enabled=false, and rolled-back decisions must not log. if (disabled.length === 0) return null return { event: 'orca_relay_regional_rehome_safety_disabled', @@ -7026,9 +6879,7 @@ export class RelayAssignmentStore { return completed } - // Repeated invariant failures quarantine the attempt out of the sweeps' - // LIMIT pages: poisoned rows are permanent and always the oldest, so - // without exclusion they eventually starve every healthy candidate. + // Quarantine repeated permanent failures or their oldest rows fill LIMIT pages and starve healthy candidates. private recordRegionalRehomeCandidateFailure( operation: 'complete' | 'abort' | 'refresh', attemptId: string, @@ -7312,11 +7163,7 @@ export class RelayAssignmentStore { }) } - // Why: counter skew left by a pre-fix sticky grant is reconstructible from the - // locked lease rows; lease shape and migration topology are not, so only a - // count mismatch is repaired and the re-assert still throws on anything else. - // reconcileReservationAccounting cannot stand in: it opens its own - // transaction, while this has to run inside the sweep's on rows it holds. + // Repair only counter skew from locked leases; shape/topology failures must still throw. reconcileReservationAccounting opens its own transaction, so cannot replace this in-sweep repair. private async repairAssignmentActivityCounts( transaction: RelayDatabase, identity: AssignmentIdentity, @@ -7393,11 +7240,7 @@ export class RelayAssignmentStore { return integer(completed[0]!, 'changes') + integer(aborted[0]!, 'changes') } - // A move the host never finished: it holds no activity on the source and is - // not present at the target, so the registered migration row can do nothing - // but occupy one of the eight concurrent slots until something clears it. - // Rolling it back leaves the durable assignment on the source, so the host - // lands where it started whenever it next reconnects. + // Unarrived hosts with no source activity waste one of eight migration slots; rollback leaves their original durable source for next reconnect. async abortUnarrivedRegionalRehomes(limit = 100): Promise { return await this.rollBackStalledRegionalRehomes({ sweep: 'abort-unarrived-regional-rehomes', @@ -7408,9 +7251,7 @@ export class RelayAssignmentStore { }) } - // The last-resort latch, and the only sweep that disables the switch. With - // the arrival sweep above running it should never reach a row; one that - // survives a day past dispatch means the rollback path itself is broken. + // Only this last-resort sweep disables rehoming: surviving a day past dispatch means normal arrival rollback failed. async abortExpiredRegionalRehomes(limit = 100): Promise { return await this.rollBackStalledRegionalRehomes({ sweep: 'abort-expired-regional-rehomes', @@ -7551,9 +7392,7 @@ export class RelayAssignmentStore { WHERE attempt_id = ?`, [now, input.abortReason, now, text(attempt, 'attempt_id')] ) - // Only the last-resort latch turns the feature off. A host that closed - // its laptop mid-move says nothing about whether rehoming is safe, and - // one such row a day would otherwise disable the switch every day. + // Only last-resort expiry disables the feature; a laptop closing mid-move is not evidence rehoming is unsafe. if (input.disableControl) { await transaction.query( `UPDATE relay_region_rehome_control @@ -7876,10 +7715,7 @@ export class RelayAssignmentStore { userId: text(candidate, 'user_id'), relayHostId: text(candidate, 'relay_host_id') } - // Why: re-check under the canonical assignment-first lock so a concurrent - // renewal wins without cleanup reaping its refreshed lease. - // Several directors sweep the same expired rows. Skip a row another - // director is settling instead of waiting and retrying the transaction. + // Recheck under assignment-first locks so concurrent renewals win; multiple directors skip busy rows instead of reaping refreshed leases or retrying contention. await this.assignmentRow(transaction, identity, true) const activityLeases = await this.lockAssignmentActivities(transaction, identity, true) const lease = activityLeaseById(activityLeases, text(candidate, 'activity_id')) @@ -8013,9 +7849,7 @@ export class RelayAssignmentStore { targetCellId ] ) - // Only the two cells this repairs need holding. The id set below is an - // existence check against a table that only reconcileCells writes, so it - // reads unlocked instead of dragging the other 21 rows into the section. + // Lock only the two repaired cells; this existence check reads the reconcileCells-owned table unlocked instead of holding the fleet. const cellIds = new Set( (await transaction.query(`SELECT cell_id FROM relay_cells`)).map((row) => text(row, 'cell_id') @@ -8134,13 +7968,7 @@ export class RelayAssignmentStore { return rows } - // Per-connection paths touch one or two cells. Locking exactly those rows, - // in the same ascending order the inventory lock uses (ORDER BY fixes the - // row-lock order), keeps them off the fleet-wide lock without a cycle. - // The wait policy follows the caller for the same reason the inventory lock's - // does: a sweep must not fail terminally on ordinary contention. Hold time is - // sampled only for a caller that names its site: short single-row holds under - // the inventory label would flatter the fleet-wide lock they replace. + // Lock touched cells in inventory’s ascending order to avoid cycles. Honor caller wait policy so sweeps tolerate contention; sample only named sites so short holds cannot flatter inventory metrics. private async lockCellRows( database: RelayDatabase, cellIds: string[], @@ -8157,10 +7985,7 @@ export class RelayAssignmentStore { ) } - // Tier 3 of the row lock order: a path that will take relay_cells and also - // touch this host's reservations takes them here, before the cell rows. The - // set is the host's own rows, so it is small and known before any placement - // decision is read. + // Lock this host’s reservations before any cell row (tier 3); the set is known before placement selection. private async lockControlConnectionReservations( database: RelayDatabase, identity: AssignmentIdentity, @@ -8283,9 +8108,7 @@ export class RelayAssignmentStore { // and an optional parameter left a second fleet-wide lock reachable here. rows: SqlRow[], preferredRegion: RelayRegion, - // Ordinary placement treats region as a preference and spills globally - // rather than refuse a host a cell. Re-placing off an isolated cell asks for - // its own region first, and leaves it only in a separate second tier. + // Ordinary placement spills globally; isolated incumbents exhaust their own region before a separate cross-region tier. regionMode: 'prefer' | 'require' = 'prefer' ): Promise { const regions = new Map( @@ -8579,22 +8402,12 @@ export class RelayAssignmentStore { ) } - // Why: a cell isolated for a roll keeps heartbeating ready=1 for the whole - // drain while its registry refuses every attach with 4503, so `cellIsLive` - // cannot see that the host's home has stopped taking it. Re-granting the pin - // is what turns a roll into 13-16 minutes of retry loop; returning true here - // sends the host down the ordinary placement lane on its next dial instead. - // - // The stamp, not the admission state, is the signal. 'migration-only' is an - // admission class that evacuation targets, Asia `--mode rollback`, a failed - // wave's re-isolate and newly registered cells all occupy durably while - // holding hosts; moving those would undo an evacuation or scatter a region. + // Roll-isolated cells heartbeat ready but reject attaches with 4503, so cellIsLive misses retry loops. Require a roll stamp: migration-only also durably holds evacuation/Asia rollback/failed-wave/new-cell hosts that must stay pinned. private async incumbentCellIsolatedForRoll( database: RelayDatabase, identity: AssignmentIdentity, existing: SqlRow, - // The caller's clock, not a second this.now(): one assign reasons about one - // instant, and the stamp's age decides whether a host moves. + // Use caller’s instant consistently; stamp age decides host movement. now: number, // Placement has already read the whole table; sticky has not, and a single // pinned cell does not justify a second fleet-wide read. @@ -8610,9 +8423,7 @@ export class RelayAssignmentStore { const pinned = pinnedAdmission ?? (await this.pinnedCellAdmission(database, cellId)) if (!rollIsolationIsCurrent(pinned, now)) return false if (integer(existing, 'migration_leases') > 0) return false - // A migration owns this assignment's epoch on both sides, and its durable - // row outlives the 15-minute lease that the counter above tracks, so the - // counter alone would re-place a host out from under a stalled migration. + // Durable migration owns both epochs beyond the 15-minute counter lease; counter-only checks could move a stalled migration’s host. const open = ( await database.query( `SELECT COUNT(*) AS open FROM relay_assignment_migrations @@ -8636,9 +8447,7 @@ export class RelayAssignmentStore { [cellId] ) )[0] - // Unknown, missing or unrecognised admission fails safe: the pin stays. This - // reader must not throw — it now sits on the sticky path every dial takes, - // and the rule it feeds is "move the host", so silence has to mean "don't". + // Unknown admission must retain the pin; never throw from the per-dial sticky reader that authorizes moves. if (!row || !isCellAdmissionState(row['admission_state'])) return undefined return { state: row['admission_state'], @@ -8984,9 +8793,7 @@ export class RelayAssignmentStore { activityId ] ) - // A released row's claim_activity_id is history, not a live claim: a control that - // reconnects under the same generation would otherwise leave its fresh reservation - // unclaimable, decaying through debt while the connection is already enforced. + // Released claim IDs are history; treating them as live leaves same-generation reconnect reservations unclaimable, decaying into debt while already enforced. const alreadyClaimed = await database.queryLocked( `SELECT reservation_id FROM relay_control_connection_reservations WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ? @@ -9121,9 +8928,7 @@ export class RelayAssignmentStore { await this.adjustActivityCount(database, identity, kind, -1, now, now) } - // Returns the request units this removal frees on `cellId`. The caller folds - // them into the one conditional cell-row write it makes just before COMMIT, - // so no statement here touches the fleet's most contended row. + // Return freed units for the caller’s final conditional cell write; do not touch that contended row here. private async removeSupersededSameCellControls( database: RelayDatabase, identity: AssignmentIdentity, @@ -9232,8 +9037,7 @@ export class RelayAssignmentStore { if (rows.length === 0) await this.refuseCellReservation(database, cellId) } - // The same write as the last statement of its transaction, committed in the same round - // trip so the shared cell row is not held while the reply crosses to a far cell. + // Commit with the final cell write in one round trip or its lock lasts across the remote reply. private async commitCellReservationAtomically( transaction: RelayDatabase, cellId: string, @@ -9309,9 +9113,7 @@ function activityLeaseById(rows: SqlRow[], activityId: string): SqlRow | undefin return rows.find((row) => text(row, 'activity_id') === activityId) } -// Why: mid-rehome a host legitimately holds the source control plus the -// target's pending control, so only the lease rows — never the counter a grant -// would overwrite — can say whether this cell is already reserved for it. +// Mid-rehome source and pending target controls coexist; only leases prove existing reservations, since grants overwrite counters. function holdsControlLease( rows: SqlRow[], cellId: string, @@ -9599,10 +9401,7 @@ function cellInventoryLockOptions(mode: CellInventoryLockMode): RelayLockOptions return { lockTimeoutMs: CELL_INVENTORY_LOCK_TIMEOUT_MS, measureHoldMs: true } } -// Background sweeps take the cell inventory NOWAIT so they never queue ahead of -// assignment traffic. A skipped candidate is re-derived from durable state on -// the next tick, so it is ordinary contention, not a sweep failure: one summary -// line per tick, never an error and never a quarantine. +// NOWAIT skips re-derive from durable state next tick; log one contention summary, never sweep failure or quarantine. function warnSweepCellInventoryBusy(sweep: string, skipped: number): void { if (skipped === 0) return console.warn( @@ -9793,9 +9592,7 @@ function cellUncleanSkip( } } -// Candidate skips are otherwise invisible: they neither latch the control off -// nor produce attempts, so an operator cannot tell "skipping" from "idle". -// Cell ids and counters only — never free-form error text. +// Log skips to distinguish them from idle; only cell IDs/counters, never free-form errors. function regionalRehomeCellSafetyIsClean( safety: SqlRow | undefined, diff --git a/cloud/apps/relay/src/assignment/assignment-cleanup-steps.ts b/cloud/apps/relay/src/assignment/assignment-cleanup-steps.ts index 9dad9e8593f..23b541f0af4 100644 --- a/cloud/apps/relay/src/assignment/assignment-cleanup-steps.ts +++ b/cloud/apps/relay/src/assignment/assignment-cleanup-steps.ts @@ -1,13 +1,6 @@ import { runRelayBackgroundOperation } from '../relay-background-operation.js' -// The twelve periodic assignment sweeps the director runs every 30s. Each step -// re-derives its state from the database and is idempotent, so they carry no -// intra-tick ordering dependency — which is what makes per-step isolation -// sound: one failing sweep costs one tick of itself, never the other eleven. -// (A single poisoned rehome row once silenced the whole chained form -// fleet-wide.) Sweep failures are logged, never fed into the rehome worker's -// dispatch-failure budget: a sweep exception is not a dispatch failure and -// must not durably disable regional rehoming. +// Each 30s sweep is idempotent and re-derives DB state, so isolate failures per step. Log sweep errors without spending dispatch-failure budget or disabling rehoming. export type AssignmentCleanupStore = { refreshRegionalRehomeLeases(): Promise completeReadyEvacuations(): Promise diff --git a/cloud/apps/relay/src/assignment/assignment-lease-shadow.ts b/cloud/apps/relay/src/assignment/assignment-lease-shadow.ts index 9e12c17bfc7..338fd938c6d 100644 --- a/cloud/apps/relay/src/assignment/assignment-lease-shadow.ts +++ b/cloud/apps/relay/src/assignment/assignment-lease-shadow.ts @@ -1,9 +1,7 @@ import { percentile } from '../relay-observability.js' import { classifyAssignmentLease, type AssignmentLeaseClass } from './assignment-lease.js' -// What `ticketCheck=enforce` would have said, beside the database answer that still decides. -// `agree`/`disagree` replace `valid`: a valid lease the database refused is the case step 5 -// has to explain before a lease alone may admit a host. +// Compare enforce’s hypothetical lease verdict with the authoritative DB answer; agree/disagree exposes valid leases the DB refuses before lease-only admission. export type AssignmentLeaseShadowClass = Exclude | 'agree' | 'disagree' export const ASSIGNMENT_LEASE_SHADOW_EVENT = 'orca_relay_assignment_lease_shadow' diff --git a/cloud/apps/relay/src/assignment/assignment-rejection-log-window.ts b/cloud/apps/relay/src/assignment/assignment-rejection-log-window.ts index 21ad4e72af9..7e432262b6c 100644 --- a/cloud/apps/relay/src/assignment/assignment-rejection-log-window.ts +++ b/cloud/apps/relay/src/assignment/assignment-rejection-log-window.ts @@ -1,10 +1,6 @@ type CancelWindow = () => void -// Admission rejections run at ~17/s per director instance, so the log summarizes -// instead of streaming: one line when a key's window opens, then a closing line -// carrying whatever that same window suppressed. The closing line is driven by the -// window's own timer, so a key that falls quiet still reports its final count -// instead of waiting for a rejection that may never arrive. +// At ~17 rejections/s, log window open and timer-driven close with suppressed count; quiet keys still report their final totals. const MAX_TRACKED_KEYS = 64 type RejectionWindow = { diff --git a/cloud/apps/relay/src/boot-database-open.ts b/cloud/apps/relay/src/boot-database-open.ts index cd7962dfda0..bc012dead26 100644 --- a/cloud/apps/relay/src/boot-database-open.ts +++ b/cloud/apps/relay/src/boot-database-open.ts @@ -6,20 +6,13 @@ import { } from './postgres-pool-pressure.js' import { postgresErrorCodeCategory } from './postgres-query-failure.js' -// Only a failure to reach Postgres at all. A retry here re-runs the schema -// apply, and applyPostgresSchema refuses to repeat a DDL lock timeout on -// purpose: relation locks are granted in queue order, so a repeat parks every -// writer behind the same statement again. 55P03, 57014 and 53300 therefore stay -// terminal at boot even though the request path calls them transient. +// Retry only unreachable PostgreSQL. Reapplying DDL after lock/statement/capacity errors (55P03/57014/53300) requeues ahead of writers, so these remain terminal at boot. function isBootDatabaseUnreachable(error: unknown): boolean { const code = postgresErrorCodeCategory(error) return isPostgresPoolConnectFailure(error) || code === '08001' || code === '08006' } -// A cell boots beside a cloud-sql-proxy that is itself still dialling, so the -// first pool acquire can outrun the 2s connect timeout that protects the -// request path. The window is longer than a proxy cold start and shorter than -// the restart loop it replaces. +// Proxy cold starts can exceed the request pool’s 2s connect timeout; retry longer than cold start but shorter than the restart loop. const BOOT_OPEN_RETRY = { attempts: 20, windowMs: 45_000, diff --git a/cloud/apps/relay/src/cell-admission-selector.ts b/cloud/apps/relay/src/cell-admission-selector.ts index dbbe5ab4505..553c28e9477 100644 --- a/cloud/apps/relay/src/cell-admission-selector.ts +++ b/cloud/apps/relay/src/cell-admission-selector.ts @@ -38,12 +38,7 @@ type ApplySelectorInput = { expectedGeneration: number expectedMembershipSha256?: string membership: CellAdmissionMembership - // Why a marker rather than reading 'migration-only' directly: that state is an - // admission class, not a drain signal (orca-relay-operations.md:229-233). - // Evacuation targets, Asia `--mode rollback`, a failed wave's re-isolate and - // newly registered cells all sit there durably while holding hosts. Only the - // same-cap isolate step names cells here; every write out of 'migration-only' - // clears the stamp, so a restore cannot leave one behind. + // migration-only also parks evacuation/Asia rollback/failed-wave/new cells (orca-relay-operations.md); only same-cap isolate writes stamps, cleared on every exit from that state. rollIsolatedCells?: string[] } @@ -201,10 +196,7 @@ export class RelayCellAdmissionSelector { const rollIsolated = new Set(input.rollIsolatedCells ?? []) for (const [state, cellIds] of membershipEntries(membership)) { for (const cellId of cellIds) { - // Three-way, in the same statement as the state so they cannot tear: - // stamp a named isolate (keeping an earlier stamp, so a failed wave's - // re-isolate stays marked), clear on any move out of 'migration-only', - // and leave an unnamed 'migration-only' cell exactly as it was. + // Update stamp atomically with state: named isolates preserve prior stamp, leaving migration-only clears it, unnamed migration-only preserves it. const marker = state === 'migration-only' ? rollIsolated.has(cellId) diff --git a/cloud/apps/relay/src/cell-admission-startup.ts b/cloud/apps/relay/src/cell-admission-startup.ts index 2a4bb24509a..57c75a70f52 100644 --- a/cloud/apps/relay/src/cell-admission-startup.ts +++ b/cloud/apps/relay/src/cell-admission-startup.ts @@ -28,9 +28,7 @@ export async function reconcileCellAdmissionAtStartup( // Admission is operator/director state. A new worker must not enable itself // before its distinct candidate has passed production preflight. if (config.role === 'cell') return - // Retries every transient failure, a lost reply included: the reconcile is upserts, a cap-only - // update and bound-change deletes, all of which converge on the same rows if a lost COMMIT had - // landed. + // Reconcile upserts/cap-only updates/bound-change deletes converge even after a lost COMMIT, so retry all transient failures. await retryTransientDatabaseStartup( async () => await assignments.reconcileCellsAtStartup(config.cells), STARTUP_RECONCILE_RETRY, diff --git a/cloud/apps/relay/src/cell-admit-effective-writer.ts b/cloud/apps/relay/src/cell-admit-effective-writer.ts index fdc579d779b..43cafe41824 100644 --- a/cloud/apps/relay/src/cell-admit-effective-writer.ts +++ b/cloud/apps/relay/src/cell-admit-effective-writer.ts @@ -1,9 +1,6 @@ import { DEAD_MAN_SLOT_HOLD_MS } from './cell-dead-man-trip-slot.js' -// The cell's raw admit mode in Postgres (relay_cell_admit_effective), so the database path can -// place on a tripped reserve cell even when no director's feed poll reaches it. Only the database -// path's PG-reserve cells need a row, so a cell writes only while its switch says reserve, plus -// one db write on the way out: with step 5 off, no cell writes anything. +// Publish raw mode so DB placement can use tripped reserve cells despite unreachable feeds. Write while switch says reserve plus one db exit write; step 5 off adds no writes. export const CELL_ADMIT_EFFECTIVE_REFRESH_MS = 15_000 const CELL_ADMIT_EFFECTIVE_FAILURE_LOG_MS = 60_000 // A held dead-man asks for a trip slot this often; each ask is one short locked transaction. diff --git a/cloud/apps/relay/src/cell-boot-without-database-postgres.test.ts b/cloud/apps/relay/src/cell-boot-without-database-postgres.test.ts index 03df1728aa4..5925f6705ba 100644 --- a/cloud/apps/relay/src/cell-boot-without-database-postgres.test.ts +++ b/cloud/apps/relay/src/cell-boot-without-database-postgres.test.ts @@ -9,8 +9,7 @@ import { createRelayServer } from './relay-server.js' const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip -// c25 exited after 45 s of boot retries when its database was unreachable, and the MIG -// recreated it into the same loop. A cell now opens a lazy pool and listens at once. +// Lazy pool/listen avoids the unreachable-DB boot/MIG restart loop that hit c25 after 45s retries. describePostgres('cell boot while PostgreSQL is unreachable', () => { const cleanups: Array<() => Promise> = [] diff --git a/cloud/apps/relay/src/cell-connection-cap.ts b/cloud/apps/relay/src/cell-connection-cap.ts index 4636b510470..dea85378d73 100644 --- a/cloud/apps/relay/src/cell-connection-cap.ts +++ b/cloud/apps/relay/src/cell-connection-cap.ts @@ -2,10 +2,7 @@ import { RELAY_ADMISSION_BUDGETS } from '@orca-cloud/relay-contract' import type { RelayCellConfig } from './config.js' import type { RelayDatabase, SqlRow } from './database.js' -// The one owner of relay_cell_connection_limits.hard_cap. The director's configured cap and the -// cap the cell last reported enforcing can differ while a cell is being resized; placement must -// never count seats a running process does not have, so every capacity reader takes the lower. -// A NULL report (a row no heartbeat has stamped yet) compares unknown and falls to the ELSE arm. +// During resize, placement uses min(configured, reported) to avoid phantom seats; unstamped NULL reports take configured cap via ELSE. export const EFFECTIVE_HARD_CAP_SQL = `(CASE WHEN limits.reported_hard_cap < limits.hard_cap THEN limits.reported_hard_cap ELSE limits.hard_cap END)` @@ -45,9 +42,7 @@ export function readCellConnectionCap(row: SqlRow): CellConnectionCap { } } -// A cell may run below the configured cap (it has not been rolled onto a raise yet), never above -// it, and its unobserved allowance is part of the placement ceiling so it must match exactly. -// Returns the cap to record, or undefined when the report does not fit. +// Reported cap may lag below configured, never exceed it; unobserved bound must match placement’s ceiling exactly. Return undefined for invalid reports. export function reportedHardCapWithinLimit( limit: SqlRow, report: { connectionHardCap?: number; connectionUnobservedBound?: number } @@ -105,11 +100,7 @@ export async function recordReportedHardCap( } } -// A cap-only change (same unobserved bound, either direction) changes nothing placement reads -// beyond min(): the stored report stands, or failing one (a row an older director wrote) the cap -// the cell was configured with until now is recorded as its report, so a raise waits for the cell -// to report it. Telemetry rows survive, so a deploy that interleaves revisions with different caps -// cannot wipe the fleet's headroom. A bound change keeps the old fail-closed reset. +// Cap-only changes preserve reports/telemetry; seed missing legacy reports from the old cap so raises wait for heartbeat. This tolerates interleaved revisions; bound changes retain fail-closed reset. export async function writeCellConnectionLimit( transaction: RelayDatabase, cell: Pick, diff --git a/cloud/apps/relay/src/cell-dead-man-trip-slot.ts b/cloud/apps/relay/src/cell-dead-man-trip-slot.ts index dff597070a3..d95979c6cad 100644 --- a/cloud/apps/relay/src/cell-dead-man-trip-slot.ts +++ b/cloud/apps/relay/src/cell-dead-man-trip-slot.ts @@ -1,21 +1,15 @@ import { commitWithFinalWrite, type RelayDatabase } from './database.js' -// A tripped cell refreshes its own row every 15 s unless its pool has waiters, so a slot outlives -// a busy stretch; a cell that died tripped gives its slot back after this. +// Outlive busy refresh gaps, but release the slot of a cell that died while tripped. export const DEAD_MAN_SLOT_HOLD_MS = 10 * 60_000 -// A reserve cell's dead-man may trip only while fewer than `slots` other reserve cells are -// tripped, so a director outage moves a bounded number of cells onto their full database pools. -// A slot is held while the cell's own row says db and its switch (relay_cell_admit_modes) still -// says reserve. Only the cell writes that row, so it stays true with every director gone; a -// restarted or re-armed cell writes reserve and frees it. +// Bound database-pool growth during director outages; cell-owned db rows hold slots until restart, re-arm, or expiry. export async function claimDeadManTripSlot( database: RelayDatabase, input: { cellId: string; cellIncarnation: string; slots: number; now: number } ): Promise { return await database.transaction(async (transaction) => { - // SQLite transactions already run one at a time. On PostgreSQL, concurrent claims would each - // count the others' uncommitted rows as free; a claim that finds the lock taken retries later. + // PostgreSQL claims must serialize to see occupied slots; SQLite already serializes transactions. if (transaction.dialect === 'postgres') { const [lock] = await transaction.query( `SELECT pg_try_advisory_xact_lock(hashtext('orca-relay'), hashtext('dead-man-trip-slots')) AS locked` diff --git a/cloud/apps/relay/src/cell-flags.ts b/cloud/apps/relay/src/cell-flags.ts index a860e67378c..8f4ace2b9e2 100644 --- a/cloud/apps/relay/src/cell-flags.ts +++ b/cloud/apps/relay/src/cell-flags.ts @@ -71,8 +71,7 @@ export type SupportedCellFlag = // Parsed, never offered: ticketCheck=enforce has no step that turns it on yet. const UNOFFERED_FLAG_VALUES: Record = { ticketCheck: ['enforce'] } -// What this image accepts, read from the schema itself: the flag tool refuses a key or value -// the cell would drop or void before it writes. +// Derive supported flags from schema so tooling refuses values the cell would drop or invalidate before writing. export function supportedCellFlags(): Record { const supported: Record = {} for (const [key, field] of Object.entries(CellFlagsSchema.shape)) { diff --git a/cloud/apps/relay/src/cell-inventory-hold-samples.test.ts b/cloud/apps/relay/src/cell-inventory-hold-samples.test.ts index 1e6fd86d2a9..0cba949e23a 100644 --- a/cloud/apps/relay/src/cell-inventory-hold-samples.test.ts +++ b/cloud/apps/relay/src/cell-inventory-hold-samples.test.ts @@ -60,9 +60,7 @@ describe('cell inventory hold samples', () => { expect(samples.consumeCounts()).toEqual(emptyCellInventoryHoldCounts()) }) - // Why: this is the case the hold metrics alone cannot see. A NOWAIT grab that - // fails has no duration, so a retry storm used to leave every hold field at - // zero while the lock was saturated. + // Failed NOWAIT has no duration; saturated retry storms must not leave all hold metrics looking healthy. it('counts failed acquisitions in a window that recorded no holds', () => { const samples = new CellInventoryHoldSamples() for (let attempt = 0; attempt < 65; attempt++) samples.recordUnavailable() diff --git a/cloud/apps/relay/src/cell-inventory-hold-samples.ts b/cloud/apps/relay/src/cell-inventory-hold-samples.ts index f8fed1bb349..a43764fb7c1 100644 --- a/cloud/apps/relay/src/cell-inventory-hold-samples.ts +++ b/cloud/apps/relay/src/cell-inventory-hold-samples.ts @@ -1,9 +1,4 @@ -// Why: the cell inventory lock is held to COMMIT, and the assignment path runs -// many statements after taking it. Tuning the request-path wait bound needs the -// hold distribution, and no runtime metric carried it before this change. -// Which lock a hold sample came from. Every site feeds the same max, so one -// alert on cellInventoryHoldMsMax covers them all; the label names the holder. -// 'isolated-replacement' is a drain return's regional target rows. +// Measure COMMIT-held lock durations to tune request waits. All sites feed shared max; labels identify holders, including drain-return regional rows as isolated-replacement. export type CellLockHoldSite = 'inventory' | 'rehome-target-row' | 'isolated-replacement' export type CellInventoryHoldCounts = { @@ -20,12 +15,7 @@ export type CellInventoryHoldCounts = { rehomeTargetRowHoldMsP99: number isolatedReplacementHoldMsP99: number isolatedReplacementHolds: number - // Why: a failed acquisition produces no hold sample, so the hold fields alone - // read healthy while the lock is saturated. Split by wait policy, not by - // caller: fail-fast covers background sweeps that step aside by design AND - // request-path first attempts that retry, so it reads as contention pressure, - // not user-visible failure. An expired bounded wait has already spent its - // budget, so that lane is the one that tracks stalls. + // Failed acquisitions have no duration. Fail-fast counts sweep skips and initial request retries as pressure; expired bounded waits measure stalls separately. cellInventoryLockUnavailable: number cellInventoryLockTimeouts: number } diff --git a/cloud/apps/relay/src/cell-inventory-lock-contention.test.ts b/cloud/apps/relay/src/cell-inventory-lock-contention.test.ts index 1bf3153f3e4..5ac0b50797c 100644 --- a/cloud/apps/relay/src/cell-inventory-lock-contention.test.ts +++ b/cloud/apps/relay/src/cell-inventory-lock-contention.test.ts @@ -173,10 +173,7 @@ describe('bounded cell-inventory lock wait', () => { await database.close() }) - // Why: testing the helper alone would pass with the store still queueing for - // the pool's one-second default. - // Why: testing the helper alone would pass with the request path still queueing - // for the pool's full second. + // Helper-only tests miss request paths still waiting for the full one-second pool default. it('never lets a request path take the unbounded wait', async () => { const database = await openInMemoryRelayDatabase() const probe = new InventoryLockProbe(database) @@ -292,9 +289,7 @@ describe('bounded cell-inventory lock wait', () => { await database.close() }) - // Why: a bounded request-path wait raises the same 55P03 without NOWAIT. Folding - // it into the deferral counter would hide user-visible stalls among by-design - // sweep skips, which outnumber them by roughly an order of magnitude. + // Separate bounded 55P03 timeouts from intentional NOWAIT skips or frequent sweep deferrals hide user-visible stalls. it('counts an expired bounded wait apart from a NOWAIT deferral', async () => { const database = await openFakePostgres() fakes.query.mockImplementation(async (sql: string) => { @@ -316,9 +311,7 @@ describe('bounded cell-inventory lock wait', () => { ).rejects.toThrow() const counts = consumeRelayCellInventoryHold(database) - // One per attempt, not per request: 55P03 is retryable, so an exhausted - // request contributes POSTGRES_TRANSACTION_ATTEMPTS timeouts. Reading the - // metric as affected-requests would overstate it threefold. + // Count attempts, not requests: three retryable 55P03 timeouts would overstate affected requests threefold. expect(counts.cellInventoryLockTimeouts).toBe(3) expect(counts.cellInventoryLockUnavailable).toBe(0) await database.close() diff --git a/cloud/apps/relay/src/cell-inventory-per-cell-locking-postgres.test.ts b/cloud/apps/relay/src/cell-inventory-per-cell-locking-postgres.test.ts index a1521404fed..43d3616ae95 100644 --- a/cloud/apps/relay/src/cell-inventory-per-cell-locking-postgres.test.ts +++ b/cloud/apps/relay/src/cell-inventory-per-cell-locking-postgres.test.ts @@ -5,10 +5,7 @@ import { openRelayDatabase, type RelayDatabase } from './database.js' const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip -// Sorted ascending, and the host is pinned to the LAST id on purpose: the -// fleet-wide lock is one ordered scan, so it holds every earlier row while it -// waits on the pinned one. Pinning to the first id would make the two locking -// models indistinguishable. +// Pin to last ascending cell: a fleet scan holds earlier rows while waiting, whereas per-cell locking leaves them free; first-cell pins cannot distinguish these. const cells = ['a', 'b', 'c'].map((suffix) => ({ id: `percell-postgres-${suffix}`, url: `https://percell-postgres-${suffix}.example.com`, @@ -102,9 +99,7 @@ describePostgres('PostgreSQL per-cell inventory locking', () => { return false } - // Why: a sticky refresh whose first NOWAIT probe loses retries by taking a - // cell row before the assignment row. That retry used to take the whole - // inventory, so one busy cell stalled every other cell's reconnects. + // Sticky NOWAIT retry must lock only the pinned cell first; fleet-wide retry previously stalled every cell’s reconnects. it('waits only on the pinned cell row while refreshing a sticky assignment', async () => { await removeTestRows(databases[0]!) const store = new RelayAssignmentStore(databases[0]!, () => 100) diff --git a/cloud/apps/relay/src/cell-poll-reachability.ts b/cloud/apps/relay/src/cell-poll-reachability.ts index c3ec8021bb7..4c748d42514 100644 --- a/cloud/apps/relay/src/cell-poll-reachability.ts +++ b/cloud/apps/relay/src/cell-poll-reachability.ts @@ -1,6 +1,4 @@ -// Whether this director's own seat-feed polls reach a cell: one half of the evidence that lets a -// host that lost its control leave a dead reserve cell without an operator fence. The other half -// is the cell's stale heartbeat, which travels a different path (cell -> director -> Postgres). +// Unanswered director feed polls plus stale heartbeat (cell→director→Postgres) prove reserve-cell death, allowing only host-requested departure without an operator fence. // The failing streak must span this long: the heartbeat TTL, so neither signal outruns the other. export const CELL_POLL_DEAD_STREAK_MS = 45_000 @@ -57,8 +55,7 @@ export class CellPollReachability { for (const cellId of this.streaks.keys()) if (!cellIds.has(cellId)) this.streaks.delete(cellId) } - // Any current streak, however young: a staggered partition must not let its first cell cross 45 s - // while its neighbours are still a few seconds behind and read as healthy. + // Count even young failure streaks or staggered partition starts can make the first dead verdict outrun its neighbors. private failing(cellId: string, now: number): boolean { const streak = this.streaks.get(cellId) return streak !== undefined && now - streak.lastFailedAt <= CELL_POLL_DEAD_MAX_GAP_MS @@ -74,8 +71,7 @@ export class CellPollReachability { ) } - // Null unless this cell is unreachable. Every director runs in one region, so a partition from - // another region fails every cell there for all of them at once: half a region is never a cell. + // Regional directors share cross-region partitions; half a region failing cannot prove individual cell death. verdict(cellId: string, polled: readonly PolledCell[], now: number): DeadCellVerdict | null { if (!this.unreachable(cellId, now)) return null const region = polled.find((cell) => cell.cellId === cellId)?.region diff --git a/cloud/apps/relay/src/cell-reserve-admission.blackbox.test.ts b/cloud/apps/relay/src/cell-reserve-admission.blackbox.test.ts index 709f7e21987..8e82caa5acd 100644 --- a/cloud/apps/relay/src/cell-reserve-admission.blackbox.test.ts +++ b/cloud/apps/relay/src/cell-reserve-admission.blackbox.test.ts @@ -16,8 +16,7 @@ import type { AppliedControlFlags } from './relay-control-flag-channel.js' import { createRelayServer, HOST_HELLO_SHED_OLDEST_WAIT_MS } from './relay-server.js' import { RELAY_WEBSOCKET_FORCE_CLOSE_MS } from './relay-websocket-close.js' -// A step-5 cell whose database pool is wedged: every query waits forever and the hello shed -// is armed. Booked and recently seated hosts must still connect; everything else must not. +// Wedged database and armed shedding must still admit booked/recently seated hosts, while refusing strangers. async function unusedPort(): Promise { const server = createNetServer() @@ -448,10 +447,7 @@ describe('a reserve-mode cell admits from memory with its database wedged', () = expect(warned('orca_relay_cell_seat_log_drift')).toEqual([]) }, 10_000) - // A drained session leaves the map at once, but its socket closes only after its release (up - // to 3 s) and the peer's close reply (up to the 1 s force-close). A desktop of the same - // identity dialling back in that window restarted at generation 1, so the old socket's late - // leave, at generation 1 too, removed the live seat from the feed. + // Drained sockets can close seconds after map removal; reusing their generation let late leaves erase reconnected live seats. it('keeps a session that reconnected while its drained predecessor was still closing', async () => { const cell = await startCell({ admitMode: 'reserve' }) const identity = await cell.host() @@ -478,8 +474,7 @@ describe('a reserve-mode cell admits from memory with its database wedged', () = expect(seatEpochs(cell)).toEqual([10]) }, 10_000) - // The same identity connecting afresh at each turn, as desktops sharing one identity do: each - // turn replaces the session and its seat together, so no sweep finds anything to repair. + // Repeated fresh joins for one identity replace session and seat together, leaving no drift to repair. it('stays on the live session when the same identity connects afresh each turn', async () => { const cell = await startCell({ admitMode: 'reserve' }) const identity = await cell.host() diff --git a/cloud/apps/relay/src/cell-reserve-book.ts b/cloud/apps/relay/src/cell-reserve-book.ts index 089f4727f89..7d3252c7318 100644 --- a/cloud/apps/relay/src/cell-reserve-book.ts +++ b/cloud/apps/relay/src/cell-reserve-book.ts @@ -4,8 +4,7 @@ import { type ReserveOutcome } from './cell-reserve-contract.js' -// The cell's half of step 5: it alone says yes or no to each booking. One event loop makes -// "check cap and budget, then count the booking" atomic with no lock. +// Single event loop makes booking cap/budget admission and accounting atomic without locks. // A connection unit held for a booked host until its hello consumes it or the TTL ends. export type BookingUnit = { release(): void } @@ -130,8 +129,7 @@ export class CellReserveBook { return booking !== undefined && booking.expiresAt > this.now() } - // The booked host's upgrade takes over its unit, so the host counts once until its hello. - // The upgrade carries no epoch; the hello's take() still requires the booked one. + // Upgrade takes the booking’s unit to count once; epoch-less upgrade still needs exact-epoch take at hello. handOff(userId: string, relayHostId: string): void { const booking = this.bookings.get(hostKey(userId, relayHostId)) if (!booking || booking.handedOff || booking.expiresAt <= this.now()) return diff --git a/cloud/apps/relay/src/cell-reserve-client.ts b/cloud/apps/relay/src/cell-reserve-client.ts index 4a52d5e34af..a17013ac055 100644 --- a/cloud/apps/relay/src/cell-reserve-client.ts +++ b/cloud/apps/relay/src/cell-reserve-client.ts @@ -9,9 +9,7 @@ import { } from './cell-reserve-contract.js' import type { ReserveAttempt } from './reserve-placement.js' -// The director's calls to cells. Bookings for one cell batch for up to 5 ms or 64 items; -// a cell that does not answer within the timeout is unverifiable, and every item in the -// batch reads `unreachable` (one of the request's three tries). +// Batch bookings per cell for 5ms/64 items; unreadable replies mark the whole batch unreachable for one of the request’s three tries. export const CELL_RESERVE_FLUSH_MS = 5 export const CELL_RESERVE_TIMEOUT_MS = 1_000 diff --git a/cloud/apps/relay/src/cell-reserve-contract.ts b/cloud/apps/relay/src/cell-reserve-contract.ts index 5a66381996f..95c7ff29310 100644 --- a/cloud/apps/relay/src/cell-reserve-contract.ts +++ b/cloud/apps/relay/src/cell-reserve-contract.ts @@ -1,8 +1,6 @@ import { z } from 'zod' -// Step 5 wire between directors and cells (v1). Directors book a seat on a cell before -// answering a desktop; the cell admits or refuses each booking itself. Readers ignore -// unknown fields, so either side may add optional ones. +// v1 director/cell booking wire: cell decides each seat before desktop assignment; ignore unknown fields for independent upgrades. export const CELL_RESERVE_PATH = '/internal/reserve' export const CELL_DEMOTE_PATH = '/internal/demote' diff --git a/cloud/apps/relay/src/cell-reserve-dead-man.ts b/cloud/apps/relay/src/cell-reserve-dead-man.ts index f564449a817..c66110b46a3 100644 --- a/cloud/apps/relay/src/cell-reserve-dead-man.ts +++ b/cloud/apps/relay/src/cell-reserve-dead-man.ts @@ -2,14 +2,11 @@ import { createHash } from 'node:crypto' import type { CellFlags } from './cell-flags.js' import type { AppliedControlFlags } from './relay-control-flag-channel.js' -// A reserve-mode cell keeps its hosts in memory only while a placing director is there to -// book and supersede them. With none for this long (a director rolled back, or placement -// turned off) it flips itself back to db and re-registers, as a switch flip would. +// Without placing-director contact, memory-only reserve ownership cannot supersede hosts; flip back to db and re-register as a switch flip would. export const RESERVE_DEAD_MAN_MS = 60_000 const RESERVE_DEAD_MAN_SPREAD_MS = 60_000 -// 60-120 s, fixed per cell: when every director goes at once, the cells trip (and start -// re-registering against the database) spread over a minute, not together. +// Stable 60–120s per-cell windows stagger fleet-wide fallback DB re-registration over one minute. export function reserveDeadManWindowMs(cellId: string): number { const digest = createHash('sha256').update(cellId).digest() return RESERVE_DEAD_MAN_MS + (digest.readUInt32BE(0) % (RESERVE_DEAD_MAN_SPREAD_MS + 1)) @@ -40,8 +37,7 @@ export class ReserveDeadMan { this.permittedGeneration = null } - // The admitMode the cell acts on. Latched per switch-file generation: once tripped, only a - // new write brings reserve back, so a director that returns cannot flap the cell. + // Latch fallback per switch generation; returning directors cannot flap mode until a new reserve write. mode(applied: AppliedControlFlags): 'db' | 'reserve' { if (applied.flags.admitMode !== 'reserve') { this.reserveGeneration = null diff --git a/cloud/apps/relay/src/cell-reserve-flip-back-postgres.test.ts b/cloud/apps/relay/src/cell-reserve-flip-back-postgres.test.ts index 833c366b126..3d9a7849f4d 100644 --- a/cloud/apps/relay/src/cell-reserve-flip-back-postgres.test.ts +++ b/cloud/apps/relay/src/cell-reserve-flip-back-postgres.test.ts @@ -7,9 +7,7 @@ import { openRelayDatabase, type RelayDatabase } from './database.js' import { openDelayedPostgresDatabase, type StatementDelay } from './test-fixtures/delayed-postgres-database.js' import { RESERVE_MODE_CELL_ID, startReserveModeCell } from './test-fixtures/reserve-mode-cell.js' -// E-mix on PostgreSQL: a reserve-mode cell flipped back to db leases every control it admitted -// from memory, disconnecting nobody, with the pool mostly left to DB-path work and one -// relay_cells write per tick, even with a cross-region round trip on every statement. +// PostgreSQL flip-back keeps controls connected, reserves most pool capacity for DB work and folds cell deltas per tick even across cross-region round trips (E-mix). const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip diff --git a/cloud/apps/relay/src/cell-reserve-rollback.blackbox.test.ts b/cloud/apps/relay/src/cell-reserve-rollback.blackbox.test.ts index 71500d48176..5caadd2ecd5 100644 --- a/cloud/apps/relay/src/cell-reserve-rollback.blackbox.test.ts +++ b/cloud/apps/relay/src/cell-reserve-rollback.blackbox.test.ts @@ -229,8 +229,7 @@ describe('flipping a reserve-mode cell back to the database', () => { expect(reply.socket.readyState).toBe(WebSocket.OPEN) }, 30_000) - // Step 5 off stays write-free: a db switch (read, generation > 0) writes no row, nor does a - // booted cell before its first flag read. Leaving reserve writes db once, then nothing. + // Step 5 off adds no writes before/after flag reads; leaving reserve owes exactly one db write. it('writes an admit-effective row only while its switch says reserve, plus one db write on the way out', async () => { let now = Date.now() const booting = await cell(() => now, { flagsUnread: true }) @@ -275,8 +274,7 @@ describe('flipping a reserve-mode cell back to the database', () => { .mock.calls.map(([line]) => String(line)) .filter((line) => line.includes('orca_relay_cell_reserve_dead_man_tripped')) expect(tripped.map((line) => JSON.parse(line).cellId)).toEqual([reserve.config.cellId]) - // The cell's own row says db at once, while Postgres still records reserve: the database - // path places on it by this row, whether or not any director's feed poll gets through. + // Publish raw db mode immediately despite PG’s reserve stamp so placement can recover without a working feed poll. await until(async () => (await reserve.admitEffectiveRow())?.mode === 'db') expect((await reserve.admitEffectiveRow())?.cell_incarnation).toBe(reserve.relay.cellIncarnation) await until(async () => (await reserve.controlLeases()).length === 2) diff --git a/cloud/apps/relay/src/cell-reserve-routes.ts b/cloud/apps/relay/src/cell-reserve-routes.ts index 46e103c1a95..2229f44fc4d 100644 --- a/cloud/apps/relay/src/cell-reserve-routes.ts +++ b/cloud/apps/relay/src/cell-reserve-routes.ts @@ -13,8 +13,7 @@ import { readBearer } from './relay-token-verifier.js' const MAX_BODY_BYTES = 16 * 1024 -// The directors' rehome credential, as the seat feed uses. Unlike the feed these write, but a -// director can already place any host through the database, so they grant nothing new. +// Reuse director rehome credentials for writes; directors can already place any host via DB, so these grant no new authority. export function registerCellReserveRoutes( app: Hono, config: RelayConfig, diff --git a/cloud/apps/relay/src/cell-seat-feed-route.ts b/cloud/apps/relay/src/cell-seat-feed-route.ts index 59cc6f51365..4991b756375 100644 --- a/cloud/apps/relay/src/cell-seat-feed-route.ts +++ b/cloud/apps/relay/src/cell-seat-feed-route.ts @@ -7,8 +7,7 @@ import type { AppliedControlFlags } from './relay-control-flag-channel.js' import type { RelayRuntimeCounts } from './relay-observability.js' import { readBearer } from './relay-token-verifier.js' -// Read-only, so it reuses the directors' rehome credential and grants it nothing new. Only -// cells that already hold the rehome pair serve it: giving a cell the pair turns rehome on. +// Reuse read-only rehome credentials only on cells already holding the pair; distributing it further would enable rehome. export function registerCellSeatFeedRoute( app: Hono, config: RelayConfig, diff --git a/cloud/apps/relay/src/cell-seat-log.ts b/cloud/apps/relay/src/cell-seat-log.ts index 0c3c891f07b..fdf326a7fc5 100644 --- a/cloud/apps/relay/src/cell-seat-log.ts +++ b/cloud/apps/relay/src/cell-seat-log.ts @@ -1,13 +1,4 @@ -// Seat changes on this cell, read by directors through `GET /v1/admin/cell-seats`. -// Wire contract (v1); readers must ignore unknown fields: -// since=: returns `changes` after seq, or `full` when the cursor is -// absent, from another incarnation, or older than the ring. `seq` in the reply is the -// cursor for the next poll; `more` says a page was cut at CELL_SEAT_FEED_PAGE_MAX. -// Apply a change only when its generation is >= the seat's: a superseded generation's -// socket can close after its successor joined. `full` is this log's own seats under the -// same rules, so a reader that applies every change holds exactly `counts.seats`. -// Step 5 adds `reserve` (a booking, never a seat) and `reservedBy` on the join it admitted; -// a full snapshot also carries `recentlyLeft`, this cell's leavers of the last 10 minutes. +// v1 feed: ignore unknown fields. since=: returns changes or full for absent/stale/wrong-incarnation cursors; reply seq continues and more signals paging. Apply generations >= current so late superseded closes cannot erase successors. Full seats obey the same rules/counts. Reserve is a booking, reservedBy identifies join’s booker, and recentlyLeft spans ten minutes. export type CellSeatState = 'active' | 'drain-only' diff --git a/cloud/apps/relay/src/cloud-run-instance-zone.ts b/cloud/apps/relay/src/cloud-run-instance-zone.ts index 17798ce519a..6cf0b6a73ff 100644 --- a/cloud/apps/relay/src/cloud-run-instance-zone.ts +++ b/cloud/apps/relay/src/cloud-run-instance-zone.ts @@ -1,8 +1,7 @@ const INSTANCE_ZONE_URL = 'http://metadata.google.internal/computeMetadata/v1/instance/zone' const REQUEST_TIMEOUT_MS = 3_000 -// The metadata server answers `projects//zones/`. Undefined on any failure: -// the zone only labels log lines, so it must never hold up or fail a boot. +// Zone only labels logs; metadata lookup failure must not fail boot. export async function readCloudRunInstanceZone( fetchImpl: typeof fetch = fetch ): Promise { diff --git a/cloud/apps/relay/src/config.ts b/cloud/apps/relay/src/config.ts index 6095cec26b5..00d4fd5999b 100644 --- a/cloud/apps/relay/src/config.ts +++ b/cloud/apps/relay/src/config.ts @@ -36,8 +36,7 @@ const OptionalServiceAccountSchema = z.preprocess( z.string().email().optional() ) -// 0 disables the window and restores the fail-on-first-error readiness answer. An unset variable -// arrives as '' from Cloud Run, which z.coerce would read as 0 rather than as the default. +// Cloud Run unset values arrive as empty strings, which coerce to 0 (fail-first readiness) instead of the grace default. const readinessGraceSchema = (defaultMs: number) => z.preprocess( (value) => (value === '' ? undefined : value), diff --git a/cloud/apps/relay/src/control-accept-cell-row-lock-postgres.test.ts b/cloud/apps/relay/src/control-accept-cell-row-lock-postgres.test.ts index fea3b3ebc09..8cd01e48655 100644 --- a/cloud/apps/relay/src/control-accept-cell-row-lock-postgres.test.ts +++ b/cloud/apps/relay/src/control-accept-cell-row-lock-postgres.test.ts @@ -19,10 +19,7 @@ const userId = 'accept-lock-postgres-user' const first = { userId, relayHostId: 'acceptlockhost01' } const second = { userId, relayHostId: 'acceptlockhost02' } -// Why: the accept path now enforces the capacity ceiling inside its single -// conditional cell-row write instead of behind a SELECT ... FOR UPDATE it held -// for the rest of the transaction. Two accepts reaching for the same last slot -// are what would expose a lost update if that check were no longer atomic. +// Race accepts for the last slot: the final conditional cell UPDATE must enforce capacity atomically without an earlier held SELECT lock. describePostgres('PostgreSQL control accept without a held cell row', () => { const databases: RelayDatabase[] = [] @@ -151,10 +148,7 @@ describePostgres('PostgreSQL control accept without a held cell row', () => { }) await heldPromise - // Retiring generation 1 and installing generation 2 leaves the cell's - // reservation where it was, so the accept has no reason to wait on the row - // at all. Reading it up front is what used to make it wait, and then fail - // at the request-path lock bound. + // Same-cell generation replacement changes no reservation; reading the shared row would needlessly wait and hit the request bound. try { await expect( store.activateControl(first, { @@ -180,8 +174,7 @@ describePostgres('PostgreSQL control accept without a held cell row', () => { expect(Number(cells[0]!.reserved_requests)).toBe(Number(units[0]!.units)) }, 20_000) - // The fused commit is optional on RelayDatabase, so a wrapper that stopped forwarding it - // would silently fall back to a separate COMMIT. Drive the store the server builds. + // Drive the production store: wrappers can omit optional fused commit and silently restore a separate COMMIT round trip. it('commits every counter write in one message through the production store', async () => { await removeTestRows(databases[0]!) const store = new RelayAssignmentStore(databases[0]!, () => 100) diff --git a/cloud/apps/relay/src/control-rebind-inventory-lock-postgres.test.ts b/cloud/apps/relay/src/control-rebind-inventory-lock-postgres.test.ts index 7185b9730a8..b05b0f7ff25 100644 --- a/cloud/apps/relay/src/control-rebind-inventory-lock-postgres.test.ts +++ b/cloud/apps/relay/src/control-rebind-inventory-lock-postgres.test.ts @@ -50,11 +50,7 @@ function heartbeat(cell: (typeof cells)[number]) { } } -// Why: every desktop control rebind used to take the fleet-wide relay_cells -// FOR UPDATE lock, so a rebind on one cell queued behind whatever held any -// other cell's row, until COMMIT (55P03 at the request bound). A rebind only -// touches its own cell row, so it must proceed while another cell's row is -// held elsewhere. +// Same-cell rebind must proceed while another cell row is held; fleet-wide locking formerly waited to COMMIT and hit request 55P03. describePostgres('PostgreSQL control rebind under a held cell row', () => { const databases: RelayDatabase[] = [] @@ -191,10 +187,7 @@ describePostgres('PostgreSQL control rebind under a held cell row', () => { } }, 15_000) - // Why: a phone's activity id is client-chosen and can follow the host across - // a migration, so acquireActivity may touch two cell rows. Moving from the - // higher cell to the lower one is where an unordered lock cycles with - // placement's ascending inventory lock (reproduced live before this fix). + // Client activity IDs can migrate across cells; high→low moves expose lock cycles against ascending placement order. it('moves an activity from a higher cell to a lower one in cell_id order', async () => { await removeTestRows(databases[0]!) const [cellA, cellB, cellC] = cells as [(typeof cells)[0], (typeof cells)[0], (typeof cells)[0]] @@ -219,9 +212,7 @@ describePostgres('PostgreSQL control rebind under a held cell row', () => { applicationName: 'orca-relay/cloud3-rebind-move' }) const moveStore = new RelayAssignmentStore(movingDatabase, () => 100) - // Hold B elsewhere. An ordered move locks B first and queues here holding - // nothing else. Locking C first (the old lease's row, as an unordered move - // does) or the whole inventory (which takes A) shows up as a held row. + // Hold B: ordered B→C move waits with no other lock; C-first or fleet-wide locking would hold C or A. let releaseRow!: () => void const rowReleased = new Promise((resolve) => { releaseRow = resolve diff --git a/cloud/apps/relay/src/control-renewal-batch.ts b/cloud/apps/relay/src/control-renewal-batch.ts index 984333b2ca6..8f3547e866d 100644 --- a/cloud/apps/relay/src/control-renewal-batch.ts +++ b/cloud/apps/relay/src/control-renewal-batch.ts @@ -4,15 +4,9 @@ import type { ControlRenewalRequest } from './control-renewal-statement.js' -// One flush per second turns the fleet's control-lease write rate into a -// function of the cell count rather than the host count: a cell's ~10 due -// renewals per second become one write transaction instead of ten. Well inside -// the 105s lease runway, so a host that misses a window is never at risk. +// One flush per cell/sec replaces ~10 per-host transactions; missed windows remain within the 105s lease runway. export const CONTROL_RENEWAL_BATCH_INTERVAL_MS = 1_000 -// Ceiling on the parameter arrays. Row locks live until the statement commits, -// so this is what bounds how long one flush holds them: measured at 11.5ms for -// 200 rows against a 20,000-row table, and 9.4ms with a host wedged in a -// per-host transaction. +// Bound row locks/parameters: 200 rows measured ~11.5ms on 20k rows and ~9.4ms with one host wedged. export const CONTROL_RENEWAL_BATCH_MAX_ROWS = 200 // A flush slower than this is the only latency worth a line; the metrics event // carries the distribution. @@ -36,10 +30,7 @@ function pendingKey(request: ControlRenewalRequest): string { return [request.identity.userId, request.identity.relayHostId].join('\u0000') } -// Collects the control-lease renewals a cell owes and spends one statement on -// them. Each caller still gets the single-renewal contract: the promise resolves -// on `renewed` and rejects with the outcome as its message otherwise, so callers -// keep their per-session error routing unchanged. +// Preserve per-caller renewal contract: resolve renewed, otherwise reject with its outcome for unchanged session error routing. export class ControlRenewalBatch { private pending = new Map() // Renewals a host cannot contribute to the flush being built; they open the diff --git a/cloud/apps/relay/src/control-renewal-postgres.test.ts b/cloud/apps/relay/src/control-renewal-postgres.test.ts index 6ae8a21c287..61cd45dea79 100644 --- a/cloud/apps/relay/src/control-renewal-postgres.test.ts +++ b/cloud/apps/relay/src/control-renewal-postgres.test.ts @@ -449,9 +449,7 @@ describePostgres('PostgreSQL control renewal', () => { it('renews both of one host\u2019s control leases in a single batch', async () => { const store = new RelayAssignmentStore(database, () => now) const identity = identities[13]! - // Two live control leases on one host. Written directly because - // activateControl retires the prior generation, and what is under test is the - // statement's row-wise behaviour, not how the second lease came to exist. + // Seed duplicate live controls directly; activateControl retires predecessors, but this case tests row-wise batch behavior. await database.query( `INSERT INTO relay_assignment_activity_leases (user_id, relay_host_id, activity_id, activity_kind, cell_id, diff --git a/cloud/apps/relay/src/control-renewal-statement.ts b/cloud/apps/relay/src/control-renewal-statement.ts index 7ccd5398e7d..b162d60610e 100644 --- a/cloud/apps/relay/src/control-renewal-statement.ts +++ b/cloud/apps/relay/src/control-renewal-statement.ts @@ -7,9 +7,7 @@ export type ControlRenewalOutcome = | 'activity_cell_not_authoritative' | 'control_activity_not_found' | 'control_activity_moved' - // The host's assignment row was already locked by one of the per-host - // transactional paths. Retryable, and never a reason to close a control: the - // next tick is 15s away and the lease has 105s on it. + // Contended assignment is retryable, never grounds to close control: next tick is 15s away with a 105s lease. | 'assignment_lock_unavailable' // Decided per row before the statement runs, so one malformed request cannot // cost the rest of the batch its renewal. @@ -64,14 +62,7 @@ export function orderedControlRenewalRows { const expiry = await cleanupPlan('UPDATE relay_invites', 0) const reservation = await cleanupPlan('UPDATE relay_invites', 1) - // Which of the two partial indexes serves the reservation pass is the planner's call: both - // predicates hold only live invites, so either one reads a handful of rows. The invariant is - // that neither pass reads the whole table any more. + // Either partial index may win; both restrict to live invites, and neither sweep may scan the whole table. for (const sweep of [expiry, reservation]) { expect(sweep).not.toContain('Seq Scan on relay_invites') expect(sweep).toMatch(/using relay_invites_sweep_(expiry|reservation)/) @@ -140,10 +135,7 @@ describePostgres('credential cleanup against PostgreSQL', () => { }) it('plans the basis sweep off the composite index rather than the 1.5 GB heap', async () => { - // The shape that made this the most expensive statement in the sweep: 20,000 settled bases to - // 50 live ones. A partial index on active = 1 looks like the answer to that ratio and is not: - // a basis is inserted active and flipped to 0, so it accumulates the same dead entries, and - // the planner picks the composite index anyway. See the schema comment beside it. + // Active→inactive basis flips leave dead partial-index entries; with 20k settled/50 live the planner still picks the composite index (see schema). await database.query( `INSERT INTO relay_connection_bases (basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation, @@ -169,11 +161,7 @@ describePostgres('credential cleanup against PostgreSQL', () => { }) it('plans the drained basis reaper off the composite index, not the heap', async () => { - // Why the composite index stays for now: it is the only one covering active = 0, and the case - // that needs it is the steady state, where every row is inside retention and the reaper must - // learn there is nothing to do. While the backlog drains the planner rightly prefers a bounded - // sequential scan, because it finds its 5,000 rows and stops; measured at 200k rows, the - // drained batch costs 5 buffers with this index and 1,274 without it. + // Keep composite index for drained active=0 retention checks (5 vs 1,274 buffers at 200k rows); backlog batches correctly favor bounded seq scans. await database.query( `INSERT INTO relay_connection_bases (basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation, diff --git a/cloud/apps/relay/src/credential-store-cleanup.test.ts b/cloud/apps/relay/src/credential-store-cleanup.test.ts index 0a750beb8d8..ef5548c2613 100644 --- a/cloud/apps/relay/src/credential-store-cleanup.test.ts +++ b/cloud/apps/relay/src/credential-store-cleanup.test.ts @@ -127,8 +127,7 @@ describe('credential cleanup invite reaper', () => { }) it('never deletes an invite that a reader could still consume, however old', async () => { - // Retention is measured on updated_at, and a long-lived available invite has an old one. The - // state filter is what keeps the reaper from deleting a credential still in use. + // Old updated_at does not settle a live invite; state guard must prevent reaping it. const database = await openInMemoryRelayDatabase() const store = new RelayCredentialStore(database, () => NOW) const ancient = NOW - 400 * DAY_MS @@ -168,8 +167,7 @@ describe('credential cleanup invite reaper', () => { }) it('still expires credentials the sweep owns, and only those past their deadline', async () => { - // The reaper runs after the sweep in the same call, so this pins that adding it did not - // displace any of the five state transitions the sweep is there for. + // Reaper follows sweep; ensure all five prior transitions still run. const database = await openInMemoryRelayDatabase() const store = new RelayCredentialStore(database, () => NOW) await insertInvite(database, { @@ -245,16 +243,12 @@ describe('credential cleanup invite reaper', () => { }) it('reaps connection bases whose deadline passed over a day ago, and nothing else', async () => { - // Retention is measured on deadline, and both readers of a basis require deadline >= now, so a - // deadline a day in the past is already unusable however the active flag reads. The active = 0 - // clause is what keeps the batch an index range, not what makes the row safe to delete. + // Passed deadlines make bases unusable regardless of active flag; active=0 provides an index range, not deletion safety. const database = await openInMemoryRelayDatabase() const store = new RelayCredentialStore(database, () => NOW) await insertBasis(database, { id: 'stale-inactive', active: 0, deadline: NOW - 2 * DAY_MS }) await insertBasis(database, { id: 'recent-inactive', active: 0, deadline: NOW - 60_000 }) - // A long-lived splice: still active hours after the 30s deadline it was created with. The - // sweep deactivates it this cycle and the reaper takes it in the same call, which is safe - // precisely because no reader would have accepted it since its deadline passed. + // Sweep deactivates this long-lived splice and reaps it immediately; no reader accepts its already-expired basis. await insertBasis(database, { id: 'stale-active', active: 1, deadline: NOW - 400 * DAY_MS }) await insertBasis(database, { id: 'live-active', active: 1, deadline: NOW + DAY_MS }) diff --git a/cloud/apps/relay/src/credential-store.ts b/cloud/apps/relay/src/credential-store.ts index 58e9ae3c47a..5447431072c 100644 --- a/cloud/apps/relay/src/credential-store.ts +++ b/cloud/apps/relay/src/credential-store.ts @@ -10,29 +10,20 @@ import { HeapWindowReaper } from './heap-window-reaper.js' import { ResumeConfirmedAuditBuffer } from './resume-confirmed-audit-buffer.js' const CREDENTIAL_GRACE_MS = 24 * 60 * 60 * 1000 -// Released desktops validate invite expiry against their own clock with zero -// tolerance at exactly inviteTtlMs; issuing under the ceiling keeps pairing -// working for clients whose clocks trail the cell by up to this margin. +// Issue below released desktops’ zero-tolerance inviteTtlMs ceiling so clients lagging by this margin can still pair. const INVITE_ISSUE_SKEW_MARGIN_MS = 30 * 1000 -// Terminal invites are read by nothing: every reader re-checks expiry and state at read time, so -// the row only serves the audit trail, which relay_audit_events already keeps. A week is long -// enough to answer a support question about a pairing that failed. +// Terminal invite readers recheck state/expiry; retain one week only for pairing support, with audit events already durable. const TERMINAL_INVITE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000 -// Why: both readers of a connection basis and of a direct authorization require it still -// active/unconsumed AND inside its deadline, and every deadline is set at most 30s past insert, so -// a settled row can never authorize anything again. A day is margin for forensics, not for reads. +// Readers require active/unconsumed and unexpired ≤30s deadlines; settled rows cannot authorize again, so one day is forensic margin. const INACTIVE_AUTHORIZATION_RETENTION_MS = 24 * 60 * 60 * 1000 -// A stored confirm result only answers a retry of the same request on the same connection basis, -// and a different basis is refused as a tuple mismatch; a basis lives for one phone connection. A -// week matches the pairing support window above. +// Confirm retries must match the same request and per-phone connection basis; keep one week for pairing support. export const CONFIRM_RESULT_RETENTION_MS = 7 * 24 * 60 * 60 * 1000 // Nothing in the relay reads audit events back; 90 days covers support and incident questions. export const AUDIT_EVENT_RETENTION_MS = 90 * 24 * 60 * 60 * 1000 // Bounded so one cycle cannot hold row locks or grow WAL without limit; the backlog drains over // however many cycles it takes. const REAP_BATCH_ROWS = 5000 -// ~14 rows per page. With ~9 director ticks a minute the row cap is ~3M rows a day, so the 7.4M-row -// backlog drains over two to three days; a walk that finds nothing reads ~1 MB a tick. +// At ~14 rows/page and ~9 ticks/min, budget reaps ~3M rows/day (~2–3 days for 7.4M backlog); empty walks read ~1MB/tick. const CONFIRM_RESULT_REAP_BUDGET = { pagesPerStatement: 16, maxPagesPerTick: 128, @@ -590,9 +581,7 @@ export class RelayCredentialStore { }) } - // Step 6's confirm for a basis the cell holds in memory: one fresh device read, and a renewal - // write at most once a day per device. A current-version confirm always answers renewed:true - // with the expiry the database now holds, so a phone keeps scheduling its renewals. + // Memory confirm reads one device and renews at most daily; current-version replies always use renewed:true with stored expiry so phones schedule future renewals. async confirmMemoryResume( input: RelayIdentity & { reqId: string @@ -643,8 +632,7 @@ export class RelayCredentialStore { } } - // Guarded by the row just read, so a revoke or install that committed since leaves it alone; - // the answer then keeps the expiry that was read, which the database still holds. + // Guard against revoke/install since the read; on no-op keep the expiry read, which the database still holds. private async renewResume( identity: RelayIdentity & { relayDeviceId: string }, device: SqlRow, @@ -782,18 +770,14 @@ export class RelayCredentialStore { await this.reapSettledCredentials(now) } - // Outside the sweep transaction on purpose: each delete is idempotent and independent of the - // state transitions above, so batching them in would only hold their row locks for longer. + // Idempotent reaping stays outside transition transaction or locks last across unrelated batch deletes. private async reapSettledCredentials(now: number): Promise { await this.reapBatch( 'relay_invites', 'state IN (?, ?, ?) AND updated_at <= ?', ['expired', 'consumed', 'invalidated', now - TERMINAL_INVITE_RETENTION_MS] ) - // deadline, not created_at: it is the second column of relay_connection_bases_active_deadline, - // so once the backlog is drained this batch learns there is nothing left to do from the index - // instead of the 1.5 GB heap. Both readers reject a passed deadline, so a day past one is - // unusable whatever the active flag says. + // Use indexed active/deadline range to avoid scanning 1.5GB drained heap; both readers reject expired deadlines regardless of flag, so one-day-old bases are unusable. await this.reapBatch('relay_connection_bases', 'active = ? AND deadline <= ?', [ 0, now - INACTIVE_AUTHORIZATION_RETENTION_MS @@ -811,9 +795,7 @@ export class RelayCredentialStore { await this.reapBatch('relay_audit_events', 'at <= ?', [now - AUDIT_EVENT_RETENTION_MS], 'at') } - // ctid/rowid, not the primary key: the physical address lets the delete re-find exactly the batch - // the subquery located instead of re-matching the predicate per row. On Postgres an array of TIDs, - // not `IN`: IN can plan as a hash join over a sequential scan of the whole table. + // Delete located physical ctid/rowid; PostgreSQL ANY(TID array) guarantees TID scan, while IN can hash-join a full heap scan. private async reapBatch( table: string, predicate: string, diff --git a/cloud/apps/relay/src/database-connect-stall-postgres.test.ts b/cloud/apps/relay/src/database-connect-stall-postgres.test.ts index 5fcb2e82cf4..26886ea83d6 100644 --- a/cloud/apps/relay/src/database-connect-stall-postgres.test.ts +++ b/cloud/apps/relay/src/database-connect-stall-postgres.test.ts @@ -11,9 +11,7 @@ import { isRelayDatabaseTransientError, openRelayDatabase, type RelayDatabase } const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip -// A Cloud SQL instance that is starting has an IP but never answers. The 2026-10-11 staging stall -// came from the transport blocking connect() itself, which no test can reproduce; these pin what -// the pool must do over a transport that does not block: fail fast and keep the event loop turning. +// Cloud SQL startup may accept TCP without replying; pin fast failure/event-loop progress. The 2026-10-11 blocking connect() transport stall itself cannot be reproduced here. type StallingDatabase = { port: number // 'forward' needs a real PostgreSQL behind it. diff --git a/cloud/apps/relay/src/database-postgres-timeout.test.ts b/cloud/apps/relay/src/database-postgres-timeout.test.ts index 9062b1d0511..6018696247a 100644 --- a/cloud/apps/relay/src/database-postgres-timeout.test.ts +++ b/cloud/apps/relay/src/database-postgres-timeout.test.ts @@ -208,9 +208,7 @@ describe('PostgreSQL relay deadlines', () => { ).toBe(5_000) }) - // Why: a statement deadline that reaches the caller as a crash converts a - // transient stall into a failed assignment. It aborts the transaction exactly - // as a lock timeout does, so it belongs on the same bounded retry. + // Real 57014 aborts transactions like lock timeout; retry rather than fail assignments on transient stalls. it('retries a transaction after a statement timeout', async () => { vi.spyOn(console, 'warn').mockImplementation(() => undefined) const database = await openRelayDatabase({ @@ -262,9 +260,7 @@ describe('PostgreSQL schema startup', () => { }) it('fails the boot on a lock timeout instead of re-entering the lock queue', async () => { - // The catalog pre-check already answered that the object is missing, so a lock timeout means - // this boot lost the queue. Relation locks are granted in queue order, so each retry parks - // every writer behind it for another timeout. + // Catalog already proved missing: requeued DDL after lock timeout parks every writer behind it again. const errors: string[] = [] vi.spyOn(console, 'error').mockImplementation((line: string) => { errors.push(line) diff --git a/cloud/apps/relay/src/database-startup-retry.ts b/cloud/apps/relay/src/database-startup-retry.ts index f7ebae8897a..78ee8fd2d7c 100644 --- a/cloud/apps/relay/src/database-startup-retry.ts +++ b/cloud/apps/relay/src/database-startup-retry.ts @@ -6,9 +6,7 @@ export type DatabaseStartupRetryPolicy = { baseDelayMs: number maxDelayMs: number jitterMs: number - // Which failures this particular startup step may repeat. Not every caller can - // repeat everything the request path calls transient: what the retry re-runs - // decides that, so the call site owns it. + // Call site owns retry policy: repeated startup work may be unsafe even for request-transient errors. isRetryable?: (error: unknown) => boolean } @@ -23,9 +21,7 @@ function retryDelayMs(policy: DatabaseStartupRetryPolicy, attempt: number): numb return backoffMs + Math.floor(Math.random() * (policy.jitterMs + 1)) } -// Startup work that a cold dependency - a proxy sidecar that just started, a -// database still accepting the fleet back - can fail once and serve a moment -// later. The wall-clock window, not the attempt count, is the real bound. +// Cold proxies/databases can recover moments later; bound startup retries by wall-clock window, not attempt count. export async function retryTransientDatabaseStartup( operation: () => Promise, policy: DatabaseStartupRetryPolicy, diff --git a/cloud/apps/relay/src/database-statement-timeout-postgres.test.ts b/cloud/apps/relay/src/database-statement-timeout-postgres.test.ts index 66e9cd94a58..6308aab6d6c 100644 --- a/cloud/apps/relay/src/database-statement-timeout-postgres.test.ts +++ b/cloud/apps/relay/src/database-statement-timeout-postgres.test.ts @@ -33,11 +33,7 @@ describePostgres('PostgreSQL statement deadline', () => { expect(result).toBe(2) }, 15_000) - // Why: relay_invites carries a CREATE INDEX IF NOT EXISTS, which (unlike CREATE TABLE IF NOT - // EXISTS) really does queue behind an ACCESS EXCLUSIVE lock on the table. The catalog pre-check - // asks pg_class whether that index is already there, and the read takes no lock on relay_invites, - // so a boot on a migrated database no longer joins the queue at all. It used to, and every writer - // queued behind it in lock order. + // CREATE INDEX IF NOT EXISTS still queues table locks before checking existence; pg_class precheck avoids blocking writers on migrated databases. it('boots without queueing behind a held ACCESS EXCLUSIVE lock', async () => { let releaseTable!: () => void const tableReleased = new Promise((resolve) => { diff --git a/cloud/apps/relay/src/database-transient-error.test.ts b/cloud/apps/relay/src/database-transient-error.test.ts index 5f2b895e01d..2e6792fa07a 100644 --- a/cloud/apps/relay/src/database-transient-error.test.ts +++ b/cloud/apps/relay/src/database-transient-error.test.ts @@ -103,9 +103,7 @@ describe('relay database transient errors', () => { expect(isRelayDatabaseTransientError(error)).toBe(true) }) - // Staging B1: pg_terminate_backend mid-request answered /v1/assign with 500. A dropped - // connection is 503 + Retry-After, as 08006 always was; transaction() still never retries it, - // since the commit outcome is unknown. + // Database-layer drops answer 503/Retry-After (staging B1); never retry transaction work with unknown commit outcome. it('classifies a mid-statement drop only from the database layer', async () => { const failures = [ (): Error => new Error('Connection terminated unexpectedly'), diff --git a/cloud/apps/relay/src/database.ts b/cloud/apps/relay/src/database.ts index ef294ed2f12..2843d8ec050 100644 --- a/cloud/apps/relay/src/database.ts +++ b/cloud/apps/relay/src/database.ts @@ -95,9 +95,7 @@ export interface RelayDatabase { close(): Promise } -// Runs a single-row write with RETURNING as the transaction's last statement and reports -// whether it changed a row. On PostgreSQL the write and COMMIT go as one message, so the -// row lock is held for no round trip; a false result has already rolled the transaction back. +// Final RETURNING write and COMMIT share one PostgreSQL message, avoiding a lock-held round trip; false means already rolled back. export async function commitWithFinalWrite( database: RelayDatabase, sql: string, @@ -686,10 +684,7 @@ CREATE TABLE IF NOT EXISTS relay_audit_events ( CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at); ` -// Rehoming is bidirectional, but tables created before that carry the -// original single-region column check. The old constraint is the one Postgres -// auto-named; the replacement is named, so both statements are no-ops on a -// database the current schema created and neither can drop the other. +// Replace auto-named legacy single-region checks with named bidirectional checks; current-schema databases no-op without either statement dropping the other. export const POSTGRES_SCHEMA_MIGRATIONS = [ POSTGRES_STATEMENT_STATS_MIGRATION, `ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS last_considered_at BIGINT NOT NULL DEFAULT 0`, @@ -707,35 +702,18 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [ // Nullable with no default, so the rewrite is catalog-only; every row // aborted before this column existed reads as an unattributed abort. `ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS abort_reason TEXT`, - // migration-only alone cannot say why a cell is parked there: five flows park loaded cells in - // that state durably. This stamps only the same-cap roll's isolate step, so placement can tell a - // cell being rolled from an evacuation target or an Asia rollback. Nullable with no default, so - // the rewrite is catalog-only; a cell isolated before this column existed reads as unmarked and - // keeps its hosts pinned, which is the pre-existing behaviour. + // Only same-cap roll stamps migration-only; other parked hosts stay pinned. Nullable/no-default is catalog-only, and pre-column isolates retain unmarked legacy behavior. `ALTER TABLE relay_cell_admission ADD COLUMN IF NOT EXISTS roll_isolated_at BIGINT`, // The cap the cell last reported enforcing; see cell-connection-cap.ts. Nullable with no default, // so the rewrite is catalog-only, and a director that predates it neither reads nor clears it. `ALTER TABLE relay_cell_connection_limits ADD COLUMN IF NOT EXISTS reported_hard_cap BIGINT`, - // Dropped, not created: see the comment on relay_assignment_activity_leases. Deferrable because - // this is the one boot where it has to take ACCESS EXCLUSIVE on a table under continuous write, - // and all 28 directors reach it at once; a lock timeout here must not restart the instance, which - // would only re-queue the same DDL behind the same writers. Once it wins, the pre-check answers - // absent and no later boot sends it at all. + // Deferrable DROP: one boot must win ACCESS EXCLUSIVE against continuous writes; restarting 28 directors would repeat the lock queue. Once absent, catalog precheck skips it. `-- schema-deferrable: one boot has to win ACCESS EXCLUSIVE on a table written ~475/s DROP INDEX IF EXISTS relay_assignment_activity_expiry`, - // The drop is what makes HOT legal; this is what makes it possible. A renewal can only reuse the - // row's own page when that page has room for a second version, and at the default fillfactor of - // 100 a freshly filled page has none - measured at 0.5% HOT with the index gone and the default, - // against 100% at 70. Takes SHARE UPDATE EXCLUSIVE, which blocks vacuum and DDL but no reader or - // writer, and only for the catalog write. Applies to pages as they refill, so the table converges - // over its own renewal cycle rather than at boot. - // Deferrable for the same reason, though SHARE UPDATE EXCLUSIVE blocks only vacuum and DDL: it - // buys nothing until the drop lands, so a boot that deferred the drop should defer this too. + // Dropping expiry index permits HOT; fillfactor 70 leaves room for versions (100% HOT vs 0.5% at 100). Catalog-only SHARE UPDATE EXCLUSIVE blocks vacuum/DDL, not reads/writes; defer with the drop and let pages converge through renewals. `-- schema-deferrable: buys nothing until the drop above lands ALTER TABLE relay_assignment_activity_leases SET (fillfactor = 70)`, - // Why: each reservations autovacuum read its ~5.5 GB of indexes in ~3 min, about hourly, evicting - // the cache of the Cloud SQL instance shared with auth. 20 ms per 200 cost units spreads a run - // over ~15 min. Deferrable: a running vacuum holds the same lock, and nothing at boot needs this. + // Slow ~5.5GB-index autovacuums from ~3 to ~15min at 20ms/200 cost units to reduce shared-DB cache eviction. Deferrable: vacuum holds the same lock, and boot needs no change. `-- schema-deferrable: a running vacuum holds the lock this needs ALTER TABLE relay_control_connection_reservations SET (autovacuum_vacuum_cost_delay = 20)`, `-- schema-deferrable: a running vacuum holds the lock this needs @@ -942,9 +920,7 @@ class PostgresTransaction implements RelayDatabase { private lockUnavailable = 0 private lockTimeouts = 0 private state: 'open' | 'committed' | 'rolled-back' = 'open' - // A read timeout leaves the statement running with its reply lost: pg keeps it as the - // client's active query, so anything sent next queues behind it, and if it then lands, a - // later COMMIT would commit a statement its caller saw fail. The connection is finished. + // Lost replies leave pg’s active query running; queued COMMIT could commit caller-failed work, so poison and destroy this connection. private lostReply: Error | undefined constructor( @@ -980,9 +956,7 @@ class PostgresTransaction implements RelayDatabase { try { await timedQuery(this.client, this.connectionId, message, undefined, this.queryTimeout) } catch (error) { - // Simple query stops at the first error, so any server error means COMMIT never ran: - // retryable codes take the caller's normal rollback-and-retry path. A lost connection - // leaves the outcome unknown, and nothing retries it. + // Simple query stops on first error, before COMMIT; retry server errors normally, but never unknown outcomes after connection loss. this.noteFailure(error) if (String((error as { code?: unknown }).code) !== '22012') { rememberPostgresTransactionPhase(error, sql) @@ -1011,8 +985,7 @@ class PostgresTransaction implements RelayDatabase { return hold } - // Drained by the owning database on both the commit and the rollback path: a - // 55P03 rolls the transaction back, so counting only on success would drop it. + // Drain metrics on rollback too or contended 55P03 attempts disappear from counts. consumeLockUnavailable(): number { const count = this.lockUnavailable this.lockUnavailable = 0 @@ -1049,8 +1022,7 @@ class PostgresTransaction implements RelayDatabase { params: unknown[] = [], options: RelayLockOptions = {} ): Promise { - // SET LOCAL lasts to COMMIT, so a bound left in place would silently govern - // every later locked statement in the transaction and misattribute its 55P03s. + // Restore SET LOCAL after lock or later statements inherit the bound and misattribute 55P03s. const bounded = options.lockTimeoutMs !== undefined && !options.failIfUnavailable try { // A blocked waiter holds its pooled client for the whole lock_timeout, so @@ -1095,16 +1067,10 @@ class PostgresTransaction implements RelayDatabase { const POSTGRES_TRANSACTION_ATTEMPTS = 3 const POSTGRES_RETRY_MAX_DELAY_MS = 25 const POSTGRES_CONNECTION_TIMEOUT_MS = 2_000 -// Derivation: a control renewal must land inside its own 30s tick -// (RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2), and a transaction gets -// POSTGRES_TRANSACTION_ATTEMPTS tries, so the worst case a renewal can spend in -// Postgres is attempts * timeout. 5s keeps that at 15s, half the tick, and still -// leaves room for the connect timeout above. +// Three attempts × 5s timeout = 15s, half the 30s renewal tick, leaving connect-timeout room. export const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000 const POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS = 5_000 -// Past statement_timeout because a whole-VM database stall delays replies the server has -// already timed: 5-7 s stalls are routine (connect-burst RCA), and a read timeout inside one -// would destroy healthy clients and fail requests that succeed today. +// Allow routine 5–7s VM stalls to deliver already-timed replies; shorter read deadlines would destroy otherwise healthy clients (connect-burst RCA). export const POSTGRES_READ_TIMEOUT_MARGIN_MS = 10_000 const POSTGRES_IDLE_SESSION_TIMEOUT_MS = 30_000 @@ -1136,9 +1102,7 @@ export function relayPostgresReadTimeoutMarginMs( function retryablePostgresTransactionError(error: unknown): boolean { const code = String((error as { code?: unknown }).code) - // 57014 is the pool statement_timeout firing. It aborts the transaction the - // same way a lock timeout does, so it belongs on the bounded retry path - // rather than surfacing as a terminal failure to the caller. + // 57014 aborts like lock timeout; use bounded transaction retry instead of terminal failure. return code === '40P01' || code === '40001' || code === '55P03' || code === '57014' } @@ -1177,18 +1141,14 @@ export function isRelayDatabaseTransientError(error: unknown): boolean { ) { return true } - // A lost reply, answered 503 like 08006. Its write may have landed, so transaction() never - // retries it. The one startup retry that can see it, the director's cell reconcile - // (cell-admission-startup.ts), re-runs only upserts that converge on the same rows. + // Lost reply answers 503 but may have committed: no transaction retry. Startup reconcile alone retries convergent upserts (cell-admission-startup.ts). if (isPostgresReadTimeout(error)) return true // A pool that cannot hand out a client reports no SQLSTATE at all, so the // acquire boundary owns that vocabulary. return isPostgresPoolConnectFailure(error) } -// The server ended the session (a terminate, shutdown, dropped database or idle timeout) or the -// socket died. pg still calls such a client queryable until the socket's end arrives, so one -// released without an error goes back to the pool and fails the next caller's statement. +// pg may still call ended sessions queryable until socket end; destroy on release or the next caller inherits a dead connection. const SESSION_ENDING_CODES = new Set(['57P01', '57P02', '57P03', '57P04', '57P05', '25P03']) function endsPostgresSession(error: unknown): boolean { if (typeof error !== 'object' || error === null) return false @@ -1428,17 +1388,14 @@ export function readRelayDatabasePoolOldestWaitMs(database: RelayDatabase): numb return database instanceof PostgresDatabase ? database.poolOldestWaitMs() : 0 } -// pg-pool swaps its own listeners on checkout and release; this one never leaves, so no gap -// between them (or a future pg-pool reordering) can turn a client `error` into a crash. The -// idle and checked-out listeners do the logging. +// Keep a listener across pg-pool checkout/release listener swaps so no error gap can crash the process; checked-out/idle listeners own logging. export function keepPostgresClientErrorsHandled(pool: Pick): void { pool.on('connect', (client) => { client.on('error', () => undefined) }) } -// An `options=` in the database URL silently replaces the pool's, so the session reports what -// it actually got. The URL itself is never read or logged. +// URL options can override pool defaults; report effective idle timeout without reading/logging the URL. function reportPostgresIdleSessionTimeoutOnce(pool: Pick): void { let reported = false pool.on('connect', (client) => { @@ -1482,11 +1439,7 @@ async function applySchema(database: RelayDatabase): Promise { } } -// Why: DDL is not a request. A CREATE INDEX on a grown table legitimately runs -// longer than the request statement_timeout, and inheriting that timeout would -// make every startup fail at the same statement instead of finishing once. One -// short-lived connection of its own, ended before the serving pool opens, keeps -// the untimed session off the request path entirely. +// DDL can exceed request deadlines; use a separate untimed schema connection and close it before serving or every startup repeats the same failure. async function applySchemaOnUntimedPool( databaseUrl: string, applicationName: string | undefined @@ -1509,11 +1462,7 @@ async function applySchemaOnUntimedPool( await applyPostgresSchema( relayPostgresSchemaStatements(), async (statement) => await database.query(statement), - // Asks the catalog whether each index or column is already there. CREATE INDEX IF NOT EXISTS - // and ALTER TABLE ADD COLUMN IF NOT EXISTS take their relation lock before the server - // evaluates the existence test, so on an already-migrated database the boot still joins the - // lock queue - and relation locks are granted in queue order, so every writer queues behind - // it. The catalog read takes no lock on the table. + // IF NOT EXISTS takes table locks before checking existence; catalog reads skip migrated objects without queueing writers behind boot DDL. { catalogQuery: async (sql, params) => await database.query(sql, params) } ) } finally { @@ -1538,9 +1487,7 @@ export type RelayDatabaseOpenInput = { readTimeoutMarginMs?: number // Read per query, so a cell's switch moves the margin without a restart. readTimeoutMarginOverrideMs?: () => number | undefined - // Directors own the PostgreSQL schema. A cell skips it and never touches the database - // at boot, so it starts listening while the database is down and stays unready until - // its first successful query. + // Directors own schema; cells skip boot DB access, listen through outages and stay unready until the first successful query. appliesPostgresSchema?: boolean } diff --git a/cloud/apps/relay/src/dead-cell-evacuation-precheck.test.ts b/cloud/apps/relay/src/dead-cell-evacuation-precheck.test.ts index c7958a5b5a7..2fc142a4aaa 100644 --- a/cloud/apps/relay/src/dead-cell-evacuation-precheck.test.ts +++ b/cloud/apps/relay/src/dead-cell-evacuation-precheck.test.ts @@ -9,9 +9,7 @@ import { type SqlRow } from './database.js' -// The sweep's host query walked every assignment by primary key to return nothing while stale -// existing-only cells sat in the fleet. These pin that a fleet with nothing to evacuate never -// reaches it, and that a cell the sweep can act on still does, on both dialects. +// Both dialects must skip host scans with no actionable cells; stale existing-only rows previously scanned every assignment for nothing. const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const schema = 'relay_dead_cell_precheck_test' const HOST_QUERY = 'FROM relay_assignments assignment' diff --git a/cloud/apps/relay/src/drain-release-cell-row-contention-postgres.test.ts b/cloud/apps/relay/src/drain-release-cell-row-contention-postgres.test.ts index 21d2249ffc9..5874b604f86 100644 --- a/cloud/apps/relay/src/drain-release-cell-row-contention-postgres.test.ts +++ b/cloud/apps/relay/src/drain-release-cell-row-contention-postgres.test.ts @@ -19,12 +19,7 @@ import { type StatementDelay } from './test-fixtures/delayed-postgres-database.js' -// Reproduces the Asia drain: a draining cell's releases each end with a one-row -// UPDATE of the source relay_cells row, held for a round trip to COMMIT, while -// the directors re-place its reconnecting hosts. Placement used to lock every -// relay_cells row and starved (about 9/s at 6 releases/s, 5/s at 18/s, every -// director backend lock-waiting); it now locks only the targets. Prints one -// line per run and asserts the after picture. +// Reproduce Asia source-row release locks during reconnect placement: fleet locking fell to ~9/s at 6 releases/s and ~5/s at 18/s. Assert/report target-only locking avoids that contention. const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip @@ -768,9 +763,7 @@ describePostgres('PostgreSQL drain releases against director placement', () => { }, 120_000) it('keeps placing while an Asia drain holds the source row', async () => { - // Before the fix: 90 and 140 of 200 dials rejected by the sticky lane and - // about 3.5 of 3.5 active director backends lock-waiting. Pass bars are on - // those, not on placements/s, which a slow runner's pacing alone can move. + // Baseline sticky rejects were 90–140/200 with ~3.5/3.5 backends lock-waiting; assert those, since runner pacing alone changes placements/s. for (const [rate, neighboursCapped] of [ [6, false], [18, false], @@ -782,9 +775,7 @@ describePostgres('PostgreSQL drain releases against director placement', () => { expect(report.director.lockWaitingMean).toBeLessThan(0.5) expect(report.dials.placedCrossRegion).toBe(neighboursCapped ? report.dials.placed : 0) if (rate === 18) { - // The release commits with its counter write, so the source row is no longer - // held for a round trip and releases stop shedding to lease expiry. With the - // separate COMMIT, 79 of 180 landed and the rest timed out waiting for the pool. + // Fused counter/COMMIT removes the lock-held reply round trip; separate COMMIT once completed only 79/180 releases before pool timeouts. expect(report.releases.ok).toBe(report.releases.attempted) } } diff --git a/cloud/apps/relay/src/drain-return-admission.ts b/cloud/apps/relay/src/drain-return-admission.ts index 5b9989b6b5f..02a1713d388 100644 --- a/cloud/apps/relay/src/drain-return-admission.ts +++ b/cloud/apps/relay/src/drain-return-admission.ts @@ -26,12 +26,7 @@ type DrainReturnLane = Pick< 'acquireDrainReturn' | 'queuedDrainReturns' > -// Hosts whose home cell is isolated for a roll get their own admission budget, -// so a drain's cohort never waits in (or starves) the sticky or placement queues. -// When the lane is full the host is told when to come back: each deferral takes -// the next free service slot after the work already promised, so a cohort larger -// than the lane can serve returns at the rate the lane drains rather than on -// every client's own 2 s retry. +// Give roll-isolated hosts separate admission; overflow books future service slots so cohorts return at drain rate without starving sticky/placement queues. export class RelayDrainReturnAdmission { private serviceMs = DRAIN_RETURN_INITIAL_SERVICE_MS private nextReturnAt = 0 diff --git a/cloud/apps/relay/src/floating-promise-census.test.ts b/cloud/apps/relay/src/floating-promise-census.test.ts index 355ae7a8b02..9e5115306d0 100644 --- a/cloud/apps/relay/src/floating-promise-census.test.ts +++ b/cloud/apps/relay/src/floating-promise-census.test.ts @@ -4,10 +4,7 @@ import { fileURLToPath } from 'node:url' import ts from 'typescript-api' import { describe, expect, it } from 'vitest' -// A promise nobody awaits rejects into Node's unhandledRejection, which ends the process: -// one lost database reply would drop every host on the cell. Production code must await, -// return, store and use, or `.catch` every promise it starts. -// Exempt: functions written to settle every failure themselves. Keyed file#name. +// Unhandled database rejections can kill a cell and drop every host; consume/catch all promises. Only self-settling functions keyed file#name are exempt. const NEVER_REJECTS = new Map([ ['relay-background-operation.ts#runRelayBackgroundOperation', 'catches and logs every failure'], ['cloud-run-instance-zone.ts#readCloudRunInstanceZone', 'one try/catch; resolves undefined on any failure'], diff --git a/cloud/apps/relay/src/heap-window-reaper.ts b/cloud/apps/relay/src/heap-window-reaper.ts index 3e7a1dd74cf..16829dc3f4d 100644 --- a/cloud/apps/relay/src/heap-window-reaper.ts +++ b/cloud/apps/relay/src/heap-window-reaper.ts @@ -11,10 +11,7 @@ export type HeapWindowReapBudget = { budgetMs: number } -// Why a TID range and not `WHERE LIMIT n`: these tables have no index on their -// retention column, so the planner answers LIMIT with a sequential scan from page 0 (production -// EXPLAIN, 2026-10-04). That scan gets longer every tick as the reaped head of the heap empties. -// A TID range bounds each statement to its own pages whatever the table holds. +// Unindexed retention LIMIT scans restart at page 0 and worsen as the head empties (production EXPLAIN 2026-10-04); TID windows bound each statement’s pages. export class HeapWindowReaper { private nextPage: number | undefined @@ -53,9 +50,7 @@ export class HeapWindowReaper { ) { if (page >= pages) page = 0 const end = Math.min(page + this.budget.pagesPerStatement, pages) - // SKIP LOCKED: a row some request holds is left for a later pass rather than waited on. - // `= ANY(ARRAY(...))`, not `IN (...)`: IN can plan as a hash join over a sequential scan of - // the whole table; an array of TIDs is always a TID scan. + // SKIP LOCKED defers busy rows; ANY(TID array) guarantees TID scan while IN may hash-join a full sequential scan. deleted += changes( await database.query( `DELETE FROM ${this.table} WHERE ctid = ANY(ARRAY( diff --git a/cloud/apps/relay/src/host-close-reason-memory.ts b/cloud/apps/relay/src/host-close-reason-memory.ts index ed01aacd666..0e716a47c52 100644 --- a/cloud/apps/relay/src/host-close-reason-memory.ts +++ b/cloud/apps/relay/src/host-close-reason-memory.ts @@ -11,10 +11,7 @@ const RETENTION_MS = ASSIGNMENT_LIMITS.dormantTtlMs // burst well above any single cell's host count without becoming a leak. const MAX_ENTRIES = 50_000 -// Why in-memory and not Postgres: a phone reaches the cell its host's assignment -// row already names, which is the same cell that watched the control socket -// close. Losing this on a cell restart degrades to the pre-existing generic -// verdict, so the failure mode is the old behaviour rather than a wrong one. +// Phones reach the cell named by assignment, which observed host close; restart loses only explanation and falls back to the legacy generic verdict. export class HostCloseReasonMemory { private readonly entries = new Map() diff --git a/cloud/apps/relay/src/host-session-client-accept.test.ts b/cloud/apps/relay/src/host-session-client-accept.test.ts index 4ca89d106f1..33a332949e1 100644 --- a/cloud/apps/relay/src/host-session-client-accept.test.ts +++ b/cloud/apps/relay/src/host-session-client-accept.test.ts @@ -14,11 +14,7 @@ import type { RelayRuntimeObserver } from './relay-observability.js' import type { RelayTokenClaims } from './relay-token-verifier.js' import { ProcessQueuedByteBudget } from './splice-forwarder.js' -// Incident 2026-09-04 ~01:05Z: the phone's dial bound ran out while the cell was -// still inside acceptClient's serialized Postgres phase (cell-inventory lock -// contention). The cell then finished the work for a socket nobody held, holding -// an activity lease for the 10s attach deadline before its timer unwound it, and -// logged `host_data_reservation_already_bound`. +// 2026-09-04: phone dial timed out during serialized SQL; late accept leaked a 10s lease and failed bind with host_data_reservation_already_bound. class FakeSocket extends EventEmitter { readonly OPEN = 1 diff --git a/cloud/apps/relay/src/host-session-registry.test.ts b/cloud/apps/relay/src/host-session-registry.test.ts index fc4bf374824..16b751650e3 100644 --- a/cloud/apps/relay/src/host-session-registry.test.ts +++ b/cloud/apps/relay/src/host-session-registry.test.ts @@ -354,10 +354,7 @@ describe('host session cleanup races', () => { })! session.activeSplices.set('conn-a', () => session.activeSplices.delete('conn-a')) - // POST /v1/admin/drain has no idempotency guard, and SIGTERM then SIGINT both - // reach drain(), so a retry re-sends to every session. It must re-arm the pending - // teardown rather than stack a second one: across a paced cell that is 800 orphaned - // timers per retry, each one holding the loop open for the rest of the window. + // Drain retries/signals must re-arm, not stack teardown; a paced 800-host cell otherwise leaks 800 timers holding exit open each retry. registry.drain(0) const scheduled = vi.getTimerCount() registry.drain(0) @@ -366,8 +363,7 @@ describe('host session cleanup races', () => { expect(vi.getTimerCount()).toBe(scheduled) vi.advanceTimersByTime(1) - // Asserting registry state, not the log line: FakeSocket closes synchronously, so - // the line is already emitted before the second teardown runs and would pass either way. + // Check state: synchronous FakeSocket close logs before second teardown, so log-only assertions miss corruption. expect(session.closingCounts).toEqual({ splices: 1, pending: 0 }) }) @@ -557,8 +553,7 @@ describe('host session cleanup races', () => { ) }) - // The root of a seat c34's feed listed for hours: a rebind whose database activation outlived - // the orphan grace revived the removed session outside the map, still listed and unreachable. + // c34 ghost seat came from activation outliving orphan grace and reviving a removed session outside the map. it('refuses a rebind whose session left the map while its activation was in flight', async () => { const blocked = deferred() const activateControl = vi diff --git a/cloud/apps/relay/src/host-session-registry.ts b/cloud/apps/relay/src/host-session-registry.ts index 1821cf65320..8c33507ef29 100644 --- a/cloud/apps/relay/src/host-session-registry.ts +++ b/cloud/apps/relay/src/host-session-registry.ts @@ -99,8 +99,7 @@ export type HostSession = { // Null for a control admitted from memory until a flip back registers it. controlActivityId: string | null readonly controlResumeSecret: string - // Why: reconnect churn is only actionable once it can be pinned to a client build. - // Refreshed on rebind so it describes the socket that closed, not the first one. + // Refresh on rebind to attribute churn to the closing socket’s build. appVersion: string state: HostState socket: WebSocket | null @@ -121,8 +120,7 @@ export type HostSession = { activeConnIds: Set activeSplices: Map void> pendingConns: Map - // Why: relay-initiated teardown drains these maps before closing the control - // socket, so the close handler would otherwise always report zero destroyed work. + // Capture before teardown drains maps or control-close logs report zero destroyed work. closingCounts: { splices: number; pending: number } | null regionalDrainAttemptId: string | null regionalDrainTimer: ReturnType | null @@ -252,19 +250,10 @@ const preAuthEnded = // stalled predecessor only accumulates doomed sockets. const ACTIVATION_QUEUE_WAIT_MS = 30_000 -// A drained host redials the director the moment it is told, and a redial that meets this -// cell's release still holding its assignment row is refused row_busy. From asia-east2 one -// release holds the row ~1 s (c34/c35 drains, 2026-10), and a host's releases run one after -// another, so 3 s covers a control plus one phone. Past that, it is told anyway. +// Release before telling host to redial or it meets row_busy; 3s covers Asia control+phone serial releases (~1s each, c34/c35), then close anyway. export const DRAIN_RELEASE_BEFORE_CLOSE_MS = 3_000 -// Why: this lease bounds how long a host lingers on a cell after a missed drain, -// and rebinding it is the only passive rebalancing we have, so it has to stay -// finite. 6h keeps both properties while cutting control-activation traffic on -// the contended cell-inventory lock ~6x; the relay JWT (5 min, refreshed by the -// desktop) and the 75s silence watchdog are enforced separately, so a longer -// grant authorizes nothing extra. Symmetric jitter walks same-minute reconnect -// cohorts apart across cycles without changing the mean rebind rate. +// Finite 6h grant bounds missed drains/passive rebalance while cutting activation traffic ~6×. JWT and silence checks stay independent; symmetric jitter disperses cohorts without changing mean rebind rate. export const CONTROL_LEASE_MS = 6 * 60 * 60 * 1000 export const CONTROL_LEASE_JITTER_MS = 30 * 60 * 1000 @@ -321,9 +310,7 @@ type ReregistrationEntry = { key: string; generation: number; attempts: number; export class HostSessionRegistry { private readonly sessions = new Map() private readonly activationQueues = new Map>() - // Why it outlives `sessions`: the orphan grace deletes the session within 30s, - // but a signed-out desktop never comes back, so the phone that asks minutes - // later would otherwise find nothing to explain its rejection with. + // Outlive 30s orphan grace so phones can explain a signed-out host’s absence minutes later. private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now()) private readonly hostCapabilities = new WeakMap>() private draining = false @@ -503,9 +490,7 @@ export class HostSessionRegistry { return this.reserveMode() && this.reserveAdmission?.phonePath?.() === 'memory' ? 'memory' : 'db' } - // Leaving reserve mode: every control admitted from memory gets the database lease it never - // took, while it stays connected. Rescans until none lacks one, so a hello whose proof lands - // after the flip is caught too. A control whose row never names this cell is closed (4409). + // Flip-back leases every memory control without disconnecting it; rescan catches proofs landing after flip. Close 4409 when its row never names this cell. reregisterMemoryControls(): number { if (this.config.role !== 'cell') return 0 if (this.reregistration) clearInterval(this.reregistration.timer) @@ -659,11 +644,7 @@ export class HostSessionRegistry { this.sessions.get(entry.key) === session && session.generation === entry.generation && socket?.readyState === socket?.OPEN - // A database that did not answer (a stall, a dropped connection, a busy lock) is retried - // for as long as the control is open: a fleet-wide flip back must not close everyone over - // a slow database. An answer that refuses (a row naming another seat, which may still be - // the ledger's write on its way, or any other refusal) counts toward giving up. A lease - // insert that lost a race (23505) is retried too: the next try reads the winner's row. + // Retry unanswered DB/transient/23505 races while control stays open; count actual refusals toward give-up, allowing delayed ledger rows without dropping fleet-wide fallback. const unanswered = isRelayDatabaseTransientError(error) || (error instanceof Error && error.message === 'database_lock_unavailable') || @@ -783,8 +764,7 @@ export class HostSessionRegistry { session.assignmentEpoch !== request.epoch || seat?.joinedAt !== request.joinedAt ) { - // The director named exactly the seat this feed lists: if no live session backs it, the - // feed is what is wrong, and every director mirrors it, so correct the feed now. + // Director named the exact logged seat; if no session backs it, repair the feed every director mirrors. const unbacked = seat?.epoch === request.epoch && seat.joinedAt === request.joinedAt ? unbackedSeatReason(seat, session) @@ -815,9 +795,7 @@ export class HostSessionRegistry { return 'demoted' } - // Renewals leave the heartbeat as an enqueue: one statement per cell per - // window replaces one write transaction per host, which is what keeps the - // shared PostgreSQL instance out of buffer-header contention. + // Batch heartbeat renewals per cell/window to replace per-host write transactions and avoid shared PostgreSQL buffer-header contention. private readonly controlRenewals = new ControlRenewalBatch( async (rows) => await this.assignments.renewControlActivities(rows), () => this.logIdentity(), @@ -855,18 +833,13 @@ export class HostSessionRegistry { credential: string, capacityReservation?: PendingHostDataReservation ): Promise { - // Not `this.draining`: a paced drain tells hosts minutes apart, and the director keeps - // pointing phones here until their own host has moved. Refusing them for the whole - // window would turn a 2 min drain into a 2 min outage for hosts not yet told. + // Paced drains tell hosts minutes apart; reject phones only after their host is told or the entire window becomes an outage. if (this.drainSentHosts.has(hostId)) { capacityReservation?.release() this.rejectClient(socket, RELAY_CLOSE_CODE.DRAINING) return } - // Why: the accept runs several serialized Postgres calls behind the contended - // cell-inventory lock, and phones bound their dial. Finishing the work for a - // phone that already hung up took an activity lease held for the 10s attach - // deadline, then failed at bind with host_data_reservation_already_bound. + // Phones may hang up while serialized SQL waits; stop late accepts to avoid 10s leaked leases and host_data_reservation_already_bound. const acceptStartedAt = this.now() const abandonedByClient = (stage: RelayClientAcceptStage, cleanup?: () => void): boolean => { if (socket.readyState === socket.OPEN) return false @@ -1075,8 +1048,7 @@ export class HostSessionRegistry { markStage: (stage: RelayClientAcceptStage) => void ): Promise { if (this.config.role === 'cell') { - // Each lookup is its own pooled round trip; stop between them once the phone - // has left instead of running the rest of the chain for nobody. + // Check abandonment between pooled round trips so departed phones cannot drive the rest of the lookup chain. let outerIdentity = deviceMissed ? null : await this.store.resolveResume(hostId, credential) if (abandonedByClient('assignment')) return null if (!outerIdentity) { @@ -1105,8 +1077,7 @@ export class HostSessionRegistry { return reservation } - // No seat for a memory resume: one assignment read tells "moved" from "offline". A row that - // names this cell may be a lost ledger write (stale up to the 10 min reconcile), so it is logged. + // One assignment read distinguishes moved/offline for missing seats; rows naming this cell may be lost ledger writes (up to 10min), so log them. private async rejectMissedSeat( socket: WebSocket, reservation: CredentialReservation, @@ -1348,10 +1319,7 @@ export class HostSessionRegistry { } } - // Every desktop build already echoes the ping's `t`, so a pong is only timed when - // it answers the outstanding ping: at most one sample per ping this cell sent, - // however many a host floods. A pong that lost the race to the next ping is - // dropped here but still counts as proof of life for the silence watchdog. + // Time only outstanding ping echoes, once per ping; superseded pongs still prove liveness but cannot flood samples/logs. private recordControlRtt(session: HostSession, echoedPingAt: unknown): void { if (typeof echoedPingAt !== 'number' || echoedPingAt !== session.pendingPingAt) return session.pendingPingAt = null @@ -1449,10 +1417,7 @@ export class HostSessionRegistry { } } - // A dependency failure (e.g. a database connect timeout) must cost one - // handshake or command, not the process: an unhandled rejection here has - // crashed whole cells and wiped their in-memory draining flag. 4429 is the - // endpoint-scoped close in the contract, so the client retries this cell. + // Dependency failure must cost one handshake/command, not crash the cell and erase draining state. Contract close 4429 retries this endpoint. private guardSessionTask( task: () => Promise, socket: WebSocket | null, @@ -1508,15 +1473,12 @@ export class HostSessionRegistry { this.draining = true // A draining cell admits no booking, so the units they hold go back now. this.reserveAdmission?.book.clear() - // A later drain (an emergency one, or shutdown) owns every session again, so nothing - // queued by an earlier paced drain may still fire: it would re-send and, worse, keep - // the event loop alive for the rest of a window the operator just cut short. + // Later drain supersedes all earlier paced timers; stale sends would repeat closes and hold exit through a shortened window. for (const timer of this.drainTimers) clearTimeout(timer) this.drainTimers.clear() const paceWindowMs = Math.max(0, Math.trunc(options.paceWindowMs ?? 0)) const targets = [...this.sessions.values()].filter((session) => session.state !== 'closed') - // The desktop re-dials the director as soon as it reads `drain`, whatever graceMs says, - // so spreading the send is the only thing that spreads the reconnect load. + // Desktop redials on drain receipt regardless of grace; pace sends to spread reconnect load. const step = paceWindowMs > 0 && targets.length > 1 ? paceWindowMs / (targets.length - 1) : 0 for (const [index, session] of targets.entries()) { const delay = Math.round(step * index) @@ -1696,9 +1658,7 @@ export class HostSessionRegistry { (existing.state === 'orphaned' || existing.state === 'active') && sameHostIdentity(existing.identity, identity) ) - // Above every generation this host has had here, not only the one in the map: a session - // dropped from the map while its socket is still closing (drain, regional deadline, idle - // rehome) leaves later, and a successor reusing its generation had its seat removed by that leave. + // Advance beyond remembered generations after map removal; delayed old socket leaves must never erase successor seats. const floor = this.generationFloors.get(key) ?? 0 const generation = rebind ? existing!.generation : Math.max(existing?.generation ?? 0, floor) + 1 const ephemeral = nacl.box.keyPair() @@ -1930,10 +1890,7 @@ export class HostSessionRegistry { } return } - // The awaits above can outlive what this hello was proven against: an orphan grace expiring - // or a drain removes `existing` from the map, and another activation may take this generation. - // Rebinding a removed session revived it outside the map: connected, still in the feed, and - // unreachable by phones or demotion for as long as the host stayed up. Redial instead. + // Recheck after awaited activation: orphan/drain may remove existing or another activation take its generation. Reviving it creates connected, listed ghosts outside the map; redial instead. const key = this.key(identity.sub, identity.relayHostId) if ( (this.sessions.get(key) ?? null) !== existing || @@ -1959,8 +1916,7 @@ export class HostSessionRegistry { // id today, since a rebind keeps its generation). if (controlActivityId) existing.controlActivityId = controlActivityId existing.socket = socket - // The hello's token was just verified; keeping the predecessor's would let the next - // heartbeat expire a control that proved itself, and every resume would repeat it. + // Use the newly verified expiry or predecessor’s expired token would close every successful resume at the next heartbeat. if (identity.exp > existing.identity.exp) existing.identity = identity existing.state = existing.regionalDrainAttemptId ? 'drain-only' : 'active' existing.appVersion = appVersion @@ -2185,9 +2141,7 @@ export class HostSessionRegistry { const socket = session.socket! const wiredAt = this.now() session.controlWiredAt = wiredAt - // Why: pin the build to THIS socket. A rebind refreshes session.appVersion and - // only then closes the predecessor, whose close event always lands after that - // write, so reading it at log time would stamp the successor's build. + // Capture build per socket before rebind updates session.appVersion; predecessor’s later close must not log successor’s build. const appVersion = session.appVersion let socketError: string | null = null // Why: an unhandled ws 'error' (e.g. an oversize control frame) would @@ -2395,10 +2349,7 @@ export class HostSessionRegistry { return } if (error instanceof Error && error.message === 'assignment_lock_unavailable') { - // A per-host transaction held the row, so the batch passed over it - // rather than making every other host in the flush wait. The next - // tick is 15s away against a 105s lease, and the flush line already - // reports the count, so this needs no line of its own. + // SKIP LOCKED defers this host, not the batch; next tick 15s vs 105s lease is safe, and flush metrics already count it. return } console.warn('[orca-relay] control activity renewal failed') @@ -2430,9 +2381,7 @@ export class HostSessionRegistry { private sendHelloAck(session: HostSession): void { if (!session.socket) return - // Without these a host that missed the conn-open cannot dial the pending - // connection: it would have to guess the pairing kind and the device the - // relay authorized. Only sent to a host that said it can read them. + // Capability-gate pairing/device details: hosts missing conn-open otherwise cannot dial pending connections, while legacy decoders reject new fields. const details = this.hostCapabilities .get(session.socket) ?.has(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS) @@ -2526,8 +2475,7 @@ export class HostSessionRegistry { close() return } - // The host redials on this close, so its row is released first; a stuck release still closes. - // The splice releases above are queued ahead of these, so they have landed too. + // Release before close triggers redial; queued splice releases land first too. Bound the wait so stuck releases still close. const startedAt = this.now() void settledWithin(Promise.all(releases), DRAIN_RELEASE_BEFORE_CLOSE_MS).then((settled) => { if (!settled) { @@ -2736,9 +2684,7 @@ export class HostSessionRegistry { if (session.socket) send(session.socket, 'control-error', { ...(reqId ? { reqId } : {}), code }) } - // closeReason rides the WebSocket close reason, never relay-hello: every shipped - // phone parses relay-hello with a strict schema that rejects an unknown key, and - // ignores a close reason it does not know. + // Put closeReason only on WebSocket close; shipped strict relay-hello parsers reject unknown keys but tolerate unknown close reasons. private rejectClient( socket: WebSocket, code: number, diff --git a/cloud/apps/relay/src/idle-regional-rehome-selection.ts b/cloud/apps/relay/src/idle-regional-rehome-selection.ts index f9757016416..28669376256 100644 --- a/cloud/apps/relay/src/idle-regional-rehome-selection.ts +++ b/cloud/apps/relay/src/idle-regional-rehome-selection.ts @@ -4,14 +4,10 @@ import type { RelayDatabase, SqlRow } from './database.js' export const IDLE_REHOME_PAGE_SIZE = 100 -// How many decision rows one poll is allowed to look at. The poll runs about -// fifty times a minute across the directors, so its cost has to be set by this -// number and not by the size of the fleet or the width of the cohort. +// Bound each of ~50 polls/min by decision rows, independent of fleet/cohort size. export const IDLE_REHOME_DECISION_WINDOW = 500 -// Where the last window ended. A keyset beats OFFSET: `OFFSET n` still has to -// produce and throw away n rows, and n grew by a page on every poll that -// dispatched, so the scan got more expensive the longer the rollout ran. +// Keyset cursor avoids OFFSET’s growing discarded-row scan cost over long rollouts. export type IdleRehomeHostCursor = { userId: string; relayHostId: string } | null export type IdleRegionalRehomeCandidate = IdleRegionalRehomeRequest & { sourceCellUrl: string } @@ -23,10 +19,7 @@ export type IdleRegionalRehomeSelection = { skippedOffRegionSourceCells: number } -// TEMPORARY STOPGAP: delete with its call sites once the rehome commit stops holding -// relay_cells row locks across cross-region round trips. The source cell runs the commit, -// and one outside the database's region holds fleet-wide locks for seconds, so it may not -// be a source. It stays a target: only the source side of the commit pays the round trips. +// Source-region stopgap keeps commits beside the database while remote cells remain targets; reassess now rehome uses a final NOWAIT cell-row write instead of prolonged fleet locks. export function rehomeSourceRegionAllowed( sourceRegion: string, directorRegion: string | undefined @@ -92,10 +85,7 @@ export async function selectIdleRegionalRehomes( const after = input.cursor ? [input.cursor.userId, input.cursor.relayHostId] : [] const afterFilter = input.cursor ? 'AND (user_id, relay_host_id) > (?, ?)' : '' - // The window is taken first and on its own so the poll knows where it stopped - // reading, not just where it stopped emitting. Every gate below this point can - // reject a host, and a cursor that only advanced past emitted rows would park - // on a rejected host forever. + // Advance by the read window, not emitted candidates, or rejected hosts can pin the cursor forever. const window = await input.database.query( `SELECT user_id, relay_host_id FROM relay_region_decisions WHERE ${decisionFilter} ${afterFilter} @@ -108,10 +98,7 @@ export async function selectIdleRegionalRehomes( const sourceList = [...cells.sources.values()] const rows = await input.database.query( - // The verification names the window's keys rather than repeating its LIMIT: - // the two reads take separate snapshots, and a decision that turned eligible - // between them would otherwise shift the second LIMIT and push the last host - // out of it while the cursor still advanced past it. + // Verify exact window keys: separate snapshots with repeated LIMIT can admit a newly eligible row while skipping the old last key forever. `SELECT d.user_id, d.relay_host_id, d.preferred_region, a.cell_id AS source_cell_id, a.assignment_epoch, host.generation FROM (SELECT user_id, relay_host_id, preferred_region, incumbent_region, assignment_epoch @@ -188,9 +175,7 @@ export async function selectIdleRegionalRehomes( } } -// Every cell predicate the candidate join used to re-evaluate per (host, cell) -// pair. There are tens of cells and tens of thousands of hosts, so this is -// resolved once per poll against the four small inventory tables. +// Resolve small inventory predicates once per poll instead of per tens-of-thousands of host/cell pairs. async function readCellInventory( input: SelectionInput ): Promise<{ @@ -290,9 +275,7 @@ function placeholders(count: number): string { return Array.from({ length: count }, () => '?').join(',') } -// A derived table the planner can hash, in the one syntax both Postgres and the -// SQLite test engine accept (`VALUES ... AS t(col)` and LATERAL are not common to -// both). Only the first branch is cast; both engines take the union's types from it. +// Derived UNION table is hashable on PostgreSQL and SQLite (unlike aliased VALUES/LATERAL); first branch sets both engines’ types. function inlineRows(columns: ReadonlyArray, rows: number): string { const first = columns.map(([name, type]) => `CAST(? AS ${type}) AS ${name}`) const rest = Array.from({ length: rows - 1 }, () => `UNION ALL SELECT ${placeholders(columns.length)}`) diff --git a/cloud/apps/relay/src/index.ts b/cloud/apps/relay/src/index.ts index 503ef3bc9a6..594b08e734d 100644 --- a/cloud/apps/relay/src/index.ts +++ b/cloud/apps/relay/src/index.ts @@ -81,9 +81,7 @@ const { if (process.env.K_SERVICE) { void readCloudRunInstanceZone().then((zone) => observability.setInstanceZone(zone)) } -// Same owner as the assignment sweep: the cleanup only expires credentials that every reader -// already re-checks at read time, so running it in all 23 cells multiplied one table scan by 23 -// without changing any answer. +// Only assignment-maintenance owner reaps credentials; readers already enforce expiry, so cell-wide copies multiplied scans without changing answers. const cleanupTimer = roleOwnsAssignmentMaintenance(config.role) ? setInterval( () => diff --git a/cloud/apps/relay/src/memory-resume-confirmations.ts b/cloud/apps/relay/src/memory-resume-confirmations.ts index c2de7a7be7a..1532f67effd 100644 --- a/cloud/apps/relay/src/memory-resume-confirmations.ts +++ b/cloud/apps/relay/src/memory-resume-confirmations.ts @@ -47,8 +47,7 @@ export class MemoryResumeConfirmations { this.bases.delete(basisConnId) } - // Routed by where the basis lives, not by the switch, so a flip between accept and confirm - // neither fails the confirm nor sends it to a database that holds no basis for it. + // Route by basis ownership, not current switch, so mid-confirm flip cannot send an in-memory basis to nonexistent DB state. owns(request: ResumeConfirmRequest): boolean { this.expireResults() return this.bases.has(request.basisConnId) || this.results.has(resultKey(request)) diff --git a/cloud/apps/relay/src/placement-load-band.ts b/cloud/apps/relay/src/placement-load-band.ts index 91cfecc6353..1a9e6c5568a 100644 --- a/cloud/apps/relay/src/placement-load-band.ts +++ b/cloud/apps/relay/src/placement-load-band.ts @@ -1,7 +1,4 @@ -// Spreads placement over cells near the minimum load, so a freshly rolled -// (empty) cell does not take a whole drain and serialize every follow-up on -// its one relay_cells row. It only orders candidates: a candidate is never -// refused, and the pool's least-loaded cell is the fallback. +// Spread near-minimum placements so one empty rolled cell cannot serialize an entire drain’s follow-ups; reorder only, keeping least-loaded fallback and never refusing candidates. // Cells within this share of capacity of the least-loaded one are equals. export const PLACEMENT_LOAD_BAND = 0.05 diff --git a/cloud/apps/relay/src/postgres-lock-wait-sample.ts b/cloud/apps/relay/src/postgres-lock-wait-sample.ts index f26c26d9b86..cb03d6a8405 100644 --- a/cloud/apps/relay/src/postgres-lock-wait-sample.ts +++ b/cloud/apps/relay/src/postgres-lock-wait-sample.ts @@ -1,16 +1,7 @@ import type { RelayDatabase } from './database.js' import type { DatabaseLockWaitSample } from './relay-observability.js' -// Every relay process connects as the same user through a socket, so neither -// pg_stat_statements nor Query Insights can tell a director's lock wait from a -// cell's. application_name (`orca-relay//`) can, so this samples it. -// The holder is the root of the wait chain: in a row-lock convoy every later -// waiter is blocked by the first waiter, not by the transaction holding the row. -// Blockers are read once per waiter; non-relay waiters stay in so chains through -// them resolve, and the depth cap bounds a cycle. The table is the first relay -// table the waiting statement names, which may be one it only references. -// It shares the director's 3-slot pool, so when every slot is a lock waiter the -// sample queues and undercounts director waiters; a dedicated connection fixes that. +// Use application_name (orca-relay//) to separate shared-user waiters. Follow blockers to root, keeping non-relay intermediates and bounding depth for cycles. First referenced relay table labels the wait. Shared 3-slot pool can queue/undercount samples when saturated; dedicated sampling avoids it. const LOCK_WAIT_SAMPLE_SQL = ` WITH RECURSIVE waiting AS MATERIALIZED ( SELECT pid, application_name, query, (pg_blocking_pids(pid))[1] AS blocker diff --git a/cloud/apps/relay/src/postgres-pool-pressure.ts b/cloud/apps/relay/src/postgres-pool-pressure.ts index 0851402cdbe..6ff2e073290 100644 --- a/cloud/apps/relay/src/postgres-pool-pressure.ts +++ b/cloud/apps/relay/src/postgres-pool-pressure.ts @@ -9,9 +9,7 @@ export type PostgresPoolPressureCounts = { databasePoolWaitMsMax: number } -// A pool that cannot hand out a client throws a bare Error with no SQLSTATE, so -// the message is all node-postgres gives us. Both of these come only from -// pg-pool's connect path, so neither can be a statement that already ran. +// pg-pool connect timeouts have no SQLSTATE; these messages prove acquisition failed before a statement ran. const POOL_CONNECT_TIMEOUT_MESSAGES = [ // No pooled client came free within connectionTimeoutMillis. 'timeout exceeded when trying to connect', @@ -87,9 +85,7 @@ export class PostgresPoolPressure { private waitersMax = 0 private waitMsMax = 0 private lastConsumed = emptyCounts() - // General work queues here, not in pg-pool, so pg-pool's own queue holds only - // priority callers. Holding a connection back instead would cost general - // throughput: one of 16 is the margin an asia-east2 drain release runs on. + // Queue general work here so pg-pool queues only priority; reserving an idle connection would lose Asia drain throughput. private readonly generalCapacity: number private generalInUse = 0 private readonly generalQueue: GeneralLaneWaiter[] = [] @@ -232,14 +228,7 @@ async function markedAcquire( return guardCheckedOutClient(client, onRelease) } -// pg-pool strips its own `error` listener when it hands a client out -// (pg-pool@3.14.0 index.js:344) and only reattaches it in `_release` -// (index.js:385), so a checked-out client has no `error` listener at all. A -// backend that terminates that session mid-statement therefore emits `error` -// with nothing listening, which is an unhandled 'error' event and kills the -// process. `pool.on('error')` cannot cover this: pg-pool routes there only from -// the idle listener. Every relay checkout awaits this function, so it is the -// one seam that sees them all. +// pg-pool removes error listeners on checkout and restores them on release; pool.on(error) covers only idle clients. Guard every checkout or terminated in-flight sessions crash the process. function guardCheckedOutClient(client: pg.PoolClient, onRelease: () => void): pg.PoolClient { let failure: Error | undefined const onError = (error: Error) => { diff --git a/cloud/apps/relay/src/public-assignment-admission.ts b/cloud/apps/relay/src/public-assignment-admission.ts index 18dd77c4514..730e5a24d8b 100644 --- a/cloud/apps/relay/src/public-assignment-admission.ts +++ b/cloud/apps/relay/src/public-assignment-admission.ts @@ -47,9 +47,7 @@ export class RelayPublicAssignmentAdmission { waitMs?: number maxReservedConcurrent?: number reservedWaitMs?: number - // Drain returns borrow placement permits: their re-placement is placement - // work, serialized on the same store mutex, so it fits the pool budget the - // placement lane already has. Placement and reserved waiters go first. + // Drain-return placement shares the store mutex/pool budget, borrowing permits only after placement and reserved waiters. maxDrainReturnConcurrent?: number maxDrainReturnQueued?: number drainReturnWaitMs?: number diff --git a/cloud/apps/relay/src/regional-host-drain-app.test.ts b/cloud/apps/relay/src/regional-host-drain-app.test.ts index 1c179a696f5..b6209127425 100644 --- a/cloud/apps/relay/src/regional-host-drain-app.test.ts +++ b/cloud/apps/relay/src/regional-host-drain-app.test.ts @@ -371,9 +371,7 @@ describe('regional rehome director controls', () => { }) it('accepts the exact safety payload the cell heartbeat composes', async () => { - // Why: index.ts spreads the FULL pool-pressure counts into safety; a - // strict schema missing any produced field 400s every heartbeat (the - // 2026-08-15..26 outage that left all cells at rehome protocol 0). + // index.ts sends full pool-pressure safety counts; missing strict-schema fields once rejected every heartbeat and left rehome protocol at 0 (2026-08-15..26). const recordCellRegionalRehomeStatus = vi.fn().mockResolvedValue(undefined) const app = createRelayApp(config({ role: 'director', cellId: 'director' }), { store: {} as never, diff --git a/cloud/apps/relay/src/regional-rehome-abort-reason.ts b/cloud/apps/relay/src/regional-rehome-abort-reason.ts index 96be5074a72..ad7a1622b40 100644 --- a/cloud/apps/relay/src/regional-rehome-abort-reason.ts +++ b/cloud/apps/relay/src/regional-rehome-abort-reason.ts @@ -7,11 +7,7 @@ export const REGIONAL_REHOME_ABORT_REASONS = ['host_not_arrived', 'max_refresh_e export type RegionalRehomeAbortReason = (typeof REGIONAL_REHOME_ABORT_REASONS)[number] -// One whole migration lease with the host absent from the target and owning -// nothing on the source. Nothing legitimately takes that long: the attach -// deadline is seconds, and a host that is still moving holds a lease at one end -// or the other. The control's `drain_grace_ms` does not fit — idle rehome -// commits with a grace of zero, so these attempts never carry one. +// One migration lease with neither target arrival nor source ownership proves abandonment; attach takes seconds. Drain grace cannot bound idle rehomes, which commit with zero grace. export const REGIONAL_REHOME_ARRIVAL_WINDOW_MS = ASSIGNMENT_LIMITS.migrationLeaseMs // The window the inventory line and the preview both report aborts over. diff --git a/cloud/apps/relay/src/regional-rehome-poll-telemetry.ts b/cloud/apps/relay/src/regional-rehome-poll-telemetry.ts index b25a9476cee..9a4eb5a8a69 100644 --- a/cloud/apps/relay/src/regional-rehome-poll-telemetry.ts +++ b/cloud/apps/relay/src/regional-rehome-poll-telemetry.ts @@ -1,7 +1,4 @@ -// A gated poll and a poll that simply found nobody to move both produce zero -// candidates and no attempt row, so an operator watching a stalled rollout -// cannot tell them apart. One aggregated line a minute per director names the -// gate and prices the selection, at a rate a 50-polls-a-minute worker can afford. +// Zero candidates/attempts hides gated vs idle polls; one aggregate line/min prices selection and names the gate without logging every ~50/min poll. export type RegionalRehomePollGate = | 'open' | 'cohort-zero' diff --git a/cloud/apps/relay/src/regional-rehome-safety.test.ts b/cloud/apps/relay/src/regional-rehome-safety.test.ts index 38e6fa34d30..86b0f1dadd0 100644 --- a/cloud/apps/relay/src/regional-rehome-safety.test.ts +++ b/cloud/apps/relay/src/regional-rehome-safety.test.ts @@ -27,8 +27,7 @@ function safety(overrides: Partial { it('passes the measured healthy-fleet baseline of pool micro-waits', () => { - // Every production cell idles at 1-2 peak waiters resolved in ~1ms; a - // zero-tolerance bar here disables the worker on its first tick. + // Zero tolerance rejects measured healthy-fleet micro-waits (1–2 waiters, ~1ms). expect( regionalRehomeSafetyFailure( safety({ databasePoolWaitersMax: 2, databasePoolWaitMsMax: 1 }), @@ -45,9 +44,7 @@ describe('regionalRehomeSafetyFailure', () => { }) it('passes asia-scale pool pressure, which excludes a cell rather than the fleet', () => { - // Measured 2026-09-16 on the asia-east2 cells: a client pool too narrow for - // a 176ms round trip, with 0.2ms server-side execution. The fleet snapshot - // is a Math.max, so gating on it here stops every region. + // Asia’s 176ms RTT caused ~2000ms pool waits despite 0.2ms SQL; fleet Math.max must not gate all regions on that cell. expect( regionalRehomeSafetyFailure( safety({ databasePoolWaitersMax: 150, databasePoolWaitMsMax: 2_005 }), @@ -55,7 +52,6 @@ describe('regionalRehomeSafetyFailure', () => { 19 ) ).toBeNull() - // Every other bar still fails closed at that same pool pressure. for (const [overrides, reason] of [ [{ sqlFailures: REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1 }, 'sql_failures'], [{ controlActivityRecoveryFailures: 1 }, 'control_recovery_failures'], @@ -90,8 +86,7 @@ describe('regionalRehomeSafetyFailure', () => { }) it('passes ambient 55P03 retry noise', () => { - // Fleet-wide retried lock timeouts peaked at 82 counted failures per - // minute over Aug 25-28; the combined snapshot can span two windows. + // Ambient retries peaked at 82/min; combined snapshots can span two windows. expect(regionalRehomeSafetyFailure(safety({ sqlFailures: 164 }), NOW, 19)).toBeNull() }) @@ -103,12 +98,10 @@ describe('regionalRehomeSafetyFailure', () => { 19 ) ).toBe('sql_failures') - // Literal storm magnitude (measured 2026-08-28) pins the bar itself: the - // limit must sit below real storm scale, not merely exist. + // Measured storm magnitude pins the threshold below real storms. expect(regionalRehomeSafetyFailure(safety({ sqlFailures: 395 }), NOW, 19)).toBe( 'sql_failures' ) - // The bar is exclusive: exactly at the limit still passes. expect( regionalRehomeSafetyFailure( safety({ sqlFailures: REGIONAL_REHOME_SQL_FAILURES_LIMIT }), diff --git a/cloud/apps/relay/src/regional-rehome-safety.ts b/cloud/apps/relay/src/regional-rehome-safety.ts index 8caf2b26e05..500fdd17a20 100644 --- a/cloud/apps/relay/src/regional-rehome-safety.ts +++ b/cloud/apps/relay/src/regional-rehome-safety.ts @@ -1,39 +1,17 @@ import type { RegionalRehomeSafetySnapshot } from './relay-observability.js' -// Limits sit well above the healthy-fleet baseline measured in production on -// 2026-08-28 (peak 2 waiters / 1ms pool waits on every cell; up to ~80 -// reconnects per published two-window row on the busiest cell). +// Healthy-fleet peaks: 2 waiters/~1ms pool wait; ~80 reconnects per two-window row (2026-08-28). export const REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT = 250 -// Pool pressure gates one cell, never the fleet. The combined snapshot is a -// Math.max, so a single narrow client pool stops rehoming everywhere: measured -// 2026-09-16, the asia-east2 cells sit at 94-156 waiters and ~2000ms waits -// against 0.2ms server-side execution -- a pool too narrow for a 176ms round -// trip, not database distress -- while us-central1 cells breach in bursts on -// ~33% of polls, and a bar that flaps between the pre-check and the commit -// re-check latches the worker off. regionalRehomeCellSafetyIsClean excludes a -// breaching cell as both source and target; the bars below stay fleet-wide -// because sql failures, control-recovery failures and reconnect storms mean -// database-wide distress. Instantaneous databasePoolWaiting is not checked -// separately: databasePoolWaitersMax bounds it within every published window. +// Gate pool pressure per cell: fleet Math.max would stop every region on one narrow Asia pool (176ms RTT, ~2000ms waits, 0.2ms SQL). +// Candidate checks exclude it as source/target; instantaneous waiters are already bounded by the window maximum. export const REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT = 16 export const REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT = 250 -// Counted SQL failures are dominated by relay_cells 55P03 lock-timeout -// retries the transaction wrapper heals in place (Aug 25-28: ~1000 retried -// attempts per day vs ~30 exhausted, those clustered in one storm), so a -// zero bar disabled the worker on ambient noise just like the original pool -// bars. Fleet-wide ambient noise peaked at 82 counted failures per minute -// over four days; genuine database distress produced 395-457. The combined -// snapshot spans up to two 30s windows per process (pathological ambient -// alignment ~164), so 250 stays clear of noise while storms still trip. -// Terminal outages also trip the per-cell pool bars and the worker's own -// dispatch-failure budget; the sql bar only needs to catch storms. +// Ambient retried 55P03s peak at 82/min (~164 across two windows), versus 395–457 in storms; 250 separates them. +// Pool bars and dispatch-failure budgets cover terminal outages; this bar catches storms. export const REGIONAL_REHOME_SQL_FAILURES_LIMIT = 250 -// Per-cell candidate cleanliness is a soft skip, not a durable latch; the -// worst ambient per-cell publish carried ~24 counted failures (two windows -// of 12). The fleet bar deliberately dominates: cells that are individually -// clean can sum past 250, and a fleet-wide sum at that scale is a storm. +// 40 exceeds ambient per-cell ~24/two windows; soft skips stay below the durable fleet storm bar of 250. export const REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT = 40 export function regionalRehomePoolPressure(safety: { @@ -65,10 +43,7 @@ export function regionalRehomeSafetyFailure( return null } -// The shared database stalls for 5-7 s about every 610 s, and one stall keeps -// the published sum over the sql bar for up to ~90 s (two 30 s windows plus -// heartbeat lag): 396 on 2026-10-06 latched rehome off fleet-wide. A breach -// therefore pauses claims, and only one that outlasts a stall latches. +// Routine 5–7s DB stalls leave SQL sums elevated ~90s through two windows and heartbeat lag; pause claims, latch only sustained breaches. export const REGIONAL_REHOME_SQL_FAILURES_SUSTAIN_MS = 150_000 // The director polls every ~6 s; a longer silence means the breach was not watched. export const REGIONAL_REHOME_SQL_FAILURES_OBSERVATION_GAP_MS = 45_000 diff --git a/cloud/apps/relay/src/regional-rehome-source-region-filter.test.ts b/cloud/apps/relay/src/regional-rehome-source-region-filter.test.ts index 3cc18c4802d..0bc5c00e149 100644 --- a/cloud/apps/relay/src/regional-rehome-source-region-filter.test.ts +++ b/cloud/apps/relay/src/regional-rehome-source-region-filter.test.ts @@ -81,7 +81,6 @@ async function setup(directorRegion: Region | undefined) { }) } } - // Homes the host in `home` with a conclusive measurement preferring the other region. const placeHost = async (identity: { userId: string; relayHostId: string }, home: Region) => { const assignment = await store.assign(identity, undefined, home) const homeCell = cells.find(({ id }) => id === assignment.cellId)! @@ -151,7 +150,6 @@ describe('rehome source-region filter', () => { it('keeps US to Asia moves and drops Asia to US moves', async () => { const context = await setup('us-central1') const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety()) - // Asia is filtered as a source yet is still every US host's target. expect(new Set(moves(candidates).map((move) => JSON.stringify(move)))).toEqual( new Set([JSON.stringify({ userId: usHost.userId, from: 'us-central1', to: 'asia-east2' })]) ) diff --git a/cloud/apps/relay/src/regional-rehome-store.test.ts b/cloud/apps/relay/src/regional-rehome-store.test.ts index c514ce7f2cc..2abcd721ff4 100644 --- a/cloud/apps/relay/src/regional-rehome-store.test.ts +++ b/cloud/apps/relay/src/regional-rehome-store.test.ts @@ -37,8 +37,7 @@ const target = { connectionHardCap: 1_000 as const, connectionUnobservedBound: 60 } -// A third general cell in the source region: never a source or target here, -// but it is in the fleet whose safety the gate reads. +// Bystander participates only in fleet safety. const bystander = { ...source, id: 'us-c2', url: 'https://us-c2.relay.example.test' } const sourceIncarnation = '11111111-1111-4111-8111-111111111111' const targetIncarnation = '22222222-2222-4222-8222-222222222222' @@ -321,8 +320,6 @@ describe('regional rehome assignment state', () => { }) it('counts only drainable cells as the rehome fleet, in every region', async () => { - // The fleet whose health gates a rehome is exactly the cells that can be a - // source or a target, and both roles require the drain protocol. const context = await setup({ targetProtocol: 0 }) expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({ @@ -430,7 +427,6 @@ describe('regional rehome assignment state', () => { const warnings = collectDisableWarnings() try { - // Per-cell, so it excludes this target rather than stopping the poll. expect(await context.store.commitIdleRegionalRehome(candidate!, safety)).toEqual({ outcome: 'deferred', reason: 'candidate-ineligible' @@ -438,8 +434,7 @@ describe('regional rehome assignment state', () => { } finally { warnings.restore() } - // A pool bar crossed between the scan and the commit skips the cell; it must - // not turn the durable switch off, or the worker never comes back. + // A scan-to-commit pool breach must skip without permanently disabling the worker. expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ generation: 1, enabled: true @@ -459,8 +454,7 @@ describe('regional rehome assignment state', () => { }) it('keeps selecting candidates while an unrelated cell is over the pool bar', async () => { - // Production case: the fleet snapshot is a Math.max, so one cell with a - // narrow client pool used to empty every page. + // Fleet Math.max once let one narrow client pool empty every page. const context = await setup() await activatePreferredSource(context, { userId: 'user-1', @@ -546,8 +540,7 @@ describe('regional rehome assignment state', () => { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }) - // A disabled cell is not a target, and the scan must say so: leaving it to - // the claim would burn a slot of the candidate batch on a certain skip. + // Exclude certain skips during selection or they consume candidate-page slots. await context.database.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [ target.id ]) @@ -596,10 +589,8 @@ describe('regional rehome assignment state', () => { const context = await setup({ hostCooldownMs: 3 * 24 * 60 * 60_000 }) const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const targetControl = await completeRehomeToTarget(context, identity) - // Past the dispatch interval the earlier claim charged, so the next tick - // really does scan and the cooldown is the only thing holding this host. + // Pass dispatch interval so only cooldown prevents another claim. context.advance(10_000) - // The desktop's region probe now says us-central1 again. await activateReversePreferredSource(context, identity) const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped') @@ -665,9 +656,7 @@ describe('regional rehome assignment state', () => { }) it('does not scan a candidate whose preferred region has no drainable cell', async () => { - // A cell without the drain protocol cannot be a target: the host would land - // where no later rehome could move it out again. The candidate query drops - // it, so the tick stays idle instead of paying for an inventory scan. + // Without drain protocol the target could never rehome hosts back out; exclude before inventory scans. const context = await setup({ targetProtocol: 0 }) await activatePreferredSource(context, { userId: 'user-1', @@ -698,8 +687,7 @@ describe('regional rehome assignment state', () => { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' }) - // Above the per-cell cleanliness bar but below the fleet storm bar: the - // candidate is skipped this tick while the worker stays enabled. + // Per-cell bar can skip a candidate while fleet safety keeps the worker enabled. await context.database.query( `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1} WHERE cell_id = ?`, [target.id] @@ -711,7 +699,6 @@ describe('regional rehome assignment state', () => { enabled: true }) - // Read-only selection does not spend the commit rate budget. expect( await context.database.query(`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`) ).toEqual([{ next_dispatch_at: 0 }]) @@ -921,13 +908,7 @@ describe('regional rehome assignment state', () => { await context.database.close() }) - // Why: the redrain lane reaches the inventory through the fleet-safety read - // rather than through candidate selection, so it needs its own coverage. - // Why: one contended candidate must cost its own tick, not the whole page. The - // sweeps are explicitly per-candidate isolated for exactly this reason. - // Why: with `continue` replaced by `break` a single contended candidate drops - // the rest of the page. Two in a row prove the sweep resumes, not just that it - // survived one, and that the summary counts both. + // Two contended candidates expose an erroneous break; the redrain lane reaches inventory through its separate fleet-safety read. it('completes a candidate behind two contended ones', async () => { const probe = new CellInventoryLockProbe() const context = await setup({ wrap: (database) => probe.wrap(database) }) @@ -975,8 +956,7 @@ describe('regional rehome assignment state', () => { await context.database.close() }) - // Why: only inventory contention is ordinary. Every other failure must keep its - // existing propagation and its dispatch-failure accounting. + // Only inventory contention is ordinary; other failures must propagate and count against dispatch. it('propagates a claim failure that is not inventory contention', async () => { const probe = new CellInventoryLockProbe() const context = await setup({ wrap: (database) => probe.wrap(database) }) @@ -995,8 +975,7 @@ describe('regional rehome assignment state', () => { await context.database.close() }) - // Why: the transaction dies at the first contended candidate, so every - // candidate behind it is abandoned too. Reporting one would understate the tick. + // One inventory failure abandons the transaction’s whole candidate page, so count all candidates skipped. it('skips a completion tick on a contended cell inventory without quarantining it', async () => { const probe = new CellInventoryLockProbe() @@ -1044,9 +1023,7 @@ describe('regional rehome assignment state', () => { await context.database.close() }) - // Why: a contended inventory is another director settling the same row, not a - // poisoned candidate. Quarantining on it would exclude a healthy attempt from - // the sweep's LIMIT pages for 15 minutes. + // Inventory contention is temporary; quarantining it would hide healthy attempts from LIMIT pages for 15min. it('skips an abort tick on a contended cell inventory without quarantining it', async () => { const probe = new CellInventoryLockProbe() const context = await setup({ wrap: (database) => probe.wrap(database) }) @@ -1114,7 +1091,6 @@ describe('regional rehome assignment state', () => { context.advance(1) await freshHeartbeats(context) expect(await context.store.abortUnarrivedRegionalRehomes()).toBe(1) - // Where the host was, so its next reconnect lands on the source it left. expect(await context.store.resolve(identity)).toMatchObject({ cellId: source.id, assignmentEpoch: 3 @@ -1147,7 +1123,6 @@ describe('regional rehome assignment state', () => { 'SELECT abort_reason FROM relay_region_rehome_attempts' ) expect(attempt!.abort_reason).toBe('host_not_arrived') - // What the rollout tracker reads to see a leak before it fills the budget. const preview = await context.store.previewRegionalRehomeEligibility() expect(preview.abortedLast24Hours).toEqual({ host_not_arrived: 1 }) expect( @@ -1161,8 +1136,7 @@ describe('regional rehome assignment state', () => { const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const sourceControl = await activatePreferredSource(context, identity) expect(await context.store.tryIdleRehome()).not.toBeNull() - // Past the window, but present at the target: the sweep reads live - // ownership, not the attempt's age alone. + // Age alone cannot abort an attempt whose target already owns the host. context.advance(REGIONAL_REHOME_ARRIVAL_WINDOW_MS) await freshHeartbeats(context) await context.store.activateControl(identity, { @@ -1312,14 +1286,11 @@ describe('regional rehome assignment state', () => { expect(await context.store.completeReadyRegionalRehomes()).toBe(0) } expect(warnings.entries).toHaveLength(REGIONAL_REHOME_QUARANTINE_FAILURES) - // Quarantined: the poisoned row leaves the candidate page entirely. expect(await context.store.completeReadyRegionalRehomes()).toBe(0) expect(warnings.entries).toHaveLength(REGIONAL_REHOME_QUARANTINE_FAILURES) - // After the quarantine window it is retried (and fails) once more. context.advance(REGIONAL_REHOME_QUARANTINE_MS + 1) await context.store.completeReadyRegionalRehomes() expect(warnings.entries).toHaveLength(REGIONAL_REHOME_QUARANTINE_FAILURES + 1) - // A free-form error (never a slug) reaches the log only as 'redacted'. expect( warnings.entries.every((entry) => entry.reason === 'regional_rehome_assignment_mismatch') ).toBe(true) @@ -1620,8 +1591,7 @@ describe('regional rehome assignment state', () => { assignmentEpoch: attempt!.assignmentEpoch }) await context.store.releaseActivity(identity, sourceControl) - // A vanished migration lease is repairable arithmetic on the first assert - // but still wrong on the re-assert: no repaired event may leak out. + // A vanished migration lease repairs on first assert but fails re-assert; no repaired event may escape rollback. await context.database.query( `DELETE FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'migration'`, @@ -1893,8 +1863,7 @@ async function activatePreferredSource( return control } -// Runs a hook inside the claim transaction, right after the candidate scan, so -// a scan-versus-claim race is deterministic instead of timing-dependent. +// Hook after scan inside the claim transaction makes scan/claim races deterministic. function hookAfterCandidateScan( database: RelayDatabase, hook: (transaction: RelayDatabase) => Promise diff --git a/cloud/apps/relay/src/regional-rehome-target-row-lock-postgres.test.ts b/cloud/apps/relay/src/regional-rehome-target-row-lock-postgres.test.ts index 6e3e4eaa4ba..d9a1829cd07 100644 --- a/cloud/apps/relay/src/regional-rehome-target-row-lock-postgres.test.ts +++ b/cloud/apps/relay/src/regional-rehome-target-row-lock-postgres.test.ts @@ -18,8 +18,7 @@ const describePostgres = databaseUrl ? describe : describe.skip // Roughly the round trip from an Asia cell to the us-central1 database. const STATEMENT_DELAY_MS = 150 -// Everything the fleet-wide rehome lock used to hold, per cell, plus the -// admission and region tables the commit reads. +// Probe the formerly fleet-locked tables plus admission/region tables read by commit. const CELL_TABLES = [ 'relay_cells', 'relay_cell_runtime', @@ -177,7 +176,6 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => { control.enabled = false expect(result).toEqual({ outcome: 'committed' }) - // The transaction really spans many cross-region round trips. expect(trips.length).toBeGreaterThanOrEqual(15) expect(trips[trips.length - 1]!.sql).toBe('COMMIT') for (const trip of trips) { @@ -186,7 +184,6 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => { expect({ sql: trip.sql, key, free }).toEqual({ sql: trip.sql, key, free: true }) } } - // Locked at exactly one trip boundary, the one before COMMIT. for (const key of TARGET_LOCKED) { expect(trips.filter((trip) => !trip.lockable[key]).map((trip) => trip.sql)).toEqual([ 'COMMIT' @@ -263,8 +260,7 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => { await expectNothingCommitted(context) }) - // A placement that reserves on the target between the candidate read and the target-row - // statement: modelled by raising the target's enforced units right before that statement. + // Inject a placement race by raising enforced target units immediately before the final target-row write. async function fillTargetBefore( context: Awaited>, freeSeats: number @@ -339,8 +335,7 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => { sequence++ let now = 1_000_000 const suffix = String(sequence) - // An Asia source correcting to a US target: the direction whose commit - // held the whole inventory for ~3.6s in production. + // Asia→US formerly held the whole inventory for ~3.6s in production. const source = cell(suffix, 'source', 'asia-east2') const target = cell(suffix, 'target', 'us-central1') const bystander = cell(suffix, 'zbystander', 'asia-east2') diff --git a/cloud/apps/relay/src/regional-rehome-worker.test.ts b/cloud/apps/relay/src/regional-rehome-worker.test.ts index 451ae524be9..38f0824b3fe 100644 --- a/cloud/apps/relay/src/regional-rehome-worker.test.ts +++ b/cloud/apps/relay/src/regional-rehome-worker.test.ts @@ -178,7 +178,6 @@ describe('regional rehome worker', () => { }) }) -// Answers each POST with the next scripted body, repeating the last one. function respondWith(bodies: { outcome: string; reason?: string }[]) { let index = 0 return vi.fn(async () => { @@ -192,8 +191,7 @@ function respondWith(bodies: { outcome: string; reason?: string }[]) { function collectSummaries() { const entries: Record[] = [] let arrived: (() => void) | undefined - // The worker polls once the moment it is constructed, so the poll under test - // is that one; `first` is how a test waits for it rather than for a tick. + // Construction immediately polls; await first instead of waiting for a timer tick. const first = new Promise((resolve) => { arrived = resolve }) diff --git a/cloud/apps/relay/src/regional-rehome-worker.ts b/cloud/apps/relay/src/regional-rehome-worker.ts index 5bf426e11b3..4e911ca046e 100644 --- a/cloud/apps/relay/src/regional-rehome-worker.ts +++ b/cloud/apps/relay/src/regional-rehome-worker.ts @@ -84,9 +84,7 @@ export function startRegionalRehomeWorker( stoppedBy = 'committed' break } - // Every remaining candidate would re-read the same durable row and - // answer the same way, so the rest of this page is wasted POSTs. - // A source on an older image sends no reason and keeps the old walk. + // Global deferrals repeat the same durable verdict for every candidate; older sources omit reason and retain the full walk. if (body.outcome === 'deferred' && isGlobalIdleRegionalRehomeDeferral(body.reason)) { stoppedBy = body.reason break @@ -102,8 +100,7 @@ export function startRegionalRehomeWorker( ) } } - // One line per poll that dispatched: silence used to be the only signal - // that 100+ candidates all came back deferred. + // Summarize dispatched polls so all-deferred pages remain visible. console.warn( JSON.stringify({ event: 'orca_relay_idle_rehome_dispatch_summary', diff --git a/cloud/apps/relay/src/relay-background-operation.ts b/cloud/apps/relay/src/relay-background-operation.ts index adf704106cd..caa917fac38 100644 --- a/cloud/apps/relay/src/relay-background-operation.ts +++ b/cloud/apps/relay/src/relay-background-operation.ts @@ -2,8 +2,7 @@ import { describeRelayFailure } from './relay-error-redaction.js' type Warn = (message: string) => void -// A swallowed error hides which sweep step is failing (a silently dead -// cleanup chain stalled production rehoming for hours). +// Log failed sweep steps; swallowed cleanup failures have stalled production rehoming for hours. export async function runRelayBackgroundOperation( operation: () => Promise, failureMessage: string, diff --git a/cloud/apps/relay/src/relay-connection-ledger.test.ts b/cloud/apps/relay/src/relay-connection-ledger.test.ts index d04a80bf665..0bb676bfc2d 100644 --- a/cloud/apps/relay/src/relay-connection-ledger.test.ts +++ b/cloud/apps/relay/src/relay-connection-ledger.test.ts @@ -175,12 +175,10 @@ describe('RelayConnectionLedger', () => { const first = ledger.tryReserveBooking(2)! expect(ledger.tryReserveBooking(2)).not.toBeNull() expect(ledger.tryReserveBooking(2)).toBeNull() - // A host control may still rise to the socket ceiling past the bookings. expect(ledger.tryReserveControl(false)).not.toBeNull() first.release() first.release() expect(ledger.counts().reservedConnectionUnits).toBe(1) - // Never above the socket ceiling, whatever the caller's ceiling says. expect(ledger.canReserveBooking(100)).toBe(true) ledger.tryReserveControl(false) ledger.tryReserveControl(false) diff --git a/cloud/apps/relay/src/relay-connection-ledger.ts b/cloud/apps/relay/src/relay-connection-ledger.ts index 8ffc6356b94..4ebdab38fa9 100644 --- a/cloud/apps/relay/src/relay-connection-ledger.ts +++ b/cloud/apps/relay/src/relay-connection-ledger.ts @@ -146,8 +146,7 @@ export class RelayConnectionLedger { return this.reserveUpgrade(this.normalAdmissionLimit) } - // A step-5 booking holds one reserved unit, but only below the placement ceiling the - // director would use, so phones keep the headroom between it and the socket ceiling. + // Bookings stop at the placement ceiling to preserve phone headroom below the socket ceiling. tryReserveBooking(limit: number): { release(): void } | null { if (!this.canReserveBooking(limit)) return null this.advanceWatermark() diff --git a/cloud/apps/relay/src/relay-control-flag-channel.test.ts b/cloud/apps/relay/src/relay-control-flag-channel.test.ts index 1245bd1f4fe..5d123bf3d17 100644 --- a/cloud/apps/relay/src/relay-control-flag-channel.test.ts +++ b/cloud/apps/relay/src/relay-control-flag-channel.test.ts @@ -92,7 +92,6 @@ describe('control flag channel', () => { }) }) - // How a flip is observed: one applied line per generation, and the seat feed's flagsApplied. it('flips a flag on and back off, logging each applied generation once', async () => { const google = fakeGoogle(cellObject(5, { readinessLocal: true, ticketCheck: 'shadow' })) const channel = cellChannel(google.fetchImpl) @@ -193,7 +192,6 @@ describe('control flag channel', () => { google.state.object = object await channel.poll() expect(channel.applied().generation).toBe(5) - // A voided generation is not re-read until it changes. await channel.poll() expect(google.state.storageRequests.at(-1)?.searchParams.get('ifGenerationNotMatch')).toBe( String(generation) diff --git a/cloud/apps/relay/src/relay-control-flag-channel.ts b/cloud/apps/relay/src/relay-control-flag-channel.ts index 827d5b6ee3f..1c2ac775844 100644 --- a/cloud/apps/relay/src/relay-control-flag-channel.ts +++ b/cloud/apps/relay/src/relay-control-flag-channel.ts @@ -1,17 +1,11 @@ -// Runtime switches in one Cloud Storage object per cell (cells/.json) in the -// `-relay-control` bucket. The bucket name comes from the metadata server, so the -// image needs no env or startup-script change. -// -// Rules: boot runs on defaults until the first good read; losing contact (404, 5xx, -// timeout, malformed body) keeps the applied values; a generation below the applied one is -// ignored; the parser voids a whole object on any bad known value. +// Per-cell runtime flags live in cells/.json in the metadata-derived -relay-control bucket. +// Boot uses defaults; failed reads retain applied values, older generations are ignored, and any invalid known value voids the object. const METADATA_ROOT = 'http://metadata.google.internal/computeMetadata/v1' const STORAGE_ROOT = 'https://storage.googleapis.com/storage/v1' const REQUEST_TIMEOUT_MS = 3_000 const MAX_OBJECT_BYTES = 64 * 1024 export const CONTROL_FLAG_POLL_MS = 5_000 -// Refresh a cached access token this long before Google says it expires. const ACCESS_TOKEN_EARLY_REFRESH_MS = 60_000 // `ignoredKeys` are names this image does not know: a newer writer's, or a typo. @@ -35,7 +29,6 @@ export function startControlFlagChannel(input: { fetch?: typeof fetch random?: () => number pollMs?: number - // Tests drive poll() directly. autoStart?: boolean }): ControlFlagChannel { const fetchImpl = input.fetch ?? fetch @@ -134,8 +127,7 @@ export function startControlFlagChannel(input: { } catch { throw new ControlFlagPollError('malformed') } - // Only a body read whole is "seen": a read cut off mid-body must be asked for again, or - // every later poll would answer 304 and strand that write. + // Mark seen only after a complete read or later 304s strand a truncated write forever. seenGeneration = generation const parsed = input.parse(body) if (parsed === null) throw new ControlFlagPollError('void') @@ -188,7 +180,6 @@ export function startControlFlagChannel(input: { } } -// Stops reading, and cancels the stream, at the first byte past the bound. async function readBounded(response: Response, maxBytes: number): Promise { if (!response.body) return '' const reader = response.body.getReader() diff --git a/cloud/apps/relay/src/relay-database-rejection-fence.test.ts b/cloud/apps/relay/src/relay-database-rejection-fence.test.ts index 28ee785d15e..23beeaa15db 100644 --- a/cloud/apps/relay/src/relay-database-rejection-fence.test.ts +++ b/cloud/apps/relay/src/relay-database-rejection-fence.test.ts @@ -96,8 +96,7 @@ describe('database rejection fence', () => { expect(() => handleRelayUnhandledRejection(duplicate, true)).not.toThrow() }) - // A whole-VM stall of 6-7 s outlasts the 5 s idle-in-transaction limit for any backend caught - // idle mid-transaction, and the server ends that session with FATAL 25P03 on resume. + // A 6–7s VM stall exceeds the 5s idle-transaction deadline, ending idle backends with FATAL 25P03. it('fences an idle-in-transaction timeout but keeps the rest of class 25 fatal', async () => { const idleTimeout = await rejectionFrom(() => databaseFailingWith( diff --git a/cloud/apps/relay/src/relay-database-rejection-fence.ts b/cloud/apps/relay/src/relay-database-rejection-fence.ts index 2a745d713c0..c1ecf911262 100644 --- a/cloud/apps/relay/src/relay-database-rejection-fence.ts +++ b/cloud/apps/relay/src/relay-database-rejection-fence.ts @@ -4,14 +4,8 @@ import { isPostgresPoolConnectFailure } from './postgres-pool-pressure.js' -// A database fault reaching unhandledRejection must not kill the cell: on 09-28 one failover -// crashed 18 cells and dropped ~16k hosts. A rejection is fenced only when both hold: -// path: the relay's database layer threw or rethrew it (marked below), and -// type: a pool acquire failure, pg's read timeout, a dropped connection, or a server error -// in class 08/40/53/55/57/58 (the database is unwell) or 23 (a constraint lost a race -// with a concurrent writer; the row it guards is intact, so the process is too). -// Everything else stays fatal: schema errors (42), data errors (22), TypeErrors from pg's -// parameter serialisation, and every bare socket errno from ws or fetch. +// Fence only database-layer errors with recognized outage/race types; one failover otherwise killed 18 cells/~16k hosts. +// Schema/data/programming errors and unmarked ws/fetch socket errors remain fatal. const databaseLayerErrors = new WeakSet() // Set by pg's query_timeout timer (pg@8.22 client.js). @@ -20,9 +14,7 @@ const CONNECTION_TERMINATED_PREFIX = 'Connection terminated' // pg@8.22 rejects the active query with the raw socket error when the connection drops. const DROPPED_SOCKET_ERRNOS = new Set(['ECONNRESET', 'EPIPE']) const FENCED_SQLSTATE_CLASSES = new Set(['08', '23', '40', '53', '55', '57', '58']) -// Class 25 is otherwise a code error (a statement in the wrong transaction state). 25P03 is the -// server ending a session left idle in a transaction past 5 s, which a whole-VM stall of 6-7 s -// does to any backend caught idle mid-transaction at its onset. +// 25P03 ends idle transactions after 5s, including during 6–7s VM stalls; other class-25 errors are code faults. const FENCED_SQLSTATES = new Set(['25P03']) export function isPostgresReadTimeout(error: unknown): error is Error { @@ -65,9 +57,7 @@ function rejectionFields(reason: unknown): { code: string; message: string } { return { code, message: message.replace(/[^\x20-\x7e]/g, '').slice(0, 120) } } -// Installed once by the process entry point. Fatal rejections are rethrown, which ends the -// process exactly as Node's default would, after one line that names them. -// `fenced` false is the per-cell switch turned off: every rejection is fatal, as before #26817. +// Entry point installs once; rethrow unfenced failures for Node’s default fatal behavior. Switch off restores that behavior for all rejections. export function handleRelayUnhandledRejection(reason: unknown, fenced = true): void { if (fenced && isFencedRelayDatabaseRejection(reason)) { console.warn( diff --git a/cloud/apps/relay/src/relay-error-redaction.ts b/cloud/apps/relay/src/relay-error-redaction.ts index f67642a0400..53184d9962c 100644 --- a/cloud/apps/relay/src/relay-error-redaction.ts +++ b/cloud/apps/relay/src/relay-error-redaction.ts @@ -1,6 +1,4 @@ -// Raw error messages can embed secrets such as connection strings. Only two -// provably inert shapes leave the process: this codebase's snake_case -// invariant slugs, and five-character SQLSTATE codes; everything else is redacted. +// Raw messages may contain secrets; emit only invariant slugs or SQLSTATE codes and redact everything else. export function relayErrorSlug(error: unknown): string | null { const message = error instanceof Error ? error.message : '' return /^[a-z0-9_]{1,64}$/.test(message) ? message : null diff --git a/cloud/apps/relay/src/relay-first-frame-close.blackbox.test.ts b/cloud/apps/relay/src/relay-first-frame-close.blackbox.test.ts index 8a3d797cdfb..e47e9bb8eb4 100644 --- a/cloud/apps/relay/src/relay-first-frame-close.blackbox.test.ts +++ b/cloud/apps/relay/src/relay-first-frame-close.blackbox.test.ts @@ -128,8 +128,7 @@ function relayAuthFrame(): Buffer { ) } -// The close frame is the last unmasked frame a rejected peer receives: 0x88, length, -// then a big-endian status code followed by the UTF-8 reason. +// Rejected peer’s last unmasked frame: 0x88, length, big-endian status, then UTF-8 reason. function readCloseFrame(received: Buffer): { code: number; reason: string } | null { const start = received.lastIndexOf(0x88) if (start < 0 || received.length < start + 4) return null @@ -210,14 +209,7 @@ it('releases admission after a director move redirect when the peer ignores clos expect(relay.connectionSnapshot()?.enforcedConnectionUnits).toBe(2) }) -// A peer that keeps draining its socket does get the rejection: the close frame is written -// before the force-close timer can fire, and TCP delivers those bytes ahead of the FIN. -// -// Scope, deliberately narrow: this peer reads every byte as it arrives, so the assertion below -// speaks only for a responsive peer. It is not evidence that delivery survives backpressure — -// `terminate()` destroys the socket a second later, and a frame still queued in the kernel or in -// `ws`'s own buffer goes unsent. Treat the rejection as best effort; the bound on the close is -// what the trade-off actually buys. +// This responsive peer proves close bytes precede FIN, not delivery under backpressure: terminate can drop queued frames after 1s. it('delivers the rejection code and a graceful FIN to a peer that keeps reading', async () => { const { relay, port } = await fixture() const peer = await silentUpgrade(port, PHONE_TARGET) @@ -233,7 +225,6 @@ it('delivers the rejection code and a graceful FIN to a peer that keeps reading' }) const BINARY_FIRST_FRAME = Buffer.from([0x82, 0x80, 0, 0, 0, 0]) -// maxPreAuthPerSource caps how many unauthenticated sockets one source may hold at once. const CONCURRENT_PEERS_PER_SOURCE = 4 it('returns capacity to its exact baseline after repeated bursts of rejections', async () => { @@ -253,8 +244,7 @@ it('returns capacity to its exact baseline after repeated bursts of rejections', } }) -// A rejection racing the peer's own disconnect must release once, not twice: the ledger -// does not clamp at zero, so a double release shows up as a negative count here. +// Ledger does not clamp at zero, so a disconnect racing rejection exposes double release as a negative count. it('releases exactly once when a rejected peer disconnects at the same moment', async () => { const { relay, port } = await fixture({ hardCap: 2 * CONCURRENT_PEERS_PER_SOURCE + 1 }) const peers = await Promise.all( diff --git a/cloud/apps/relay/src/relay-fix-level.ts b/cloud/apps/relay/src/relay-fix-level.ts index d29b70dd8a7..040c1eed39d 100644 --- a/cloud/apps/relay/src/relay-fix-level.ts +++ b/cloud/apps/relay/src/relay-fix-level.ts @@ -1,6 +1,3 @@ -// Bump by one in any change that fixes a cell crash or a cell safety bug. Every runtime -// metrics line carries it, and an alert pages on a serving cell that reports a level below -// the fixed floor `relay_cell_min_fix_level` (production.tfvars) for 6 hours; raise the floor -// once every serving cell runs the new level. Images from before this constant report nothing, -// which the alert reads as below. +// Bump for cell crash/safety fixes; runtime metrics carry the level and page after 6h below relay_cell_min_fix_level (missing also counts below). +// Raise the production.tfvars floor only after every serving cell upgrades. export const RELAY_FIX_LEVEL = 2 diff --git a/cloud/apps/relay/src/relay-host-hello-shed.test.ts b/cloud/apps/relay/src/relay-host-hello-shed.test.ts index f5e116f5c9c..db0dc09a56a 100644 --- a/cloud/apps/relay/src/relay-host-hello-shed.test.ts +++ b/cloud/apps/relay/src/relay-host-hello-shed.test.ts @@ -74,7 +74,6 @@ describe('host hello shedding under database pool pressure', () => { vi.restoreAllMocks() }) - // A cell whose pool holds `waiters` stuck queries, and a clock the test moves. async function startCell(waiters: number) { vi.spyOn(console, 'log').mockImplementation(() => undefined) const keys = await generateKeyPair('ES256') @@ -129,7 +128,6 @@ describe('host hello shedding under database pool pressure', () => { } } - // Resolves with the refusal's status, or 101 once the upgrade is accepted. async function dial( cell: Awaited> ): Promise<{ status: number; retryAfter?: string }> { @@ -170,8 +168,7 @@ describe('host hello shedding under database pool pressure', () => { cell.advance(HOST_HELLO_SHED_OLDEST_WAIT_MS) expect(await dial(cell)).toEqual({ status: 503, retryAfter: '2' }) - // Shipped desktops dial with no unexpected-response listener, so the refusal - // reaches them as the ordinary connect error their retry backoff handles. + // Shipped desktops lack unexpected-response listeners; refusal must reach their ordinary connect-error backoff. const desktop = new WebSocket(cell.url, { headers: { authorization: `Bearer ${cell.token}` } }) diff --git a/cloud/apps/relay/src/relay-host-log-digest.ts b/cloud/apps/relay/src/relay-host-log-digest.ts index b232170a6d4..9c42ac6d3b3 100644 --- a/cloud/apps/relay/src/relay-host-log-digest.ts +++ b/cloud/apps/relay/src/relay-host-log-digest.ts @@ -1,7 +1,6 @@ import { createHash } from 'node:crypto' -// Store-level 503s were previously silent; the digest keeps per-host log -// correlation possible without emitting the raw relay host id. +// Correlate per-host 503s without logging raw host IDs. export function relayHostLogDigest(relayHostId: string): string { return createHash('sha256').update(relayHostId).digest('hex').slice(0, 12) } diff --git a/cloud/apps/relay/src/relay-host-proof-failure.blackbox.test.ts b/cloud/apps/relay/src/relay-host-proof-failure.blackbox.test.ts index 924523685e3..8b583fc0b9f 100644 --- a/cloud/apps/relay/src/relay-host-proof-failure.blackbox.test.ts +++ b/cloud/apps/relay/src/relay-host-proof-failure.blackbox.test.ts @@ -27,9 +27,7 @@ describe('relay host proof failures', () => { vi.restoreAllMocks() }) - // The production crash signature: a pg-pool connect timeout rejecting out of - // verifyCellAssignment inside beginProof killed whole cells as an unhandled - // rejection. The guard must contain it to this one handshake. + // A verifyCellAssignment connect-timeout rejection once escaped beginProof and crashed whole cells. it('contains a database timeout during host hello to one socket', async () => { const keys = await generateKeyPair('ES256') const publicJwk = await exportJWK(keys.publicKey) diff --git a/cloud/apps/relay/src/relay-local-readiness.test.ts b/cloud/apps/relay/src/relay-local-readiness.test.ts index e3a35266cc1..6dd91449e01 100644 --- a/cloud/apps/relay/src/relay-local-readiness.test.ts +++ b/cloud/apps/relay/src/relay-local-readiness.test.ts @@ -61,7 +61,7 @@ describe('local readiness verdict', () => { }) }) -// E-dep-3 in miniature: the database is down from boot, so it never enters a grace window. +// Boot-down database never earns a grace window (E-dep-3). describe('cell /ready with the database down', () => { const servers: Server[] = [] afterEach(async () => { diff --git a/cloud/apps/relay/src/relay-local-readiness.ts b/cloud/apps/relay/src/relay-local-readiness.ts index 02e8635d84f..9d8ae339f64 100644 --- a/cloud/apps/relay/src/relay-local-readiness.ts +++ b/cloud/apps/relay/src/relay-local-readiness.ts @@ -4,10 +4,7 @@ export type RelayLocalReadinessVerdict = | { ready: true; failing: RelayReadinessDependency[] } | { ready: false; reason: 'not_listening' | 'keys_not_loaded' } -// `readinessLocal`: the load balancer keeps a cell that can still serve its connected hosts and -// verify new tokens from cached keys, so a database or auth outage costs a per-hello refusal -// instead of the whole hostname. The heartbeat keeps the SQL verdict, so directors still stop -// placing here. +// Keep cells serving connected hosts/cached-key auth through dependency outages; SQL heartbeats still stop director placement. export function createRelayLocalReadiness(input: { listening: () => boolean // The host token verifier's own key set: a fetch by any other client proves nothing about it. diff --git a/cloud/apps/relay/src/relay-observability.test.ts b/cloud/apps/relay/src/relay-observability.test.ts index dfdab9031c4..564113f3ecf 100644 --- a/cloud/apps/relay/src/relay-observability.test.ts +++ b/cloud/apps/relay/src/relay-observability.test.ts @@ -27,10 +27,7 @@ const counts: RelayProcessCounts = { databasePoolWaitMsMax: 1_250 } -// Two schema keys legitimately spell a policed word: the abandoned-accept bucket -// is keyed by stage name and one stage is `credential`. Rename those exact keys in -// a clone instead of rewriting the JSON, so a stray raw field or value anywhere -// else still trips the guard below. +// Alias only legitimate credential schema keys in a clone so raw secret fields/values elsewhere still trip the guard. const SCHEMA_KEY_ALIASES: Record = { clientAcceptCredentialMsP95: 'clientAcceptStageTwoMsP95' } @@ -244,7 +241,6 @@ describe('relay observability', () => { for (let index = 0; index < 40; index++) { observability.recordAssignmentUnavailable('drain-return-deferred') } - // A drained reserve host's intake pace is scheduled by the drain too. observability.recordAssignmentUnavailable('reserve-paced-drained') observability.recordAssignmentUnavailable('sticky-lane') observability.recordAssignmentUnavailable('relay_capacity_exhausted') @@ -615,7 +611,6 @@ describe('relay observability', () => { controlRttMsP95: 240, controlRttMsMax: 240 }) - // Only-add: the pre-existing fields still read the same after the extension. expect(entries[0]).toMatchObject({ event: 'orca_relay_runtime_metrics', metricVersion: 2, @@ -623,8 +618,7 @@ describe('relay observability', () => { clientAcceptAbandonedMsMax: 0 }) expect(entries[0]).toMatchObject({ clientAcceptsByPhonePathDelta: {} }) - // An empty window publishes counts only: a zero percentile point is - // indistinguishable from a real zero once Cloud Logging aggregates it. + // Omit empty percentiles: aggregated zeros are indistinguishable from actual zero-latency samples. expect(entries[1]).toMatchObject({ clientAcceptCompletedDelta: 0, controlRttSamplesDelta: 0 }) for (const omitted of [ 'clientAcceptTotalMsP50', @@ -657,12 +651,10 @@ describe('relay observability', () => { } observability.flush(counts) - // Dropped is observed minus retained, so this pins the retained window at the cap. expect(entries[0]).toMatchObject({ controlRttSamplesDelta: flooded, controlRttSamplesDroppedDelta: flooded - CONTROL_RTT_RESERVOIR_LIMIT }) - // The kept samples are real observations, not a truncated or synthesised window. expect(entries[0]!.controlRttMsP50 as number).toBeGreaterThanOrEqual(10) expect(entries[0]!.controlRttMsMax as number).toBeLessThanOrEqual(49) @@ -765,7 +757,6 @@ describe('relay observability', () => { } } }) - // The window resets, and an empty window publishes counts only. expect(entries[1]!.preAuthLatencyMs).toEqual({ beforeCheck: none, db: none, reserve: none }) }) diff --git a/cloud/apps/relay/src/relay-observability.ts b/cloud/apps/relay/src/relay-observability.ts index 7fafffdb04b..d040d1faf33 100644 --- a/cloud/apps/relay/src/relay-observability.ts +++ b/cloud/apps/relay/src/relay-observability.ts @@ -53,8 +53,7 @@ export type AssignmentAdmissionOutcome = export type AssignmentAdmissionLane = 'sticky' | 'placement' | 'drain-return' -// Why every /v1/assign 503 has a cause: drain-return deferrals are scheduled -// 503s, so a gate on all 503s trips on every fast drain. +// Separate scheduled drain 503s or every fast drain trips availability gates. export type AssignmentUnavailableCause = | 'disabled' | 'sticky-lane' @@ -119,9 +118,7 @@ export interface RelayRuntimeObserver { recordHostPreAuth?(path: HostPreAuthPath, outcome: HostPreAuthOutcome, durationMs: number): void } -// Which check decided a host hello: the cell's memory in reserve mode (a booking, a seat it -// remembers, or a lease), the database assignment check, or neither (beforeCheck: draining, -// idle cutover, a bad or late hello, the pool shed, or a check that threw). +// Hello decision source: reserve memory, database assignment, or beforeCheck for pre-check refusals/errors. export type HostPreAuthPath = 'beforeCheck' | 'db' | 'reserve' // Abandoned: the host hung up before a decision, or before the cell could act on one. export type HostPreAuthOutcome = 'admitted' | 'refused' | 'abandoned' @@ -163,9 +160,7 @@ const emptyPreAuthLatency = (): PreAuthLatency => { // Which serialized accept step the phone had already hung up behind. export type RelayClientAcceptStage = 'assignment' | 'credential' | 'activity' -// The attach window and the basis writes that follow it are only measurable once -// the host data leg lands, so they join the serialized pre-attach steps on -// completed accepts only. +// Attach/basis durations join pre-attach steps only once the host data leg completes acceptance. export type RelayClientAcceptTimedStage = RelayClientAcceptStage | 'attach' | 'basis' export const RELAY_CLIENT_ACCEPT_TIMED_STAGES = [ @@ -251,13 +246,11 @@ type StickyConnectionStatements = { samplesMs: number[] } -// A window sees each pooled connection the sticky lane used; a pool that keeps -// replacing connections must not grow the line without bound. +// Cap per-window connection tracking because pool replacements can otherwise grow log lines without bound. export const STICKY_CONNECTION_LIMIT = 64 export const STICKY_STATEMENT_RESERVOIR_LIMIT = 1024 -// A host chooses how often it answers a ping, so the process-wide window is a -// reservoir: the heap cost of a flood is capped and the percentiles stay unbiased. +// Host-driven pong frequency needs a bounded reservoir to cap flood memory and keep unbiased percentiles. export const CONTROL_RTT_RESERVOIR_LIMIT = 1024 // Per path and outcome; a reconnect burst can put thousands of hellos in one window. export const PRE_AUTH_RESERVOIR_LIMIT = 1024 @@ -365,17 +358,13 @@ function roundMs(value: number): number { return Number(value.toFixed(3)) } -// Spreading a window into Math.max blows the stack once a busy cell samples -// enough of it, so the maximum is folded instead. The fold is also not -// interchangeable with the sorted last element: it is seeded with zero, so an -// all-negative or NaN window reads differently. +// Fold maximum to avoid spreading unbounded samples onto the stack; zero seed also preserves negative/NaN behavior. function latencySummary(samples: number[]): { p50: number p95: number p99: number max: number } { - // One sorted copy serves every rank. const sorted = samples.length === 0 ? samples : ascending(samples) return { p50: roundMs(nearestRank(sorted, 0.5)), @@ -714,8 +703,7 @@ export class RelayObservability implements RelayRuntimeObserver { drainReturnRejectionsByReasonDelta: deltas.drainReturnRejectionsByReason, drainReturnRetryAfterSecondsMax: deltas.drainReturnRetryAfterSecondsMax, drainReturnRetryAfterSecondsSum: deltas.drainReturnRetryAfterSecondsSum, - // Slot hold times; a lane serves concurrency / service time per second. - // Omitted when empty for the same reason as the accept percentiles below. + // Slot hold time determines throughput; omit empty percentiles like accepts below. ...(deltas.drainReturnServiceMs.length === 0 ? {} : { @@ -733,8 +721,7 @@ export class RelayObservability implements RelayRuntimeObserver { assignNonDrain503sDelta: Object.entries(deltas.assign503sByCause) .filter(([cause]) => !DRAIN_503_CAUSES.has(cause)) .reduce((sum, [, count]) => sum + count, 0), - // Waiters summed over samples; divided by samples it is the mean number of - // backends waiting, i.e. lock-wait seconds per second. + // Waiters summed over samples / sample count is mean waiting backends (lock-wait seconds per second). dbLockWaitSamplesDelta: deltas.dbLockWaitSamples, dbLockWaitersByKeyDelta: deltas.dbLockWaitersByKey, dbCellRowLockWaitersDirectorDelta: lockWaiters(deltas.dbLockWaitersByKey, 'director'), @@ -759,8 +746,7 @@ export class RelayObservability implements RelayRuntimeObserver { phoneAcceptDatabaseErrorsDelta: deltas.phoneAcceptDatabaseErrors, resumeConfirmFailuresByCodeDelta: deltas.resumeConfirmFailuresByCode, resumeRenewalFailuresDelta: deltas.resumeRenewalFailures, - // Accepts are sparse: publishing a zero percentile for every empty window - // would pin the p50 at 0 forever and collapse the p95 at low accept rates. + // Omit empty accept percentiles or sparse windows pin p50 to zero and depress p95. ...(deltas.clientAcceptTotalsMs.length === 0 ? {} : { @@ -773,8 +759,7 @@ export class RelayObservability implements RelayRuntimeObserver { clientAcceptAttachMsP95: acceptStageP95('attach'), clientAcceptBasisMsP95: acceptStageP95('basis') }), - // Every round trip observed in the window, including the ones the reservoir - // above declined to keep; the percentiles summarise only what it kept. + // Count all observed RTTs, including reservoir drops; percentiles describe retained samples only. controlRttSamplesDelta: deltas.controlRttObserved, controlRttSamplesDroppedDelta: deltas.controlRttObserved - deltas.controlRttSamplesMs.length, ...(deltas.controlRttSamplesMs.length === 0 @@ -796,9 +781,7 @@ export class RelayObservability implements RelayRuntimeObserver { controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures, hostHellosShedDelta: deltas.hostHellosShed, preAuthLatencyMs: preAuthLatencyFields(deltas.preAuthLatency), - // Meaning changed when renewals began batching: for a batched row this is - // the flush's duration, not that row's own statement latency. The - // per-flush fields below are the ones to read for statement cost. + // Batched row latency is flush duration; use per-flush fields for statement cost. controlRenewalLatencyMsP50: controlRenewal.p50, controlRenewalLatencyMsP95: controlRenewal.p95, controlRenewalLatencyMsMax: controlRenewal.max, @@ -815,10 +798,8 @@ export class RelayObservability implements RelayRuntimeObserver { } } -// Flat siblings of the nested region maps, always emitted for every region including zeros. -// A log-based metric cannot reach `requestedRegionsDelta."asia-east2"` without a quoted field -// path, and an absent key would drop a series out of the inner join the region-skew alert does. -// The maps stay authoritative and keep carrying anything outside the catalog, such as `unhinted`. +// Emit flat per-region siblings including zeros: quoted nested keys resist log metrics, and absent series disappear from skew joins. +// Nested maps remain authoritative for uncatalogued values such as unhinted. function regionCounterFields( prefix: 'requestedRegion' | 'selectedRegion', counts: Record @@ -831,8 +812,7 @@ function regionCounterFields( ) } -// Count and buckets cover every hello in the window, so buckets sum exactly across windows and -// cells; the percentiles summarise the reservoir and are omitted when empty. +// Counts/buckets include every hello and sum across cells/windows; percentiles use the reservoir and omit empty windows. function preAuthLatencyFields( latency: PreAuthLatency ): Record>> { diff --git a/cloud/apps/relay/src/relay-readiness.ts b/cloud/apps/relay/src/relay-readiness.ts index e4945113d78..be233668c1c 100644 --- a/cloud/apps/relay/src/relay-readiness.ts +++ b/cloud/apps/relay/src/relay-readiness.ts @@ -35,13 +35,10 @@ export type RelayReadinessProbe = { failingDependencies: () => RelayReadinessDependency[] } -// The token verifier caches keys in process, so a cell keeps verifying tokens right through a JWKS -// outage: the only thing an unreachable JWKS endpoint blocks is a key rotation nobody is running. +// Cached verifier keys survive JWKS outages; failed fetches prevent key rotation, not verification. export const RELAY_READINESS_JWKS_GRACE_MS = 900_000 -// Each cell is its own load balancer backend, so failing readiness never re-routes a host, it only -// makes that hostname unreachable. A host that lands on a SQL-dead cell gets WRONG_CELL and is -// re-placed by the director, and existing control sockets survive a generic Postgres error. Three -// minutes rides a Cloud SQL failover without hiding a per-cell fault for a quarter of an hour. +// Failing a cell’s readiness loses its hostname rather than rerouting hosts; 3min rides failover without masking longer cell faults. +// SQL-dead hellos can get WRONG_CELL and director replacement; generic DB errors preserve existing controls. export const RELAY_READINESS_SQL_GRACE_MS = 180_000 export const RELAY_MAX_READINESS_GRACE_MS = 3_600_000 @@ -76,8 +73,7 @@ function graceTransition( return failure === undefined ? 'recovered' : 'expired' } -// One dependency's own last-known-good clock; collapsing the two would let a healthy JWKS poll keep -// a dead Postgres inside its window forever. +// Separate dependency clocks or healthy JWKS polls keep dead PostgreSQL in grace forever. function createDependencyGrace(dependency: RelayReadinessDependency, graceMs: number) { let lastSuccessAt: number | undefined let inGrace = false diff --git a/cloud/apps/relay/src/relay-region-app.test.ts b/cloud/apps/relay/src/relay-region-app.test.ts index 6161bcf5a1c..eccd916da70 100644 --- a/cloud/apps/relay/src/relay-region-app.test.ts +++ b/cloud/apps/relay/src/relay-region-app.test.ts @@ -230,7 +230,6 @@ describe('Relay region API', () => { expect(fromAssign.status).toBe(503) expect(fromAssign.headers.get('Retry-After')).toBe('2') expect(causes).toEqual(['reserve-paced-drained', 'reserve-paced-drained']) - // Any other host's pace keeps today's cause. plan.mockRejectedValue(new ReservePaceError(1)) await app.request('/v1/assign', assignmentRequest('abcdefghijklmnop', {})) expect(causes.at(-1)).toBe('reserve-paced') diff --git a/cloud/apps/relay/src/relay-region-placement.test.ts b/cloud/apps/relay/src/relay-region-placement.test.ts index 502119ae28b..de576b66a43 100644 --- a/cloud/apps/relay/src/relay-region-placement.test.ts +++ b/cloud/apps/relay/src/relay-region-placement.test.ts @@ -208,7 +208,6 @@ describe('Relay regional placement', () => { const place = vi.fn(async () => onReserve) await expect(assignAsia(place)).resolves.toEqual({ ...onReserve, placed: true }) expect(place).toHaveBeenCalledOnce() - // The refused in-region attempt wrote nothing. expect(await database.query(`SELECT cell_id FROM relay_assignments`)).toEqual([]) }) @@ -230,7 +229,6 @@ describe('Relay regional placement', () => { region: 'us-central1' }) expect(place).toHaveBeenCalledOnce() - // No in-region reserve room at plan time: today's single spill. await expect( store.assign({ ...identity, relayHostId: 'spillhost0000002' }, 'asia-east2', 'asia-east2') ).resolves.toMatchObject({ cellId: 'us-c1' }) diff --git a/cloud/apps/relay/src/relay-region-selection-app.test.ts b/cloud/apps/relay/src/relay-region-selection-app.test.ts index 39a6d1ef63d..88ca8c1202b 100644 --- a/cloud/apps/relay/src/relay-region-selection-app.test.ts +++ b/cloud/apps/relay/src/relay-region-selection-app.test.ts @@ -81,7 +81,6 @@ describe('Relay region selection counters', () => { it('counts a placement out of the target region on any lane as a fallback', async () => { const recordRegionSelection = vi.fn() - // The reconnect verifies on the sticky lane; the store then re-places the host. await assign( { recordRegionSelection, @@ -145,7 +144,6 @@ describe('Relay region selection counters', () => { const placed = vi.fn(async () => usAssignment()) await assign({ recordRegionSelection: vi.fn(), plan, assign: placed }, { reconnect: true }, { regionalPlacementEnabled: false }) await assign({ recordRegionSelection: vi.fn(), plan, assign: placed }, { reconnect: true, preferredRegion: undefined }) - // Asking for the default region while regional placement is off is not a preference either. await assign( { recordRegionSelection: vi.fn(), plan, assign: placed }, { reconnect: true, preferredRegion: 'us-central1' }, @@ -162,7 +160,6 @@ describe('Relay region selection counters', () => { await assign( { recordRegionSelection, - // A report carries no preference, so its target is the default region. resolve: vi.fn(async () => usAssignment({ cellId: 'asia-c1', region: 'asia-east2' })), exchangeRegionCorrection: vi.fn(async () => ({ v: 1, reportStatus: 'accepted' })) }, diff --git a/cloud/apps/relay/src/relay-schema-catalog-precheck-postgres.test.ts b/cloud/apps/relay/src/relay-schema-catalog-precheck-postgres.test.ts index 0dd3f278bed..45ec44faa0d 100644 --- a/cloud/apps/relay/src/relay-schema-catalog-precheck-postgres.test.ts +++ b/cloud/apps/relay/src/relay-schema-catalog-precheck-postgres.test.ts @@ -13,16 +13,12 @@ import { type RelayDatabase } from './database.js' -// The outage this guards against: CREATE INDEX IF NOT EXISTS takes its relation lock before the -// server evaluates the existence test, so on a database that already has the index the boot still -// joins the lock queue, and relation locks are granted in queue order, so every writer queues -// behind it. Only a real server can show that the catalog pre-check removes those statements. +// IF NOT EXISTS acquires relation locks before checking existence, queueing writers behind boot DDL; prove catalog prechecks avoid it on a real server. const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip const schema = 'relay_schema_precheck_test' -// A table the boot would otherwise touch with two CREATE INDEX statements, and the largest table -// in production. +// Largest production table, otherwise touched by two boot CREATE INDEX statements. const LOCKED_TABLE = 'relay_control_connection_reservations' function scopedUrl(): string { @@ -56,8 +52,7 @@ describePostgres('relay boot-time schema against PostgreSQL', () => { await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`) await client.query(`CREATE SCHEMA ${schema}`) }) - // Same lock bound the boot pool uses, so a statement that still queues fails instead of - // hanging the test. + // Match boot’s lock timeout so any DDL still queued fails promptly. pool = new pg.Pool({ connectionString: url, max: 1, lock_timeout: POSTGRES_LOCK_TIMEOUT_MS }) sent = [] }) @@ -90,7 +85,6 @@ describePostgres('relay boot-time schema against PostgreSQL', () => { sent = [] const warm = await applyRecording() - // Zero, with no exceptions: every statement that takes a relation lock has a pre-check. expect(lockTaking()).toEqual([]) const preCheckedCount = relayPostgresSchemaStatements().filter( (statement) => schemaLockTarget(statement) !== undefined @@ -104,8 +98,7 @@ describePostgres('relay boot-time schema against PostgreSQL', () => { it('skips an index a failed concurrent build left invalid instead of rebuilding it', async () => { await applyRecording() - // A cancelled CREATE INDEX CONCURRENTLY leaves exactly this state, and IF NOT EXISTS skips it - // too, so reading indisvalid as a condition would newly take the lock it used to avoid. + // Cancelled concurrent indexes can be invalid; IF NOT EXISTS still skips them, so indisvalid must not newly trigger DDL. await pool.query( `UPDATE pg_catalog.pg_index SET indisvalid = false WHERE indexrelid = to_regclass('${schema}.relay_audit_events_at')` @@ -117,8 +110,7 @@ describePostgres('relay boot-time schema against PostgreSQL', () => { }) it('re-adds a constraint an operator dropped, then skips it again', async () => { - // The pre-check matches the constraint by name only, so this is the one shape where a changed - // body needs an operator: drop it, and the next boot puts the current definition back. + // Name-only constraint checks require operators to drop changed definitions before boot recreates them. await applyRecording() const named = async (): Promise => Number( @@ -148,9 +140,7 @@ describePostgres('relay boot-time schema against PostgreSQL', () => { }) it('does not let a same-named index on a sibling table answer for this one', async () => { - // Index names are unique per schema, not per table, so a name freed on one table and taken on - // another is reachable. Without tying the index to the table, the pre-check reads that sibling - // as this table's index and skips the real CREATE INDEX for good. + // Index names are schema-wide; bind name to table or a sibling’s index can permanently suppress the real CREATE. await applyRecording() const ask = async (table: string): Promise => ( @@ -171,8 +161,7 @@ describePostgres('relay boot-time schema against PostgreSQL', () => { }) it('boots while another session holds ACCESS EXCLUSIVE on the largest table', async () => { - // The end-to-end proof through openRelayDatabase: with the lock held, any DDL the boot still - // sent against this table would hit lock_timeout, and 55P03 is no longer retried. + // Exercise openRelayDatabase: leftover DDL against the held lock would fail with unretried 55P03. const cold = await openRelayDatabase({ databaseUrl: url, dataDir: '' }) opened.push(cold) await pool.end() @@ -232,9 +221,7 @@ describePostgres('relay boot-time schema against PostgreSQL', () => { }) it('defers the activity-lease migrations and still boots while their table is locked', async () => { - // The migration boot, reproduced: the index is there, the table is locked by someone else, and - // all the drop can do is time out. It has to leave the statement for the next boot rather than - // fail, or 28 directors crash-loop through a stall on a table written ~475/s. + // Migration DROP must defer behind writers or 28 directors crash-loop on a table written ~475/s. const cold = await openRelayDatabase({ databaseUrl: url, dataDir: '' }) opened.push(cold) // Put the database back in its pre-migration shape, which is what makes the drop lock-taking. @@ -267,7 +254,6 @@ describePostgres('relay boot-time schema against PostgreSQL', () => { warn.mockRestore() } - // Both statements deferred, and the boot still applied everything else. expect(summary.deferred).toBe(2) expect(summary.ran).toBeGreaterThan(0) const deferred = warned @@ -277,14 +263,12 @@ describePostgres('relay boot-time schema against PostgreSQL', () => { 'relay_assignment_activity_expiry', 'fillfactor=70' ]) - // Nothing was applied, so the next boot has the same work to do, not half of it. const stillThere = await pool.query( `SELECT 1 FROM pg_indexes WHERE schemaname = $1 AND indexname = $2`, [schema, 'relay_assignment_activity_expiry'] ) expect(stillThere.rowCount).toBe(1) - // And the next boot, with the lock gone, finishes the job. const retry = await applyPostgresSchema( relayPostgresSchemaStatements(), (statement) => pool.query(statement), diff --git a/cloud/apps/relay/src/relay-schema-lock-targets.test.ts b/cloud/apps/relay/src/relay-schema-lock-targets.test.ts index 9978eccd2e6..b1acbc646ba 100644 --- a/cloud/apps/relay/src/relay-schema-lock-targets.test.ts +++ b/cloud/apps/relay/src/relay-schema-lock-targets.test.ts @@ -13,17 +13,13 @@ function lockTakingStatements(): string[] { describe('relay boot-time lock targets', () => { it('derives a target for every CREATE INDEX and every ALTER TABLE ADD COLUMN', () => { - // A census over the real schema, not two hand-picked cases: a statement that lands here - // without a target is sent on every boot and takes the lock the pre-check exists to avoid. - // requireSchemaLockTarget is what boot calls, so this fails the same way boot would. + // Use boot’s classifier on every real statement; an unclassified one would lock tables on every warm boot. for (const statement of relayPostgresSchemaStatements()) { expect(() => requireSchemaLockTarget(statement)).not.toThrow() } }) it('pre-checks every lock-taking statement, with no exceptions', () => { - // The invariant the rule comment beside SCHEMA depends on: nothing that takes a relation lock - // reaches the server on a warm boot. A statement with no target breaks it. const unchecked = lockTakingStatements().filter( (statement) => schemaLockTarget(statement) === undefined ) @@ -32,8 +28,7 @@ describe('relay boot-time lock targets', () => { }) it('derives a target through the comment block a split schema glues on', () => { - // Not vacuous: SCHEMA really does carry a comment-prefixed statement, and it is a CREATE INDEX - // on relay_connection_bases. Classifying the raw text would give it no target at all. + // Real split-schema statements carry comment prefixes; raw-text classification would miss their CREATE INDEX. const commented = relayPostgresSchemaStatements().filter((statement) => statement.startsWith('--') ) @@ -51,8 +46,7 @@ describe('relay boot-time lock targets', () => { }) it('leaves the dollar-quoted statement-stats migration byte-identical', () => { - // Its body is a PL/pgSQL block full of commas and parentheses. Reading the tag as anything but - // opaque would change the text classification sees, and it is the only such statement relay has. + // Dollar-quoted PL/pgSQL commas/parentheses must remain opaque to classification. const doBlock = relayPostgresSchemaStatements().find((statement) => statement.startsWith('DO ')) expect(doBlock).toBeDefined() expect(sqlWithoutComments(doBlock!)).toBe(doBlock) diff --git a/cloud/apps/relay/src/relay-server.ts b/cloud/apps/relay/src/relay-server.ts index 6abac741f03..bec223e1770 100644 --- a/cloud/apps/relay/src/relay-server.ts +++ b/cloud/apps/relay/src/relay-server.ts @@ -60,8 +60,7 @@ import { } from './reserve-ledger-writer.js' import { ProcessQueuedByteBudget } from './splice-forwarder.js' -// A malformed percent-escape in the request target must be a client error, never a URIError -// thrown out of the `upgrade` listener (which is uncaught and kills the process). +// Malformed upgrade paths must return client errors; an uncaught listener URIError kills the process. function decodePathSegment(value: string): string | null { try { return decodeURIComponent(value) @@ -70,10 +69,7 @@ function decodePathSegment(value: string): string | null { } } -// Healthy cells peak at 2 waiters (p99, 2026-10-07). A head that has waited half -// the 2 s acquire timeout with a whole pool's worth queued behind it means -// hellos are already timing out; the count floor keeps one slow waiter from -// tripping it. On 10-07 every window that met both also logged SQL failures. +// Healthy cells peak at 2 waiters; a pool’s worth queued past half the 2s timeout means hellos already fail. Count floor ignores lone slow waiters. export const HOST_HELLO_SHED_OLDEST_WAIT_MS = 1_000 const HOST_HELLO_SHED_RETRY_AFTER_SECONDS = 2 // Well inside the flag channel's 5 s read, so a flip back starts re-registering within ~6 s. @@ -100,9 +96,7 @@ function noDelay(socket: WebSocket): void { transport?.setNoDelay(true) } -// Why: a ws receiver error (oversize or malformed frame) with no 'error' -// listener throws process-wide; ws itself already closes the socket after -// emitting it, so logging is all that is left to do. +// Handle ws receiver errors or they crash the process; ws already closes the socket. function guardSocketErrors(socket: WebSocket, kind: string): void { socket.on('error', (error) => { console.warn(`[orca-relay] ${kind} socket error: ${error.message}`) @@ -182,8 +176,7 @@ export function createRelayServer( observeGrace: (event) => observability.recordReadinessGrace(event) }) const ready = readiness.check - // Set once placement is built: a director configured `on` that could not build it (no map or - // no rehome credential) books nothing, so it must not keep a reserve cell's dead-man fed. + // Only actual booking activity may feed reserve dead-man; configured-on directors missing map/credentials book nothing. const placing = { on: false } const shadowSeatPoller = startShadowSeatPoller(config, { listCells: () => assignments.seatFeedCells(), @@ -193,7 +186,6 @@ export function createRelayServer( const shadowCompare = shadowSeatPoller ? new ShadowDirectoryCompare(shadowSeatPoller.directory, options.now) : undefined - // Step 5 on directors: needs the map and the rehome credential the cells verify. const directorId = randomUUID() const rehomeAudience = config.rehomeAudience // Only a director that books writes rows, and only for the joins it booked. @@ -201,7 +193,6 @@ export function createRelayServer( config.role === 'director' && shadowSeatPoller && config.reservePlacement === 'on' ? new ReserveLedgerWriter(observedDatabase, { now: options.now }) : undefined - // Step 1 C: the operator's rehome switch is read only on a director with C switched on. const rehomePercent = config.reserveRehomePercent ?? 0 const rehomeSwitch = config.role === 'director' && shadowSeatPoller && rehomeAudience && config.reservePlacement === 'on' && rehomePercent > 0 @@ -241,8 +232,7 @@ export function createRelayServer( }) : undefined placing.on = reservePlacement?.placementMode === 'on' - // A director that is not placing while a cell is in reserve mode strands that cell's hosts on - // its memory: its dead-man flips it back, and this says so loudly until then. + // Warn until dead-man flips back: non-placing directors otherwise strand reserve-cell hosts in memory. const offWithReserveTimer = config.role === 'director' && shadowSeatPoller && !placing.on ? setInterval(() => { @@ -337,7 +327,6 @@ export function createRelayServer( configuredConnectionLimits.hardCap, configuredConnectionLimits.controlReserve ) - // Step 5: a booking may fill the cell only to the ceiling the directors place against. const placementCeiling = config.connectionHardCap === undefined || config.connectionUnobservedBound === undefined ? null @@ -354,14 +343,12 @@ export function createRelayServer( options.now ) : null - // The hello shed's test: the pool is already timing queries out. const databaseShedding = (): boolean => readRelayDatabasePoolPressure(database).databasePoolWaiting >= config.databasePoolMax && readRelayDatabasePoolOldestWaitMs(database) >= HOST_HELLO_SHED_OLDEST_WAIT_MS const reserveDeadMan = reserveBook ? new ReserveDeadMan(config.cellId, options.now, reserveDeadManWindowMs(config.cellId)) : null - // The switch file's admitMode, unless the dead-man has flipped this cell back. const effectiveAdmitMode = (): 'db' | 'reserve' => { const applied = options.cellFlags?.() if (!applied) return 'db' @@ -402,8 +389,7 @@ export function createRelayServer( } : undefined ) - // Expired bookings give their units back even while no director is reserving. A flip out of - // reserve mode voids the bookings and registers every control admitted from memory. + // Expire booking units even with no reserving directors; leaving reserve also re-registers memory controls. let appliedAdmitMode = effectiveAdmitMode() const admitEffective = config.role === 'cell' && reserveBook @@ -461,8 +447,7 @@ export function createRelayServer( ? { cellReserve: (request: ReserveRequest) => sessions.reserve(request), cellReserverPoll: () => reserveDeadMan?.contact(), - // Reserve until every control it admitted from memory holds a lease again: the flag - // workflow records db in Postgres (and sweeps resume) only after this says db. + // Report reserve until every memory control has a lease; flag workflow must wait before recording db and resuming sweeps. cellAdmitModeRaw: effectiveAdmitMode, cellAdmitModeEffective: (): 'db' | 'reserve' => effectiveAdmitMode() === 'reserve' || sessions.reregistrationPending() > 0 @@ -825,23 +810,18 @@ export function createRelayServer( userId: identity.sub, relayHostId: identity.relayHostId }) - // Null outside reserve mode, so today's path is untouched there. const reserveAdmission = sessions.reserveAdmissionFor({ userId: identity.sub, relayHostId: identity.relayHostId }) - // A hello costs several pooled queries, each failing after a 2 s wait. Shed - // only while the pool is already timing them out: the refused desktop gets - // the same connect error and backoff a timed-out hello gives it today, 2 s - // sooner (shipped desktops ignore Retry-After). A rebind over a live control - // is a lease rotation, not a reconnect, so it is never refused here. + // Shed before several 2s pool waits; shipped desktops use ordinary connect-error backoff and ignore Retry-After. + // Preserve live-control rebinds: they rotate leases rather than reconnect. if (!isRebind && reserveAdmission === null && databaseShedding()) { observability.recordHostHelloShed() rejectUpgrade(socket, 503, 'Service Unavailable', HOST_HELLO_SHED_RETRY_AFTER_SECONDS) return } - // A booked host may rise to the hard cap. Once its upgrade holds a unit, the booking's - // goes back, so the host counts once; a refused upgrade leaves the booking whole. + // Booked upgrades can reach hard cap; return booking unit only after upgrade reserves its own, counting host once. const controlUpgrade = connectionLedger?.tryReserveControl(isRebind || reserveAdmission === 'booked') ?? null if (controlUpgrade && reserveAdmission === 'booked') { @@ -859,9 +839,7 @@ export function createRelayServer( ) return } - // Register the release before anything else can throw: the outer catch - // destroys the socket, so a close-registered release cannot leak the - // reserved connection unit. + // Register release before other work can throw; outer-catch socket destruction must return the connection unit. const releaseControlUpgrade = (): void => controlUpgrade?.release() socket.once('close', releaseControlUpgrade) observability.recordAuth(true) diff --git a/cloud/apps/relay/src/relay-sweep-schedule.ts b/cloud/apps/relay/src/relay-sweep-schedule.ts index f73e6b69ead..7d5918ad44a 100644 --- a/cloud/apps/relay/src/relay-sweep-schedule.ts +++ b/cloud/apps/relay/src/relay-sweep-schedule.ts @@ -1,7 +1,4 @@ -// Why: every director instance boots from the same rollout, so its periodic -// sweeps land on the same wall-clock second across instances and pile onto the -// one global cell-inventory lock together. A per-process offset spreads the -// arrivals; the sweeps are idempotent, so a slightly longer period is free. +// Same-rollout directors otherwise contend on inventory locks together; longer idempotent sweep periods spread arrivals without increasing load. export const SWEEP_JITTER_FRACTION = 0.2 export function jitteredSweepIntervalMs( diff --git a/cloud/apps/relay/src/relay-upgrade-malformed-uri.blackbox.test.ts b/cloud/apps/relay/src/relay-upgrade-malformed-uri.blackbox.test.ts index c2dce833d14..0b25ff62948 100644 --- a/cloud/apps/relay/src/relay-upgrade-malformed-uri.blackbox.test.ts +++ b/cloud/apps/relay/src/relay-upgrade-malformed-uri.blackbox.test.ts @@ -96,8 +96,7 @@ describe('relay upgrade with a malformed request target', () => { vi.spyOn(console, 'log').mockImplementation(() => undefined) vi.spyOn(console, 'warn').mockImplementation(() => undefined) - // Vitest installs its own uncaughtException listener; capture ours first so the test reports - // the exception as a verdict instead of dying with it. + // Capture uncaughtException before Vitest’s listener so crashes become test verdicts. const uncaught: unknown[] = [] const onUncaught = (error: unknown): void => { uncaught.push(error) @@ -116,13 +115,11 @@ describe('relay upgrade with a malformed request target', () => { ]) { results.push(await rawUpgrade(port, target)) } - // A malformed percent-escape must be a client error, never a process-level throw. expect(uncaught).toEqual([]) for (const result of results) { expect(result.status).toMatch(/^HTTP\/1\.1 4\d\d/) expect(result.closed).toBe(true) } - // The server must still serve a well-formed upgrade afterwards. const after = await rawUpgrade(port, '/v1/connect/abcdefghijklmnop') expect(after.status).toMatch(/^HTTP\/1\.1 101/) }) diff --git a/cloud/apps/relay/src/relay-websocket-close.ts b/cloud/apps/relay/src/relay-websocket-close.ts index 74554253885..2624ab1088f 100644 --- a/cloud/apps/relay/src/relay-websocket-close.ts +++ b/cloud/apps/relay/src/relay-websocket-close.ts @@ -3,14 +3,8 @@ import type WebSocket from 'ws' export const RELAY_WEBSOCKET_FORCE_CLOSE_MS = 1_000 const forceCloseTimers = new WeakMap>() -// Delivering the rejection is best effort, and deliberately so. The close frame carrying the -// code and reason is written before the timer can fire, so a peer reading normally gets its -// rejection ahead of the FIN — that much is asserted in relay-first-frame-close.blackbox.test.ts. -// It is not a delivery guarantee: `ws` writes the frame to the socket, and `terminate()` destroys -// the socket a second later, so under backpressure the frame (and any `relay-moved` message queued -// before it) can still be dropped unsent even though the peer never stopped reading. Bounding the -// close is what keeps a stalled peer from holding admission, and no finite grace makes delivery -// certain. Keep the close write ahead of any new wait added here. +// Write close before any wait so responsive peers receive rejection before FIN; the 1s termination bound frees admission. +// Backpressure can still drop queued close/relay-moved frames, even for reading peers: delivery is best effort. export function closeRelayWebSocket(socket: WebSocket, code: number, reason: string): void { if (socket.readyState === socket.CLOSED) return if (!forceCloseTimers.has(socket)) { diff --git a/cloud/apps/relay/src/relay.blackbox.test.ts b/cloud/apps/relay/src/relay.blackbox.test.ts index d2a7d950f31..92e667a8457 100644 --- a/cloud/apps/relay/src/relay.blackbox.test.ts +++ b/cloud/apps/relay/src/relay.blackbox.test.ts @@ -733,9 +733,7 @@ describe('served relay URL', () => { }) it('restates a pending connection to the rebound control, detailed only when advertised', async () => { - // The one link the unit tests cannot reach: an upgrade that really carries - // x-orca-host-capabilities must reach acceptControl and change the ack. A - // typo in the header name here passes every other test in the suite. + // Real upgrade must carry x-orca-host-capabilities into acceptControl; unit tests cannot catch a misspelled header here. const host = await openHostControl() const hostId = createHash('sha256') .update(host.keyPair.publicKey) @@ -785,8 +783,7 @@ describe('served relay URL', () => { controlResumeSecret: String(capable.ack.controlResumeSecret), previousGeneration: 1 }) - // A shipped host parses these entries strictly, so an unannounced key would - // fail the whole ack and kill a control that was working. + // Legacy hosts strictly decode entries; unadvertised fields would kill an otherwise healthy control. expect(legacy.ack.pendingConns).toEqual([ { connId: connection.connId, connTicket: connection.connTicket } ]) @@ -2191,8 +2188,7 @@ describe('served relay URL', () => { blockedExpiredUnregistered: 0, blockedExpiredOnNewerTargetAssignment: 0 }) - // Completion must prove the source has stopped admitting new work, even in - // the served admin workflow that performs its own topology preflight. + // Admin completion must prove source admission stopped despite its separate topology preflight. expect((await cellState('cell-a', false)).status).toBe(200) cellB = spawnTopologyRelay({ diff --git a/cloud/apps/relay/src/released-reservation-prune.test.ts b/cloud/apps/relay/src/released-reservation-prune.test.ts index e1cbf5922f2..2b10402ef00 100644 --- a/cloud/apps/relay/src/released-reservation-prune.test.ts +++ b/cloud/apps/relay/src/released-reservation-prune.test.ts @@ -11,8 +11,7 @@ import { } from './database.js' import { HeapWindowReaper } from './heap-window-reaper.js' -// Released reservations were never deleted: 13.7M rows and 9 GB in production, every one of them -// still locked by each placement of its host. These pin what the prune may take and how it walks. +// Unpruned released rows reached 13.7M/9GB and were locked on every host placement. const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip const schema = 'relay_released_reservation_prune_test' @@ -106,7 +105,7 @@ describe.each(dialects)('released reservation prune (%s)', (_dialect, open) => { describePostgres('heap window reaper against PostgreSQL', () => { let database: RelayDatabase - // 200 rows of this shape fill about two pages; 6,000 spread the table over ~60 pages. + // 6,000 rows span ~60 pages; 200 would exercise only ~2. const ROWS = 6_000 afterEach(async () => { diff --git a/cloud/apps/relay/src/reserve-assignment.test.ts b/cloud/apps/relay/src/reserve-assignment.test.ts index 69e8b4480d7..2472e1c2c7c 100644 --- a/cloud/apps/relay/src/reserve-assignment.test.ts +++ b/cloud/apps/relay/src/reserve-assignment.test.ts @@ -127,8 +127,7 @@ describe('reserve assignment on a director', () => { expect(await tripped.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({ kind: 'database' }) const leasing = setup({ feeds: [feed('c1', { admitMode: 'db', admitModeEffective: 'reserve' }), feed('c2')] }) expect(leasing.directory.admitModeOf('c1')).toBe('reserve') - // A tripped cell still re-registering reports reserve as effective but db as raw: placement - // treats it as a database cell at once, books nothing there and paces nobody for it. + // Raw db must immediately stop booking/pacing even while effective mode stays reserve during re-registration. const reregistering = setup({ feeds: [ feed('c1', { admitModeEffective: 'reserve', admitModeRaw: 'db' }), @@ -150,7 +149,6 @@ describe('reserve assignment on a director', () => { row: async (identity) => identity.relayHostId === onUnheard.relayHostId ? { cellId: 'c2', assignmentEpoch: 3 } : null }) - // A fresh host goes to today's path, with no booking yet. expect(await gated.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({ kind: 'database', epochFloor: 0 @@ -180,7 +178,6 @@ describe('reserve assignment on a director', () => { const { assignment, directory, reserved, now } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')] }) - // The cell restarted: its full snapshot no longer holds the seat. directory.apply('c1', feed('c1', { incarnation: 'c1-incarnation-2' }), now.value, now.value) const plan = await assignment.plan(HOST, { reconnect: true, region: US }) expect(reserved[0]).toEqual({ cellId: 'c1', epoch: 7, sticky: true, dryRun: false }) @@ -201,7 +198,6 @@ describe('reserve assignment on a director', () => { }) expect(reserved.length).toBeGreaterThan(0) expect(reserved.every((call) => !call.sticky && call.epoch === 9)).toBe(true) - // The floor lapses, so a host it no longer protects answers from memory again. now.value += RESERVE_ROW_FLOOR_TTL_MS + 1 for (const response of [feed('c1', { seq: 2, full: [seated(7)] }), feed('c2', { seq: 2 })]) { directory.apply(response.cellId, response, now.value, now.value) @@ -266,7 +262,6 @@ describe('reserve assignment on a director', () => { expect( await assignment.plan({ ...HOST, relayHostId: 'qrstuvwxyzabcdef' }, { reconnect: true, region: US }) ).toMatchObject({ kind: 'database', epochFloor: 5 }) - // A cell that rolled back to an image without the feed takes no placement. directory.markNoFeed('c1', now.value) expect(directory.cellState('c1')?.ceiling).toBeUndefined() expect(assignment.placementCells().find((cell) => cell.cellId === 'c1')?.general).toBe(false) @@ -317,7 +312,6 @@ describe('reserve assignment on a director', () => { assignment: { assignmentEpoch: 9 } }) expect(row).toHaveBeenCalledTimes(1) - // The database down: placed as before the read existed, above what the map knows. const down = leftWrongCell(async () => Promise.reject(new Error('timeout'))) expect(await down.assignment.plan(HOST, { reconnect: true, region: US })).toMatchObject({ kind: 'answer', @@ -331,7 +325,6 @@ describe('reserve assignment on a director', () => { assignment: { assignmentEpoch: 8 } }) expect(busyRow).not.toHaveBeenCalled() - // A stalled database: the read gives up after its bound and places as before. vi.useFakeTimers() try { const stalled = leftWrongCell(() => new Promise(() => undefined)) @@ -372,7 +365,6 @@ describe('reserve assignment on a director', () => { expect(reserved).toEqual([]) await expect(regionReserve?.()).resolves.toMatchObject({ cellId: 'c1', region: ASIA }) expect(reserved.map((call) => call.cellId)).toEqual(['c1']) - // A full reserve cell is not offered, and the US reserve cell never is. const full = feed('c1', { counts: { seats: 500, controls: 500, bookings: 0, units: 500, ceiling: 500 } }) directory.apply('c1', full, now.value, now.value) expect(await regionReserveOf(assignment, OTHER)).toBeUndefined() @@ -426,7 +418,6 @@ describe('reserve assignment on a director', () => { expect(plan).toEqual({ kind: 'database', reserveSpill: expect.any(Function) }) expect(row).not.toHaveBeenCalled() expect(reserved).toEqual([]) - // The database lane read the row: the spill mints above it. await expect( plan.kind === 'database' ? plan.reserveSpill?.({ skipOwnRegion: false, rowEpoch: 3 }) : null ).resolves.toMatchObject({ assignmentEpoch: 4 }) @@ -610,7 +601,6 @@ describe('reserve assignment on a director', () => { }) }) - // Jinwoo's condition: no move into a region without real room. Asia at 98% full gets none. it('moves no one into Asia at 98% full, or anywhere under 10% free', async () => { for (const share of [0.98, 0.91]) { const { assignment, reserved, warm } = setupRehome({ asia: asiaFull(share) }) @@ -622,14 +612,12 @@ describe('reserve assignment on a director', () => { }) expect(reserved.filter((call) => call.cellId === 'asia-r')).toEqual([]) } - // 10% free is the line: the host moves. const { assignment, warm } = setupRehome({ asia: asiaFull(0.9) }) await warm() expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' }) }) - // Room under 10% still takes placements (and B's spill, which books down to 0%), but C does - // not move anyone into it: the gap that keeps a spilled host from bouncing back. + // 10% rehome headroom above the 0% spill line prevents spilled hosts bouncing back. it('keeps a gap above the spill line: 5% free takes placements, not rehomes', async () => { const { assignment, warm } = setupRehome({ asia: asiaFull(0.95) }) await warm() @@ -652,7 +640,6 @@ describe('reserve assignment on a director', () => { await closed.warm() expect(await closed.assignment.plan(HOST, ask)).toMatchObject(sticky) const cold = setupRehome() - // No minute has passed: the US cell's move budget is still empty. expect(await cold.assignment.plan(HOST, ask)).toMatchObject(sticky) const sameRegion = setupRehome() await sameRegion.warm() @@ -662,9 +649,7 @@ describe('reserve assignment on a director', () => { it('never runs for a fresh ask, an unknown host, a database-mode source, or an incomplete map', async () => { const fresh = setupRehome() await fresh.warm() - // A fresh ask is placed as today (the share draw), not rehomed. expect(await fresh.assignment.plan(HOST, { ...ask, reconnect: false })).toMatchObject({ lane: 'placement' }) - // Never seen: placed fresh in its own region by the share draw, not rehomed. expect(await fresh.assignment.plan(OTHER, ask)).toMatchObject({ kind: 'answer', lane: 'placement' }) const onDatabase = setup({ cells: [...cellList(['us-db']), ...cellList(['asia-r'], ASIA)], @@ -698,7 +683,6 @@ describe('reserve assignment on a director', () => { }) await warm() expect(await assignment.plan(HOST, ask)).toMatchObject({ kind: 'answer', lane: 'sticky' }) - // The booking was asked for and refused on intake: the pace path ran. expect(reserved.map((call) => call.cellId)).toEqual(['asia-r']) }) @@ -741,7 +725,6 @@ describe('reserve assignment on a director', () => { expect(await assignment.plan(OTHER, ask)).toMatchObject({ lane: 'sticky' }) now.value += 60_000 await assignment.plan({ ...HOST, relayHostId: 'flushflushflush1' }, ask) - // The warm-up minute, then this one. expect(lines.filter((line) => line.event === 'orca_relay_reserve_rehome')).toEqual([ { event: 'orca_relay_reserve_rehome', fromRegion: US, toRegion: ASIA, count: 0, skippedByRule: { 'source-budget': 1 } }, { event: 'orca_relay_reserve_rehome', fromRegion: US, toRegion: ASIA, count: 1, skippedByRule: { 'source-budget': 1 } } @@ -760,7 +743,6 @@ describe('reserve assignment on a director', () => { it('tells the old reserve seat to go when it books the host elsewhere, naming that seat exactly', async () => { const { assignment, directory, demoted, now } = setup({ feeds: [feed('c1', { full: [seated(5, 1_000)] }), feed('c2')], - // c1 is full, so the host lands on c2. reserve: (cellId) => (cellId === 'c1' ? { outcome: 'full' } : { outcome: 'ok' }) }) // The host left c1 for a reason that moves it: its next placement is fresh, above 5. @@ -779,8 +761,7 @@ describe('reserve assignment on a director', () => { expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }]) }) - // The booking map had not seen the c1 seat (half-open, joined within a poll lag): the feed - // demotes it once both joins are in the map, instead of the 75 s silence close. + // A half-open seat missed by the booking map must be demoted by the feed, before its 75s silence close. it('tells an old reserve seat the booking missed to go once a newer join shows in the feed', () => { const { assignment, directory, demoted, now } = setup() directory.onOutranked = (seat) => assignment.supersedeOutranked(seat) @@ -794,7 +775,6 @@ describe('reserve assignment on a director', () => { const { assignment, reserved } = setup({ feeds: [feed('c1', { full: [seated(6, 1_000)] }), feed('c2', { full: [seated(5, 3_000)] })] }) - // c2's seat joined later but at an older epoch: a stale rebind, never the answer. expect(await assignment.plan(HOST, { reconnect: true, region: US })).toMatchObject({ kind: 'answer', lane: 'sticky', @@ -851,7 +831,6 @@ describe('reserve assignment on a director', () => { .map((line) => JSON.parse(line)) .filter((entry) => entry.event === 'orca_relay_reserve_demote_outcome') - // The map follows the cell's feed: a refusing cell repairs that feed itself. it('logs the answer by host digest and leaves the seat to the feed', async () => { const lines: string[] = [] const logged = Promise.withResolvers() @@ -1007,7 +986,6 @@ describe('reserve assignment on a director', () => { log: (line) => lines.push(line) }) await assignment.plan(HOST, { reconnect: true, region: US }) - // The host came back to the restarted c1, which drains again. directory.apply( 'c1', feed('c1', { incarnation: 'c1-incarnation-2', draining: true, full: [seated(20)] }), @@ -1034,7 +1012,6 @@ describe('reserve assignment on a director', () => { it('leaves every other host as today: no drain count, and a plain pace', async () => { const lines: string[] = [] const { assignment } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')], log: (line) => lines.push(line) }) - // A seat on another draining cell than the newest one does not count either. const paced = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2', { draining: true, full: [seated(6)] })], reserve: () => ({ outcome: 'intake' }) diff --git a/cloud/apps/relay/src/reserve-assignment.ts b/cloud/apps/relay/src/reserve-assignment.ts index 1d7f8605e7e..51b156afcaf 100644 --- a/cloud/apps/relay/src/reserve-assignment.ts +++ b/cloud/apps/relay/src/reserve-assignment.ts @@ -17,9 +17,7 @@ import { } from './reserve-placement.js' import type { OutrankedSeat, SeatFeedCell, ShadowSeat, ShadowSeatDirectory } from './shadow-seat-directory.js' -// Step 5 on a director: answer a desktop from the in-memory map for cells that report -// admitMode=reserve, and fall back to today's database path for everything else. The cell -// decides every booking; this only estimates, picks and asks. +// Director estimates/picks from its memory map, but the cell decides every booking; other hosts retain database placement. // A leaver may rejoin its cell from memory this long, as the cell's own rule 2 allows. export const RESERVE_STICKY_RECENT_MS = 10 * 60_000 @@ -61,8 +59,7 @@ export type ReservePlan = } | ({ kind: 'database' } & Pick) -// A host whose seat the ledger reconcile demoted mints above its row for this long, so the -// re-assign does not collide with the row again (the mirror refuses an equal epoch). +// Reconcile-demoted hosts must mint above their row: equal epochs cannot replace the guarded mirror. export const RESERVE_ROW_FLOOR_TTL_MS = 2 * 60 * 60_000 const RESERVE_ROW_FLOOR_MAX = 100_000 const OWN_BOOKINGS_MAX = 100_000 @@ -123,8 +120,7 @@ export class ReserveAssignment { client: CellReserveClient // One primary-key read of the host's row: only for a host the map has never seen. readRow: (identity: Identity) => Promise<{ cellId: string; assignmentEpoch: number } | null> - // True while the pool has waiters: the optional WRONG_CELL read is skipped, so timed-out - // reads cannot stack up behind a stall. + // Skip optional WRONG_CELL reads while pool waits, avoiding timed-out queries stacking behind a stall. databaseBusy?: () => boolean // Step 1 C: rehome on reconnect; absent or at 0% it never runs. rehome?: ReserveRehome @@ -210,22 +206,18 @@ export class ReserveAssignment { ...(rowFloor === undefined ? [] : [rowFloor]) ] let row: { cellId: string; assignmentEpoch: number } | null = null - // A seat closed with WRONG_CELL was demoted (or failed to re-register) because its row is - // ahead, which every director sees in that cell's feed: read the row, so the re-assign mints - // above it on any director, not just the one that demoted it. + // Read the row after WRONG_CELL so every director mints above it, including directors that only saw the leave. const leftWrongCell = left?.closeCode === RELAY_CLOSE_CODE.WRONG_CELL && !(this.input.databaseBusy?.() ?? false) if ((known.length === 0 && !request.regionCorrection) || leftWrongCell) { try { - // Only the never-seen host needs the row; for the WRONG_CELL one it is an optimisation, - // bounded so a database stall cannot hold the reconnect (the reconcile repairs a miss). + // WRONG_CELL read is bounded/optional: known hosts can recover via reconcile; never-seen hosts require the row. row = known.length === 0 ? await this.input.readRow(identity) : await withinMs(this.input.readRow(identity), RESERVE_WRONG_CELL_ROW_READ_MS) } catch { - // A host the map never saw, with the database down: its epoch cannot be minted safely. - // One the map knows places as it did before this read existed. + // Without row or map history the epoch cannot be minted safely; known hosts retain placement during DB outages. if (known.length === 0) { this.summary.retries.database += 1 return { kind: 'retry', retryAfterSeconds: 2, reason: 'database' } @@ -241,8 +233,7 @@ export class ReserveAssignment { return { kind: 'answer', assignment: rehomed, lane: 'rehome' } } } - // The database lane answers it, so the window exchange follows: no memory answer, no share - // draw, only the in-region fallback, once the map is complete and knows the host. + // Region-window exchange belongs to the database lane; bypass memory/share draws and offer only complete-map in-region fallback. if (request.regionCorrection) { if (incomplete) { return { kind: 'database' } @@ -279,8 +270,7 @@ export class ReserveAssignment { const reserveSpill = incomplete ? undefined : this.reserveSpill(identity, request.region, epochFloor, seats, drainedFrom) - // Where the host was last seen decides who owns it: a database-mode cell keeps today's - // path, sticky included; only a fresh placement may be booked on a reserve-mode cell. + // Last-seen cell determines ownership: database cells retain sticky placement; reserve cells book only fresh placements. const lastOnReserve = lastCellId !== undefined && directory.placementModeOf(lastCellId) === 'reserve' if (!lastOnReserve && (seats.length > 0 || left !== undefined || row !== null)) { return { kind: 'database', epochFloor, placeFresh, regionReserve, reserveSpill } @@ -297,9 +287,7 @@ export class ReserveAssignment { return !this.input.directory.isComplete() && now - this.input.startedAt < RESERVE_STARTUP_GATE_MS } - // Keyed on the cell, not the leave: a same-cap drain (graceMs 0) has the host re-ask before - // its leave reaches the map, but the cell's feed says draining from the first poll after. - // Only the newest grant counts, so a host already re-booked elsewhere is not a drain re-ask. + // Key drain re-asks by newest grant’s cell: zero-grace hosts redial before leave reaches the map, while feed already says draining. private drainSource( identity: Identity, seats: readonly ShadowSeat[], @@ -317,9 +305,7 @@ export class ReserveAssignment { return drained ? newest.cellId : undefined } - // Only the reserve lane's own answers: a database-lane placement off the drained (isolated) - // cell already logs orca_relay_sticky_replaced_off_isolated_cell, and the canary sums both. - // Each host counts once per drain: the canary bar must never pass on a host counted twice. + // Count reserve answers once per host/drain; database replacements log separately and the canary sums both, so duplicates inflate success. private recordDrainReplaced(identity: Identity, fromCellId: string, placed: RelayAssignment): void { const incarnation = this.input.directory.cellState(fromCellId)?.incarnation let counted = this.drainCounted.get(fromCellId) @@ -385,8 +371,7 @@ export class ReserveAssignment { } } - // For the share draw's losers: the database lane tries the region first, then this, then spills. - // Undefined while no in-region reserve cell has a seat, so the database lane runs as today. + // Share-draw losers try database in-region, then reserve fallback, then spill; absent reserve room retains database behavior. private regionReserve( identity: Identity, region: RelayRegion, @@ -406,8 +391,7 @@ export class ReserveAssignment { } } - // For a database lane that found no cell with room anywhere: each region with a reserve seat, - // the host's own first, without the share draw. A pace is the answer only if no region places. + // After database exhausts all cells, try reserve regions own-first without share draw; pace only if none places. private reserveSpill( identity: Identity, region: RelayRegion, @@ -457,9 +441,7 @@ export class ReserveAssignment { this.summary.spills[key] = (this.summary.spills[key] ?? 0) + 1 } - // What stickyFromMemory may answer, in its order: only the newest grant (highest epoch, then - // the newest join), on a cell that is a candidate in its own region. Lazy, so each cell is - // judged after the previous candidate's sticky reserve settled. + // Newest epoch/join wins; judge each cell lazily after previous sticky reserves settle. private *stickyCandidates( seats: readonly ShadowSeat[], left: { cellId: string; epoch: number; incarnation?: string; closeCode?: number } | undefined, @@ -574,8 +556,7 @@ export class ReserveAssignment { return record } - // The same epoch on the same cell: no booking, no database, no row lock. A cell that - // restarted since forgot the seat, so it is asked to hold it first. + // Same-cell/epoch rebind needs no booking or DB lock unless the restarted cell forgot its seat. private async stickyFromMemory( identity: Identity, seats: ShadowSeat[], @@ -613,8 +594,7 @@ export class ReserveAssignment { seats: readonly ShadowSeat[] = [], // The database lane found the region full: no share draw, and no other region. regionFull = false, - // The caller counts its own outcome (spill, rehome): no database lane follows it, and a pace - // may not be the answer. + // Caller owns spill/rehome accounting; no database lane follows, and pace may be disallowed. spill = false, drainedFrom?: string ): Promise { @@ -763,8 +743,7 @@ export class ReserveAssignment { ) } - // Supersede at placement: the old seats this director's map holds on reserve cells are told - // to go. Each cell checks the seat's epoch and join itself, so a stale map only misses. + // Supersede mapped reserve seats at placement; cells check epoch/join so stale maps only miss old seats. private supersede(identity: Identity, seats: readonly ShadowSeat[], cellId: string, epoch: number): void { for (const seat of seats) { if (seat.cellId === cellId || seat.epoch >= epoch) continue @@ -773,8 +752,7 @@ export class ReserveAssignment { } } - // The feed's half of supersede: a seat the booking map did not hold, found once both joins - // are in this map. Without it that seat holds its cell until the 75 s silence close. + // Feed supersedes half-open seats the booking map missed; otherwise they linger until 75s silence close. supersedeOutranked(seat: OutrankedSeat): void { this.demoteSuperseded(seat, seat.byCellId, 'feed') } diff --git a/cloud/apps/relay/src/reserve-ledger-writer-postgres.test.ts b/cloud/apps/relay/src/reserve-ledger-writer-postgres.test.ts index 2276ed767d9..6dfbff9d8ac 100644 --- a/cloud/apps/relay/src/reserve-ledger-writer-postgres.test.ts +++ b/cloud/apps/relay/src/reserve-ledger-writer-postgres.test.ts @@ -3,8 +3,6 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest' import { openRelayDatabase, type RelayDatabase } from './database.js' import { reconcileReserveLedger, ReserveLedgerWriter } from './reserve-ledger-writer.js' -// The ledger's statements on real PostgreSQL: the guarded upsert, its counter reset on a -// move, and the batched row read. const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip diff --git a/cloud/apps/relay/src/reserve-ledger-writer.test.ts b/cloud/apps/relay/src/reserve-ledger-writer.test.ts index 7b54d8d1a7f..8f17bb40835 100644 --- a/cloud/apps/relay/src/reserve-ledger-writer.test.ts +++ b/cloud/apps/relay/src/reserve-ledger-writer.test.ts @@ -64,7 +64,6 @@ describe('reserve ledger writer', () => { writer.enqueue({ ...HOST, cellId: 'c1', epoch: 5 }) await writer.flush() expect(writer.pending()).toBe(1) - // Backoff: no retry before it is due. await writer.flush() expect(query).toHaveBeenCalledTimes(1) now += 1_000 @@ -86,7 +85,6 @@ describe('reserve ledger writer', () => { it('reconciles the map’s seats and never lowers an epoch', async () => { const { database: db, writer } = await setup() - // Another path wrote a higher epoch on c7: the mirror leaves it, whatever the map says. writer.enqueue({ ...HOST, cellId: 'c7', epoch: 9 }) await writer.flush() writer.enqueue({ ...HOST, cellId: 'c1', epoch: 9 }) @@ -98,14 +96,12 @@ describe('reserve ledger writer', () => { seats: () => [ { ...HOST, cellId: 'c1', epoch: 8, joinedAt: 0 }, { ...other, cellId: 'c1', epoch: 2, joinedAt: 0 }, - // Too young: it may still be superseded. { userId: 'user-3', relayHostId: 'dddddddddddddddd', cellId: 'c1', epoch: 2, joinedAt: 990_000 } ], demote: (seat) => demoted.push(seat), now: 1_000_000 }) await writer.flush() - // The row is ahead of the c1 seat: the row stays, and that seat is told to go. expect(result).toEqual({ upserted: 1, demoted: 1 }) expect(demoted).toEqual([{ ...HOST, cellId: 'c1', epoch: 8, joinedAt: 0 }]) expect(await row(db)).toMatchObject({ cell_id: 'c7', assignment_epoch: 9 }) @@ -123,7 +119,6 @@ describe('reserve ledger writer', () => { writer, seats: () => [ { ...HOST, cellId: 'c1', epoch: 9, joinedAt: 0 }, - // Level with its own row: nothing to do. { ...HOST, relayHostId: 'eeeeeeeeeeeeeeee', cellId: 'c7', epoch: 9, joinedAt: 0 } ], demote: (seat, found) => demoted.push({ seat: seat.cellId, row: found }), diff --git a/cloud/apps/relay/src/reserve-ledger-writer.ts b/cloud/apps/relay/src/reserve-ledger-writer.ts index d10d5d954cf..3aa59452cae 100644 --- a/cloud/apps/relay/src/reserve-ledger-writer.ts +++ b/cloud/apps/relay/src/reserve-ledger-writer.ts @@ -1,14 +1,7 @@ import type { RelayDatabase } from './database.js' -// Step 5's after-the-fact record: a reserve-mode cell admits a booked host without reading -// Postgres, and the director that booked it writes the host's relay_assignments row about a -// second later. Phones still resolve through that row until step 6. -// -// A guarded mirror, which nothing reads for a desktop's correctness: only a higher epoch -// replaces a row, and nothing ever lowers one. No relay_cells row and no activity lease is -// touched. A row whose cell changes drops its counters: the leases that back them keep -// their own cell, and unbacked units stay counted on the old cell until its next reconcile, -// which only makes that cell look fuller. +// Mirror booked joins to relay_assignments after admission; desktop correctness never reads this record. +// Only higher epochs replace rows. Cell moves reset counters but leave leases/unbacked units on the old cell until reconcile, conservatively overstating load. export const RESERVE_LEDGER_FLUSH_MS = 250 export const RESERVE_LEDGER_QUEUE_MAX = 200_000 @@ -59,8 +52,7 @@ export class ReserveLedgerWriter { this.timer = null } - // During a database stall the queue grows; past its bound the oldest entries go first, - // since the 10-minute reconcile rewrites every seat anyway. + // Bound outage growth by dropping oldest queued rows; ten-minute reconcile rewrites seats. enqueue(entry: LedgerEntry): void { this.queue.push(entry) const max = this.input.queueMax ?? RESERVE_LEDGER_QUEUE_MAX @@ -99,7 +91,6 @@ export class ReserveLedgerWriter { } private async write(batch: LedgerEntry[]): Promise { - // The newest entry per host wins inside one batch. const latest = new Map() for (const entry of batch) { const key = `${entry.userId}\u0000${entry.relayHostId}` @@ -147,7 +138,6 @@ export class ReserveLedgerWriter { ) } - // Rows for these hosts, in one read per batch. async readRows( hosts: ReadonlyArray<{ userId: string; relayHostId: string }> ): Promise> { @@ -193,11 +183,8 @@ const RECONCILE_SETTLE_MS = 30_000 type ReconcileSeat = { userId: string; relayHostId: string; cellId: string; epoch: number; joinedAt: number } -// Every director upserts the map's seats on reserve-mode cells: guarded and idempotent, so -// concurrent runs are harmless. A row ahead of the seat (today's path answered a newer epoch, -// and the desktop was then answered the old seat from memory) is never rewritten: that seat is -// demoted (epoch-checked, closes with 4409), so the desktop re-assigns to what the row names. -// The demoting director then mints above the row's epoch for that host (see raiseEpochFloor). +// Every director idempotently mirrors reserve seats; ahead/other-cell equal rows instead trigger epoch-checked 4409 demotion. +// Demoting director raises the next minted epoch above that row (raiseEpochFloor). export async function reconcileReserveLedger(input: { writer: ReserveLedgerWriter seats: () => ReconcileSeat[] diff --git a/cloud/apps/relay/src/reserve-placement-simulation.test.ts b/cloud/apps/relay/src/reserve-placement-simulation.test.ts index 13bd867b93c..e19731139d3 100644 --- a/cloud/apps/relay/src/reserve-placement-simulation.test.ts +++ b/cloud/apps/relay/src/reserve-placement-simulation.test.ts @@ -124,8 +124,7 @@ describe('step 5 deterministic simulation', () => { it.each(SEEDS.flatMap((seed) => [[1_200_000, seed], [300_000, seed]] as const))( 'a drain onto an empty cell at the %i ms pace spreads within each cell intake budget (seed %i)', async (paceMs, seed) => { - // The c27 shape: the drained cell's hosts move at the 1,200 s pace onto five cells - // with free seats 2,805 / 841 / 777 / 667 / 26. + // Production c27 drain: 1,200s pacing toward cells with 2,805/841/777/667/26 free seats. const free = [2_805, 841, 777, 667, 26] const cells: SimulationConfig['cells'] = free.map((seats, index) => ({ cellId: `asia${index + 1}`, @@ -175,9 +174,7 @@ describe('step 5 deterministic simulation', () => { 120_000 ) - // A desktop seated at the same epoch on two cells (a seat half-open within a poll lag, then - // placed again by a director that had not seen it): only the row breaks the tie. Short - // sessions and all five directors restarting each minute put boot reconciles on those seats. + // Same-epoch seats across cells require the DB row to break ties; short sessions/five director restarts per minute exercise boot reconcile. const levelSeats = (seed: number, faults: SimulationConfig['faults']): SimulationConfig => ({ seed, durationMs: 16 * 60_000, @@ -277,8 +274,7 @@ describe('step 5 deterministic simulation', () => { durationMs: 25 * 60_000, oldDirectors: 1 }) - // Demote, re-assign, converge: a placement that never mints above the row is demoted again - // and again behind the same row. + // Reassigning without advancing past the row repeatedly demotes behind the same row. const rowAheadNeverRises = (): SimulationConfig => ({ ...mixed({ mintAtFloor: true }), durationMs: 25 * 60_000, @@ -314,8 +310,7 @@ describe('step 5 deterministic simulation', () => { expect(report.rowAheadDemotions).toBeGreaterThan(0) expect(report.violations.filter((violation) => violation.invariant === 8)).toEqual([]) expect(report.rowAheadDemotionsPerHost.max).toBeLessThanOrEqual(2) - // A director whose map has not seen the leave yet may re-place level with the row; the - // WRONG_CELL row read keeps that a small share (the rest are repaired by one more reconcile). + // Unseen leaves can briefly re-place level with the row; WRONG_CELL reads limit this, then reconcile repairs it. expect(report.badFirstReseats / report.rowAheadDemotions).toBeLessThan(0.1) }, 240_000) @@ -338,9 +333,7 @@ describe('step 5 deterministic simulation', () => { pollOutages: [{ at: 60_000, durationMs: 150_000 }] }) - // Token minting, the cells' token check or a partition: no director's poll gets through again. - // The cells' own rows still say db once they trip, so today's path recovers without the feed; - // before the trips a fully reserve region has no fresh placement (60-120 s). + // Permanent feed/auth partition: tripped cells’ own db rows must recover placement without feed; fully reserve regions wait 60–120s for trips. it('recovers fresh placement while feed polls fail and stay failed, once the cells trip (invariant 9)', async () => { const stuck = (faults: SimulationConfig['faults']) => ({ ...fleetTrip(faults), diff --git a/cloud/apps/relay/src/reserve-placement-simulation.ts b/cloud/apps/relay/src/reserve-placement-simulation.ts index c6b9483ab61..b7d84f75508 100644 --- a/cloud/apps/relay/src/reserve-placement-simulation.ts +++ b/cloud/apps/relay/src/reserve-placement-simulation.ts @@ -11,10 +11,8 @@ import { } from './reserve-placement.js' import { seatOutranks } from './shadow-seat-directory.js' -// A seeded discrete-event model of step 5 that drives the real CellReserveBook and -// ReservePlacer. Everything else (sockets, the database, desktops) is modelled. The checks -// are the design's invariants 1-9; a violation is recorded, never thrown, so one run reports -// every class it hit. +// Seeded event model uses real CellReserveBook/ReservePlacer; sockets, DB and desktops are modeled. +// Record invariant violations without throwing so each run exposes every class it hits. export function seededRandom(seed: number): () => number { let state = seed >>> 0 @@ -134,9 +132,7 @@ export type SimulationFaults = { localTieBreak?: boolean } -// relay_cell_admit_effective: each cell writes its raw mode every 15 s (cell-admit-effective-writer.ts), -// and today's path takes a reserve cell whose current-process row says db within 45 s -// (assignment-store.ts CELL_ADMIT_EFFECTIVE_FRESH_MS). +// Model cell-owned raw-mode writes every 15s; current-process db evidence permits placement within its 45s freshness bound. const CELL_ADMIT_EFFECTIVE_REFRESH_MS = 15_000 const CELL_ADMIT_EFFECTIVE_FRESH_MS = 45_000 // A tripped cell reports reserve until every control it admitted from memory holds a lease. @@ -201,8 +197,7 @@ class SimCell { bookingUnits = 0 // What relay_cells says to a director that predates step 5: frozen at the flip. frozenDatabaseCount = 0 - // The dead-man (cell-reserve-dead-man.ts): a reserve cell no placing director has booked on or - // polled with reserver=1 for its window flips itself to db, latched. + // Latch db after 60–120s without placing-director booking/poll contact (cell-reserve-dead-man.ts). lastContactAt = 0 trippedAt: number | null = null reregisteredAt = 0 @@ -403,9 +398,7 @@ export type SimulationConfig = { rttMs: (region: RelayRegion) => number faults?: SimulationFaults cellRestarts?: Array<{ at: number; cellId: string }> - // Cold start budget: a restarted director's estimates start empty and refill at its share of - // each cell's intake, so it books nothing on a cell for 1 / (intakePerSec / directors) s and - // paces the rest (Retry-After >= 1 s); the startup gate holds unheard hosts up to 30 s. + // Restarted estimates refill from zero at intake/directors: no booking until one token; unheard hosts also face the 30s startup gate. directorRestarts?: Array<{ at: number; director: number }> databaseStalls?: Array<{ at: number; durationMs: number }> // Every director misses every feed poll for this long (a director-side outage). @@ -430,8 +423,7 @@ export type SimulationReport = { rowAheadDemotions: number // Demotions whose winner is level with the loser (same epoch, another cell): the tie-break's cases. levelDemotions: number - // Level divergences (same epoch, two cells) two or more directors judged: the only cases where - // invariant 4 can break, since a higher epoch wins for every director. + // Two directors judging a same-epoch divergence exercise invariant 4; higher epochs already win consistently. contestedDivergences: number // Row-ahead demotions per demoted host: a repair that converges needs one. rowAheadDemotionsPerHost: { max: number; p99: number } @@ -581,7 +573,6 @@ export async function runReservePlacementSimulation( } const between = ([low, high]: [number, number]) => low + random() * (high - low) - // Desktops: the cell and epoch each one currently uses. type Host = { id: string; region: RelayRegion; current: { cellId: string; epoch: number } | null } const hosts: Host[] = Array.from({ length: config.hosts }, (_, index) => ({ id: `h${index}`, @@ -613,8 +604,7 @@ export async function runReservePlacementSimulation( if (!current || quiescing) return const cell = cellById.get(current.cellId)! if (cell.seats.get(host.id)?.epoch !== current.epoch) return - // A half-open control: the desktop moves on, the cell still holds the seat until the - // control's silence timeout closes it. + // Half-open controls retain seats through silence timeout after desktops move elsewhere. if (random() >= config.halfOpenShare) cell.leave(host.id) else { const seat = cell.seats.get(host.id) @@ -627,7 +617,6 @@ export async function runReservePlacementSimulation( }) } - // --- director polling --- function poll(director: SimDirector, cell: SimCell): void { if (inPollOutage(clock.now)) return // A placing director's poll carries reserver=1: it is the cell's dead-man contact. @@ -747,8 +736,7 @@ export async function runReservePlacementSimulation( } } - // Supersede at placement: the director that books a host on `cellId` tells the old seats its - // map holds on reserve cells to go. Each cell checks epoch and join itself. + // Booking director supersedes old reserve seats its map held; cell verifies epoch/join. function supersede(director: SimDirector, host: string, cellId: string, epoch: number): void { const held = new Set() for (const [seatCellId, view] of director.views) { @@ -766,8 +754,7 @@ export async function runReservePlacementSimulation( } } - // As shadow-seat-directory.ts onOutranked: once both joins are in this map, a reserve seat behind - // a later join at a newer epoch is told to go. A lower epoch that joined later is the row's. + // Model feed supersede once both joins appear: only later joins at higher epochs outrank; later lower epochs need the row. function supersedeOutranked(director: SimDirector, hosts: Iterable): void { if (faults.skipSupersede || faults.skipFeedSupersede) return for (const host of new Set(hosts)) { @@ -789,8 +776,7 @@ export async function runReservePlacementSimulation( // The seats the booking director's map held when it placed each booking: only those it supersedes. const heldAtBooking = new Map>() - // Invariant 4: each demotion names its winner (the newer booking, or the row). Every director - // names the same one per divergence, so no two seats of a host are each demoted for the other. + // Invariant 4: all directors must name the same winner per divergence or two seats can each be demoted for the other. const namedWinners = new Map() const contested = new Set() const demotedFor = new Map() @@ -855,9 +841,7 @@ export async function runReservePlacementSimulation( director.ledgerQueue = [] } - // Invariant 8: a reserve seat behind its row (today's path wrote a newer epoch, and the - // desktop was answered the old seat from memory), or level with it on another cell, is gone - // by the next reconcile, and the director that demoted it re-places the host above the row. + // Invariant 8: reconcile removes ahead/other-cell equal-row divergences; demoting director must then mint above the row. const behindRow = new Map() const rowAheadDemotionsByHost = new Map() const demotedBehindRow = new Map() @@ -884,9 +868,7 @@ export async function runReservePlacementSimulation( const row = database.rows.get(host) const live = cellById.get(cellId)!.seats.get(host) const diverged = row && (row.epoch > seat.epoch || (row.epoch === seat.epoch && row.cellId !== cellId)) - // A map that has not yet seen another director's demotion still judges the seat; the - // cell drops the demote, but the winner it names must be the same. - // relay-server.ts raises the floor on every demote, so a dropped one raises it too. + // Even demotions dropped by the cell must agree on winner; relay-server raises epoch floors on every demote. if (row && diverged && live && live.joinedAt === seat.joinedAt && live.demoted && !faults.noRowAheadDemote) { const thisSeat = { cellId, epoch: seat.epoch } const rowSeat = { cellId: row.cellId, epoch: row.epoch } @@ -911,8 +893,7 @@ export async function runReservePlacementSimulation( report.rowAheadDemotions += 1 raiseRowFloor(director, host, row.epoch) rowAheadDemotionsByHost.set(host, (rowAheadDemotionsByHost.get(host) ?? 0) + 1) - // A director that had not seen the leave may re-place level with the row once; a later - // divergence (a newer row) may demote the host again. One row never takes a third. + // A director missing a leave can re-place level once; newer rows create new divergences, but one row must never take a third demotion. const sameRow = `${host}\u0000${row.cellId}\u0000${row.epoch}` const behindThisRow = (demotedBehindRow.get(sameRow) ?? 0) + 1 demotedBehindRow.set(sameRow, behindThisRow) @@ -926,7 +907,6 @@ export async function runReservePlacementSimulation( } } - // --- assignment --- function reserveCall( director: SimDirector, cell: SimCell, @@ -948,8 +928,7 @@ export async function runReservePlacementSimulation( function databaseAssign(director: SimDirector, host: Host, floor: number): Answer { if (!database.up) return { kind: 'retry', afterMs: 2_000, calls: 0 } const row = database.rows.get(host.id) - // Postgres still says reserve after a trip; the cell's own fresh row (db, current process) - // lets today's path back in, with or without any feed poll getting through. + // Fresh current-process raw db evidence overrides stale reserve stamp, even without feed polls. const admitsDatabase = (cell: SimCell): boolean => { const own = database.admitEffective.get(cell.cellId) return ( @@ -1022,8 +1001,7 @@ export async function runReservePlacementSimulation( const sticky = found && rowFloor !== undefined && found.epoch <= rowFloor ? null : found if (sticky) { const cell = cellById.get(sticky.cellId)! - // The map's own view of the cell's incarnation: a restart not yet polled answers stale, - // and the hello then falls to the database check. + // Unpolled restarts leave a stale incarnation in the map; hello falls back to the DB check. const view = director.views.get(sticky.cellId) if (sticky.incarnation === view?.incarnation || cell.old) { report.stickyAnswers += 1 @@ -1092,7 +1070,6 @@ export async function runReservePlacementSimulation( return seated || director.placer.hasSeat(cell) } - // --- desktops --- let directorCursor = 0 function pickDirector(): SimDirector { directorCursor = (directorCursor + 1 + Math.floor(random() * directors.length)) % directors.length @@ -1142,8 +1119,7 @@ export async function runReservePlacementSimulation( } report.placements += 1 report.arrivalsByCell[cellId] = (report.arrivalsByCell[cellId] ?? 0) + 1 - // Reported, not a violation: the first seat after a demotion behind the row lands on that - // row or above it, unless the answering director's map had not yet seen the leave. + // Report first post-demotion seats behind/level with the row; stale-map directors may not have seen the leave. const above = reassignAbove.get(host.id) if (above !== undefined) { reassignAbove.delete(host.id) @@ -1153,15 +1129,13 @@ export async function runReservePlacementSimulation( scheduleLeave(host) } - // --- schedule --- for (const cell of cells) { const windowMs = deadManWindowMs(cell.cellId) const check = (): void => { const wasReserve = cell.mode === 'reserve' cell.checkDeadMan(windowMs, config.reregistrationMs ?? DEFAULT_REREGISTRATION_MS) if (wasReserve && cell.mode === 'db') report.deadManTrips += 1 - // Best-effort through the cell's own pool: nothing to write while the database is down, - // and only while its switch says reserve (a db cell never writes one). + // Model best-effort cell writes only with DB up and switch in reserve; db-mode cells write no admit row. const own = database.admitEffective.get(cell.cellId) if ( !cell.old && @@ -1276,7 +1250,6 @@ export async function runReservePlacementSimulation( }) } - // --- invariants checked after every event --- const afterEach = (): void => { for (const cell of cells) { if (cell.occupancy() > cell.hardCap) { @@ -1286,17 +1259,13 @@ export async function runReservePlacementSimulation( report.maxOccupancyShare[cell.cellId] = Math.max(report.maxOccupancyShare[cell.cellId] ?? 0, share) } } - // A booking supersedes the old seats its director's map held within a round trip, and the feed - // any seat a later join outranks. Any other duplicate (a half-open seat today's path placed past - // at a lower epoch than it) has no lease to re-check and lasts until that seat's silence close. + // Mapped/booked or feed-outranked seats have a short supersede bound; unleased half-open duplicates missed by placement can last until silence close. const supersedeBound = Math.max(...directors.map((director) => config.pollLagMs(director.index))) + 2 * Math.max(...cells.map((cell) => config.rttMs(cell.region))) + 3_000 const duplicateBound = CONTROL_SILENCE_CLOSE_MS + 2 * Math.max(...cells.map((cell) => config.rttMs(cell.region))) + 1_000 - // Invariant 3: a seat outranked by a newer one (a higher epoch elsewhere) is gone within the - // bound. Only seats on reserve cells are step 5's: between database-mode cells today's lease - // renewal closes the stale control, as it does now. + // Invariant 3 bounds outranked reserve seats; database-mode duplicates use existing lease renewal. const outrankedSince = new Map() const checkDuplicates = (): void => { const seatsByHost = new Map>() @@ -1360,10 +1329,7 @@ export async function runReservePlacementSimulation( }) } } - // Invariant 5: the ledger is a guarded mirror. After one reconcile with the database up, a - // host seated once, at its highest epoch, on a reserve cell has a row at that epoch or above. - // No final reconcile: what the queues, the boot reconcile and the 10-minute one wrote is - // what a reader would see. + // Invariant 5: guarded mirror must converge through queues/boot/ten-minute reconciles, with no final reconcile hiding lost writes. database.up = true for (const director of directors) flushLedger(director) for (const cell of cells) { diff --git a/cloud/apps/relay/src/reserve-placement.test.ts b/cloud/apps/relay/src/reserve-placement.test.ts index 38f92e5f898..8df18dfe9ab 100644 --- a/cloud/apps/relay/src/reserve-placement.test.ts +++ b/cloud/apps/relay/src/reserve-placement.test.ts @@ -127,7 +127,6 @@ describe('reserve placer', () => { placer.pick(cells, 'us-central1', new Set()) now.value = 1_000 cells[0]!.polledAt = 1_000 - // Two of this director's own bookings fill the estimated ceiling. void placer.place({ cells, region: 'us-central1', epoch: 1, reserve: async () => ({ outcome: 'ok' }) }) void placer.place({ cells, region: 'us-central1', epoch: 1, reserve: async () => ({ outcome: 'ok' }) }) expect(placer.pick(cells, 'us-central1', new Set())).toBeNull() diff --git a/cloud/apps/relay/src/reserve-placement.ts b/cloud/apps/relay/src/reserve-placement.ts index 2cba1baaf84..5caf3fad5f0 100644 --- a/cloud/apps/relay/src/reserve-placement.ts +++ b/cloud/apps/relay/src/reserve-placement.ts @@ -1,15 +1,12 @@ import type { RelayRegion } from '@orca-cloud/relay-contract' import { CELL_INTAKE_BURST, CELL_RESERVE_TTL_MS, type ReserveOutcome } from './cell-reserve-contract.js' -// The director's half of step 5: every director picks a cell from its own map and asks the -// cell to hold a seat. It only estimates; the cell decides. Pure, so the simulation drives -// exactly this code. +// Directors estimate/pick from their own maps; cells decide bookings. Simulation drives this same pure logic. export const RESERVE_MAX_TRIES = 3 // A cell not polled this recently is not a candidate (unverifiable, never gone). export const RESERVE_CELL_FRESH_MS = 3_000 -// Each director assumes it owns this share of a cell's intake (5 = the configured director -// count). Fewer live directors only under-use the budget; more only cause refusals. +// Each director estimates 1/5 of intake; fewer under-use budget, more only cause refusals. export const RESERVE_DIRECTOR_SHARE = 5 export const RESERVE_PACE_MIN_SECONDS = 1 export const RESERVE_PACE_MAX_SECONDS = 10 @@ -63,12 +60,10 @@ export class ReservePlacer { ) } - // Seats, the cell's bookings and this director's own bookings since that poll. load(cell: PlacementCell): number { return cell.seats + cell.bookings + this.ownBookingsSince(cell) } - // Estimated free seats under the ceiling; 0 when the cell reports none. free(cell: PlacementCell): number { return cell.ceiling === undefined ? 0 : Math.max(0, cell.ceiling - this.load(cell)) } @@ -132,8 +127,7 @@ export class ReservePlacer { return this.fallback(input.cells, input.region, calls, { regionFull: input.regionFull, refused }) } - // Database cells in the region come first (unless known full), then pacing on a cell that - // only lacks budget; anything else falls to the database path, which spills as today. + // Try regional DB cells before pacing budget-only reserve shortages; otherwise database placement may spill. fallback( cells: readonly PlacementCell[], region: RelayRegion, @@ -164,8 +158,7 @@ export class ReservePlacer { } } - // Share of the region's free seats on reserve cells; 0 unless every database cell in the - // region reports its ceiling, so this never places only on the new pool by accident. + // Without every regional DB cell’s ceiling, return zero share to avoid accidentally placing only on reserve cells. reservePoolShare(cells: readonly PlacementCell[], region: RelayRegion): number { const now = this.now() let reserveFree = 0 @@ -215,7 +208,7 @@ export class ReservePlacer { .length } - // Returns the entry it added, so a refusal removes that one and not a concurrent call's. + // Remember exact added entry so refusal cannot undo another concurrent call’s booking. private countBooking(cell: PlacementCell): number { const estimate = this.refill(cell, this.now()) estimate.tokens = Math.max(0, estimate.tokens - 1) diff --git a/cloud/apps/relay/src/reserve-rehome.test.ts b/cloud/apps/relay/src/reserve-rehome.test.ts index 4805e1dd67b..7165d33e215 100644 --- a/cloud/apps/relay/src/reserve-rehome.test.ts +++ b/cloud/apps/relay/src/reserve-rehome.test.ts @@ -71,7 +71,6 @@ describe('rehome on reconnect rules', () => { expect(admitted(rehome, { now: RESERVE_REHOME_HOST_COOLDOWN_MS })).toBeNull() }) - // Jinwoo's condition: no move into a region without real room. Asia at its 98% peak gets none. it('moves no one into a region under 10% free, however full the source is', () => { expect(admitted(warmed(), { target: { free: 20, ceiling: 1_000, bookable: true } })).toBe('target-room') expect(admitted(warmed(), { target: { free: 99, ceiling: 1_000, bookable: true } })).toBe('target-room') @@ -84,7 +83,6 @@ describe('rehome on reconnect rules', () => { const rehome = warmed() expect(admitted(rehome)).toBeNull() expect(admitted(rehome, { identity: { ...HOST, relayHostId: 'qrstuvwxyzabcdef' } })).toBe('source-budget') - // Another source cell has its own bucket. expect(admitted(rehome, { sourceCellId: 'us-r2', now: 0 })).toBe('source-budget') rehome.refund('us-r') expect(admitted(rehome, { identity: { ...HOST, relayHostId: 'qrstuvwxyzabcdef' } })).toBeNull() @@ -140,7 +138,6 @@ describe('the rehome switch under a stalled read', () => { return { enabled: false, notBefore: 0 } }, () => now) const stalled = control.refresh() - // The next tick finds the first read still out, so it does not start another. await control.refresh() expect(reads).toEqual(['stalled']) now += RESERVE_REHOME_SWITCH_STALE_MS diff --git a/cloud/apps/relay/src/reserve-rehome.ts b/cloud/apps/relay/src/reserve-rehome.ts index 9ccdf642a3a..2b9e19c68c1 100644 --- a/cloud/apps/relay/src/reserve-rehome.ts +++ b/cloud/apps/relay/src/reserve-rehome.ts @@ -1,12 +1,9 @@ import { RELAY_CLOSE_CODE, type RelayRegion } from '@orca-cloud/relay-contract' import { hostCohortBucket } from './host-cohort-bucket.js' -// Rehome on reconnect (step 1, C): a host that reconnects anyway, sitting on a reserve cell -// outside the region it asked for, is booked in its own region instead. Never a disconnect, -// never a 503; every rule that fails leaves today's answer. +// Rehome only on an existing reconnect outside the requested region; failed rules retain sticky answer, never force disconnect/503. -// Below this share of free seats over the target region's reserve ceilings, nobody moves in. -// B spills at 0% free, so a spilled host comes back only once there is real room. +// Require 10% target headroom above the 0% spill line so spilled hosts return only with real room. export const RESERVE_REHOME_TARGET_FREE_SHARE = 0.1 // Two installs sharing one identity replace each other with PEER_DROPPED. export const RESERVE_REHOME_PEER_DROPPED_MS = 10 * 60_000 @@ -51,8 +48,7 @@ export class ReserveRehome { return this.input.percent > 0 } - // Rules 1 and 4-7 for a host already known to be on a reserve cell outside its region. The - // first rule that fails, or null with the source cell's token taken. + // For a reserve host outside its region, return first failing admission rule or take source token and return null. admit(input: { identity: Identity sourceCellId: string @@ -134,8 +130,7 @@ function hostKey(identity: Identity): string { return `${identity.userId}\u0000${identity.relayHostId}` } -// The operator's regional rehome switch (relay_region_rehome_control), cached: unknown, failed -// or stale reads as off. Polled only on a director with C switched on. +// Cache operator regional rehome switch; unknown/failed/stale means off. Poll only enabled directors. export class ReserveRehomeSwitch { private last: { enabled: boolean; notBefore: number; readAt: number } | null = null private timer: ReturnType | null = null diff --git a/cloud/apps/relay/src/reserve-spill-fleet.test.ts b/cloud/apps/relay/src/reserve-spill-fleet.test.ts index 170ecb2a604..aece2551fea 100644 --- a/cloud/apps/relay/src/reserve-spill-fleet.test.ts +++ b/cloud/apps/relay/src/reserve-spill-fleet.test.ts @@ -19,8 +19,7 @@ vi.mock('./relay-token-verifier.js', () => ({ import { createRelayApp } from './app.js' -// A fleet with no database-mode cell, through /v1/assign: the real plan and the real store, -// with each reserve cell's own booking rule played by a ceiling of two seats. +// Exercise real /v1/assign plan/store with no database-mode cells; booking fakes enforce two seats per cell. const CEILING = 2 const CELLS: RelayCellConfig[] = [ diff --git a/cloud/apps/relay/src/resume-confirmed-audit-buffer.test.ts b/cloud/apps/relay/src/resume-confirmed-audit-buffer.test.ts index 2c510db7156..b6d1db7bf6e 100644 --- a/cloud/apps/relay/src/resume-confirmed-audit-buffer.test.ts +++ b/cloud/apps/relay/src/resume-confirmed-audit-buffer.test.ts @@ -116,7 +116,6 @@ describe('the batched resume-confirmed audit', () => { aged: RESUME_AUDIT_BUFFER_MAX_ROWS }) expect(await rows()).toEqual([]) - // Quiet once nothing more is lost. await buffer.flush() expect(lines).toHaveLength(2) buffer.stop() @@ -127,7 +126,7 @@ describe('the batched resume-confirmed audit', () => { database.failing = () => true for (let index = 0; index < RESUME_AUDIT_FLUSH_ROWS + 50; index += 1) buffer.add(event(`r${index}`)) await buffer.flush() - // The 50 adds past the batch size sent nothing more: no insert per confirm into a failing database. + // Crossing batch size schedules one insert; further adds must not hammer a failing database. expect(database.statements).toHaveLength(1) buffer.stop() }) diff --git a/cloud/apps/relay/src/resume-confirmed-audit-buffer.ts b/cloud/apps/relay/src/resume-confirmed-audit-buffer.ts index f23239126fa..63a300c7f2a 100644 --- a/cloud/apps/relay/src/resume-confirmed-audit-buffer.ts +++ b/cloud/apps/relay/src/resume-confirmed-audit-buffer.ts @@ -1,14 +1,11 @@ import { randomUUID } from 'node:crypto' import type { RelayDatabase } from './database.js' -// Step 6 (decision 4): a memory cell's resume-confirmed audit rows leave in batches, off the -// confirm's round trips. A crash or shutdown loses at most one flush interval of rows: accepted, -// since a drain at exit would race the database pool closing. +// Batch audit writes off confirm’s round trips; unflushed rows are lost on crash/shutdown because exit flushing would race pool closure. export const RESUME_AUDIT_FLUSH_MS = 5_000 export const RESUME_AUDIT_FLUSH_ROWS = 500 export const RESUME_AUDIT_BUFFER_MAX_ROWS = 5_000 -// An erasure deletes an account's audit rows by hand; a row this old is dropped rather than -// retried, so a database outage cannot insert it after the account is gone. +// Drop aged rows rather than reinserting erased accounts’ audit data after an outage. export const RESUME_AUDIT_MAX_AGE_MS = 10 * 60 * 1000 export type ResumeConfirmedAudit = { @@ -18,8 +15,7 @@ export type ResumeConfirmedAudit = { detail: { reqId: string; basisConnId: string; renewed: boolean } } -// The id is fixed when the event happens, so a retried flush after an unknown commit outcome -// cannot write a row twice (ON CONFLICT (id) DO NOTHING). +// Keep event IDs fixed across unknown commit outcomes so ON CONFLICT prevents duplicate audit rows. type BufferedRow = ResumeConfirmedAudit & { id: string; at: number } export type ResumeAuditDrops = { overflow: number; aged: number } diff --git a/cloud/apps/relay/src/seat-feed-cells-postgres.test.ts b/cloud/apps/relay/src/seat-feed-cells-postgres.test.ts index be621f89bc8..6e5e3e97369 100644 --- a/cloud/apps/relay/src/seat-feed-cells-postgres.test.ts +++ b/cloud/apps/relay/src/seat-feed-cells-postgres.test.ts @@ -4,7 +4,6 @@ import { RelayAssignmentStore } from './assignment-store.js' import type { RelayCellConfig } from './config.js' import { openRelayDatabase, type RelayDatabase } from './database.js' -// The seat-feed cell list (with the process incarnation the reserve switch keys on) on real PostgreSQL. const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip diff --git a/cloud/apps/relay/src/shadow-directory-compare.test.ts b/cloud/apps/relay/src/shadow-directory-compare.test.ts index c2a36d64ede..aefda8206c6 100644 --- a/cloud/apps/relay/src/shadow-directory-compare.test.ts +++ b/cloud/apps/relay/src/shadow-directory-compare.test.ts @@ -39,7 +39,6 @@ function directoryWith( ) { const directory = new ShadowSeatDirectory() const cellIds = [...Object.keys(seats), ...noFeed] - // Read at 0; live cells' heartbeats run out far in the future. directory.setCells(cellIds, cellIds, { readAt: 0, expiresAt: new Map( @@ -108,7 +107,6 @@ describe('classifyShadowSeat', () => { }) it('explains a long-seated host moved by a rehome, then stops once the move is old', () => { - // Seated for three hours on cell-b; a rehome grants cell-a at a newer epoch. const directory = directoryWith({ 'cell-b': [seat(2)], 'cell-a': [] }) const threeHours = 3 * 60 * 60 * 1_000 const poll = (cellId: string, now: number) => @@ -146,7 +144,6 @@ describe('classifyShadowSeat', () => { expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), 10)).toMatchObject({ explained: true }) - // cell-b was last polled at 10; 20 s later only cell-a is fresh. directory.apply('cell-a', feed('cell-a', { full: undefined, changes: [] }), 20_010) expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), 20_010)).toEqual({ class: 'cell-unpolled', @@ -155,7 +152,6 @@ describe('classifyShadowSeat', () => { }) it('keeps a cell without a feed out of the coverage count, on either side of the compare', () => { - // cell-b reported a seat, then rolled back to an image without the feed. const directory = directoryWith({ 'cell-b': [seat(2)], 'cell-a': [] }) directory.markNoFeed('cell-b', 20) for (const answer of [at('cell-a', 3), at('cell-b', 2)]) { @@ -270,7 +266,6 @@ describe('classifyShadowSeat', () => { }) it('classifies before recording, so a later epoch bump is explained', () => { - // Seated on cell-a at epoch 2 and on cell-b at epoch 1; the database names cell-a at 2. const directory = directoryWith({ 'cell-a': [seat(2, 'active', HOST, firstSeen)], 'cell-b': [seat(1, 'active', HOST, 1)] @@ -281,7 +276,6 @@ describe('classifyShadowSeat', () => { class: 'superseded-unknown-age', explained: false }) - // A re-assign to cell-b at epoch 3 supersedes cell-a's epoch-2 seat. now = firstSeen + 1_700 directory.apply( 'cell-b', @@ -423,7 +417,6 @@ describe('ShadowDirectoryCompare', () => { unseen.compare('sticky-verify', IDENTITY, assignment('cell-b')) unseen.compare('resolve', IDENTITY, null) unseen.flush(0) - // The database says cell-a at epoch 3; the map seats the host on cell-b at a newer epoch. const mismatch = compareOn({ 'cell-a': [], 'cell-b': [seat(4)] }) mismatch.compare('sticky-verify', IDENTITY, assignment('cell-a')) mismatch.flush(0) @@ -435,8 +428,7 @@ describe('ShadowDirectoryCompare', () => { const draining = compareOn({ 'cell-a': [seat(3)], 'cell-b': [seat(2, 'drain-only')] }) draining.compare('sticky-verify', IDENTITY, assignment('cell-a')) draining.flush(0) - // A superseded older seat: the first-ever sighting has no age, so it is its own unexplained - // class on both cells; seen before, it is explained and counts only on the database's cell. + // First sighting cannot date a superseded seat; subsequent compares explain it and count only the database cell. const superseded = compareOn({ 'cell-a': [seat(3)], 'cell-b': [seat(2)] }) superseded.compare('sticky-verify', IDENTITY, assignment('cell-a')) superseded.compare('sticky-verify', IDENTITY, assignment('cell-a')) @@ -466,7 +458,6 @@ describe('unexplained samples', () => { () => 20, (line) => lines.push(JSON.parse(line)) ) - // Map ahead of the database: unexplained, so it is sampled. compare.compare('sticky-verify', IDENTITY, { ...assignment('cell-a'), assignmentEpoch: 2 }) compare.flush(20) const sample = lines.find((line) => line.event === 'orca_relay_shadow_compare_unexplained') diff --git a/cloud/apps/relay/src/shadow-directory-compare.ts b/cloud/apps/relay/src/shadow-directory-compare.ts index 0fbf24f90cb..1074ea83ae3 100644 --- a/cloud/apps/relay/src/shadow-directory-compare.ts +++ b/cloud/apps/relay/src/shadow-directory-compare.ts @@ -2,9 +2,7 @@ import { RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract' import { relayHostLogDigest } from './relay-host-log-digest.js' import type { ShadowSeatDirectory } from './shadow-seat-directory.js' -// Step 3: compares the shadow seat map with the database answer on the two -// reads step 5 will serve from memory. It only counts and logs; the database -// answer is always the one returned. +// Compare memory seats against database answers for sticky verification/resolve; only count/log, preserving the database response. export type ShadowCompareRoute = 'sticky-verify' | 'resolve' @@ -34,11 +32,9 @@ export type ShadowCompareVerdict = { class: ShadowCompareClass; explained: boole export type ShadowCompareDatabaseAnswer = { cellId: string; assignmentEpoch: number } | null export const SHADOW_COMPARE_FLUSH_MS = 60_000 -// One poll, its timeout, and a reconnect (p95 ~6 s on 10-02). A move older than -// this is not "the map catching up": a lost leave must not read as explained forever. +// One poll/timeout/reconnect (~6s p95); cap explanations so lost leaves cannot look like lag forever. export const SHADOW_COMPARE_LAG_BOUND_MS = 10_000 -// A superseded half-open control lingers until the cell's silence close (the timeout plus up -// to one ping interval), then the map trails that close by a poll. +// Half-open superseded controls linger through silence timeout + ping interval + map polling lag. export const SHADOW_COMPARE_SUPERSEDED_CLOSE_MS = RELAY_PROTOCOL_LIMITS.controlSilenceTimeoutMs + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs + @@ -57,8 +53,7 @@ export function classifyShadowSeat( // When this instance began comparing; omitted, a missing sighting is never excused. startedAt = -Infinity ): ShadowCompareVerdict { - // Pure: only ShadowDirectoryCompare records sightings. An epoch never seen is new; unlike the - // superseded rule, a wrong guess here only explains a mismatch for one lag bound. + // Classification must not record sightings; a new-epoch guess explains mismatch for only one lag bound. const movedAt = answer ? (directory.databaseEpochFirstSeen( identity.userId, @@ -100,8 +95,7 @@ export function classifyShadowSeat( if (!supersededShape) return { class: 'duplicate-seat', explained: false } const known = directory.databaseEpochOf(identity.userId, identity.relayHostId) if (!known) { - // Later, no sighting means another instance served the earlier compares, or the map evicted - // the host, so a harmless supersede still reads as an unexplained duplicate-seat. + // Later missing sightings can mean other instances served compares or eviction; keep supersedes unexplained. const starting = now - startedAt <= SHADOW_COMPARE_SUPERSEDED_CLOSE_MS return { class: starting ? 'superseded-unknown-age' : 'duplicate-seat', explained: false } } @@ -117,8 +111,7 @@ export function classifyShadowSeat( .some((entry) => entry.cellId === answer.cellId) return { class: left ? 'db-only-left' : 'db-only-unseen', explained: true } } - // The map trails the cells by a poll. The database answer carries no grant time, so the - // move's age is how long this director has seen the newer epoch. + // No database grant time: date moves by this director’s first sighting of the newer epoch. const mapBehind = answer.assignmentEpoch > seat.epoch && now - movedAt <= SHADOW_COMPARE_LAG_BOUND_MS if (seat.cellId !== answer.cellId) { @@ -138,8 +131,7 @@ export function classifyShadowSeat( return { class: 'agree', explained: true } } -// When the move to the answer's epoch happened, from the sighting recorded before this compare. -// A regressed epoch means unknown, not new. +// Date moves from prior sightings; regressed epochs are unknown, not new. function movedSince( known: { epoch: number; firstSeenAt: number }, answer: { assignmentEpoch: number }, @@ -150,9 +142,7 @@ function movedSince( return known.epoch < answer.assignmentEpoch ? now : known.firstSeenAt } -// The database's cell, or with no row every seat's cell; an unexplained cell-mismatch or -// duplicate-seat also counts on every seat cell, so a stale seat feed shows on its own cell. -// A compare with neither (agree-absent, or map-incomplete with no row and no seat) has none. +// Count DB cell and, for unexplained mismatch/duplicate, every seated cell so stale feeds show locally; no row/seats means no cells. function comparedCells( directory: ShadowSeatDirectory, identity: { userId: string; relayHostId: string }, @@ -217,8 +207,7 @@ export class ShadowDirectoryCompare { return verdict } - // One line per route and class per window; the log metric sums `count`. `countByCell` - // counts each compare once on every cell it involves (see comparedCells). + // Flush per route/class; count each compare once on every involved cell (comparedCells). flush(now: number = this.now()): void { for (const [key, { count, countByCell }] of this.counts) { const [route, verdictClass, explained] = key.split('\u0000') diff --git a/cloud/apps/relay/src/shadow-seat-directory.ts b/cloud/apps/relay/src/shadow-seat-directory.ts index 4a9528a80a9..6b72d72f8eb 100644 --- a/cloud/apps/relay/src/shadow-seat-directory.ts +++ b/cloud/apps/relay/src/shadow-seat-directory.ts @@ -8,8 +8,7 @@ import { import type { RelayConfig } from './config.js' import { googleMetadataIdentityToken, reusedIdentityToken } from './google-metadata-identity-token.js' -// Step 3: each director polls every cell's seat feed and keeps who is seated -// where in memory. It decides nothing; readers only compare and report. +// Each director mirrors cell seat feeds; callbacks reconcile booked joins and superseded seats. export const SHADOW_SEAT_POLL_MS = 1_000 export const SHADOW_SEAT_POLL_TIMEOUT_MS = 2_000 @@ -91,8 +90,7 @@ const SeatFeedSchema = z.object({ flagsApplied: z .object({ generation: z.union([z.string(), z.number()]), flags: z.record(z.string(), z.unknown()) }) .optional(), - // What the cell acts on: reserve also while a flip back is still leasing its controls, or - // db once its dead-man has flipped it back. Absent from older images. + // Effective reserve persists while flip-back leases controls; raw mode reports dead-man db immediately. Older images omit both. admitModeEffective: z.enum(['db', 'reserve']).optional(), // Switch file and dead-man only, without the flip back: db the moment the cell trips. admitModeRaw: z.enum(['db', 'reserve']).optional(), @@ -326,8 +324,7 @@ export class ShadowSeatDirectory { return cursor?.flagsApplied?.flags.admitMode === 'reserve' ? 'reserve' : 'db' } - // What placement treats the cell as: a tripped cell is a database cell at once, so nothing - // books into it or paces for it while it re-registers. Older images report no raw mode. + // Raw db stops bookings/pacing immediately during re-registration; older images fall back to effective mode. placementModeOf(cellId: string): 'db' | 'reserve' { const raw = this.cells.get(cellId)?.admitModeRaw return raw ?? this.admitModeOf(cellId) @@ -337,7 +334,6 @@ export class ShadowSeatDirectory { return [...this.cells.keys()] } - // One cell's seats as the map holds them. seatsOnCell(cellId: string): Array<{ userId: string; relayHostId: string; seat: ShadowSeat }> { const cursor = this.cells.get(cellId) if (!cursor) return [] @@ -358,8 +354,7 @@ export class ShadowSeatDirectory { cursor.units = undefined cursor.ceiling = undefined cursor.intake = undefined - // A reserve switch stands until a feed says db, or the cell's process is replaced: one 404 - // must not read a reserve cell's seated hosts as idle. Anything else is forgotten. + // Keep reserve stamp until feed says db or process changes; one 404 must not misclassify seated reserve hosts as idle. const reserve = cursor.flagsApplied?.flags.admitMode === 'reserve' const replaced = runtimeIncarnation !== undefined && runtimeIncarnation !== cursor.flagsIncarnation @@ -378,8 +373,7 @@ export class ShadowSeatDirectory { cursor.lastFailure = reason } - // Every live cell has answered (a feed or a 404) since this director started; - // a dead, empty or isolated cell must not hold the measurement back. + // Completeness waits only for required live cells’ feed/404 answers, not dead/empty/isolated cells. isComplete(): boolean { if (this.required.size === 0) return false for (const cellId of this.required) { @@ -388,8 +382,7 @@ export class ShadowSeatDirectory { return true } - // What the database's resolve would call this cell now, from a cell-list read up to - // 30 s old. `unknown`: its heartbeat ran out after that read, and the next read decides. + // Database liveness from <=30s-old cell list; expired-since-read heartbeat is unknown pending refresh. cellLiveness(cellId: string, now: number): 'live' | 'unlive' | 'unknown' { if (!this.heartbeats) return this.cells.has(cellId) ? 'live' : 'unlive' const expiresAt = this.heartbeats.expiresAt.get(cellId) @@ -601,9 +594,7 @@ export class ShadowSeatDirectory { if (this.onOutranked && seats.size > 1) this.joinedSinceApply.add(key) } - // After the whole response, so a cell's first snapshot is judged by the mode it carries. Only a - // join triggers it, once per seat: a cell flipped to reserve later, or a lost demote, falls to - // the silence close or the ledger reconcile, as does a lower epoch that joined later. + // Judge after full response applies mode, once per join. Later reserve flips/lost demotes/later lower epochs rely on silence close or reconcile. private reportOutranked(): void { const report = this.onOutranked const keys = [...this.joinedSinceApply] @@ -756,8 +747,7 @@ export function startShadowSeatPoller( cellsReadAt = now() cellListFailures = 0 } catch (error) { - // Keep polling the last list; the database being down is not the cells being down. - // Back off so a stalled database is not asked once a second on a 3-connection pool. + // Keep polling last list through DB outages; back off refreshes to protect the three-connection pool. cellListFailures += 1 const backoffMs = CELL_LIST_RETRY_BASE_MS * 2 ** (cellListFailures - 1) cellListRetryAt = now() + Math.min(CELL_LIST_RETRY_MAX_MS, backoffMs) @@ -831,8 +821,7 @@ export function startShadowSeatPoller( const round = async (awaitPolls: boolean): Promise => { if (stopped) return const refresh = refreshCells() - // Only the first list is waited for: later rounds poll the last list, so the round that starts - // a refresh no longer skips its polls (one round, ~1 s, per refresh) while the read is slow. + // Await only the first cell list; later slow refreshes must not skip a poll round. if (cellsReadAt === undefined) await refresh // A slow cell holds only its own slot; the timer never waits on it. const polls = cells.map((cell) => poll(cell)) @@ -860,9 +849,7 @@ export function startShadowSeatPoller( startedAt, tick: () => round(true), deadCellVerdict: (cellId, at) => { - // Peers count once they have served this director a feed. The candidate always does: the - // store asks only for a cell Postgres records as reserve, which only a feed image runs, and a - // director restarted after it died would otherwise never hear from it. + // Count peers with feed evidence, plus the candidate: PostgreSQL reserve implies a feed image, even if director restarted after it died. const feedCells = cells.filter((cell) => { if ((cell.capacityRequests ?? 1) <= 0) return false if (cell.cellId === cellId) return true diff --git a/cloud/apps/relay/src/splice-forwarder.test.ts b/cloud/apps/relay/src/splice-forwarder.test.ts index b6b416c68e3..b72c1ce420c 100644 --- a/cloud/apps/relay/src/splice-forwarder.test.ts +++ b/cloud/apps/relay/src/splice-forwarder.test.ts @@ -106,8 +106,7 @@ describe('splice forwarder', () => { }) it('queues one full catalog-sized frame for a backpressured peer without closing', () => { - // Why: the desktop's worktree catalog response exceeds 1MiB on large - // workspaces; a single maxFrameBytes frame must survive backpressure. + // Large workspace catalogs exceed 1MiB; one maxFrameBytes frame must survive backpressure. vi.useFakeTimers() const client = new FakeSocket() const host = new FakeSocket() @@ -126,7 +125,6 @@ describe('splice forwarder', () => { expect(host.closes).toEqual([]) expect(budget.current()).toBe(8 * 1024 * 1024) - // Peer drains; the queued frame flushes and the reservation releases. host.bufferedAmount = 0 vi.advanceTimersByTime(25) expect(host.sent).toHaveLength(1) diff --git a/cloud/apps/relay/src/splice-forwarder.ts b/cloud/apps/relay/src/splice-forwarder.ts index b0a31b49830..3a03586ae24 100644 --- a/cloud/apps/relay/src/splice-forwarder.ts +++ b/cloud/apps/relay/src/splice-forwarder.ts @@ -39,8 +39,7 @@ function transport(socket: WebSocket): { pause(): void; resume(): void; setNoDel export type SpliceCloseInfo = { code: number; reason: string; trigger: string } -// The ws receiver kills a connection whose frame exceeds maxPayload with this -// message; surfacing it separately is what makes catalog-growth kills visible. +// Separate ws maxPayload errors so catalog-growth disconnects are visible. function errorTrigger(side: 'client' | 'host', error: Error): string { return /max payload/i.test(error.message) ? `${side}-oversize-frame` : `${side}-error` } diff --git a/cloud/apps/relay/src/stale-if-error-jwks.test.ts b/cloud/apps/relay/src/stale-if-error-jwks.test.ts index e290b7a6496..26177e4d826 100644 --- a/cloud/apps/relay/src/stale-if-error-jwks.test.ts +++ b/cloud/apps/relay/src/stale-if-error-jwks.test.ts @@ -28,7 +28,6 @@ async function relayToken(key: SigningKey): Promise { .sign(key.privateKey) } -// Auth's JWKS endpoint: serves the current key, or fails like a down service. function auth(initial: SigningKey) { const state = { key: initial, up: true } const fetch = vi.fn(async () => { diff --git a/cloud/apps/relay/src/stale-if-error-jwks.ts b/cloud/apps/relay/src/stale-if-error-jwks.ts index a998b5dd4d6..5f98c7da6c1 100644 --- a/cloud/apps/relay/src/stale-if-error-jwks.ts +++ b/cloud/apps/relay/src/stale-if-error-jwks.ts @@ -16,19 +16,13 @@ export type RelayJWKSet = JWTVerifyGetKey & { reload: () => Promise } -// Auth signs every relay token with one key and serves it with no overlap on rotation, and a -// relay token lives 5 min. Past ~16 min of a full auth outage (10 min cache age plus a token's -// life and the cell's 60 s expiry grace) no token the stale keys could accept is still valid; -// the bound covers that with margin and caps how long a rotated-out (say, leaked) key stays -// trusted on a cell that cannot reach auth, including one an attacker keeps auth from. -// Emergency purge: restarting the process drops the set, and it then fails closed until a fetch -// succeeds (see "Auth outage and host token keys" in cloud/docs/orca-relay-operations.md). +// Auth rotates without key overlap: 10min cache + 5min token + 60s grace expires legitimate stale-key tokens by ~16min. +// 30min caps trust in rotated/leaked keys during outages or isolation; restart purges cache and fails closed until fetch succeeds (orca-relay-operations.md). export const RELAY_JWKS_MAX_STALE_MS = 30 * 60 * 1000 // After a failed fetch, verifies use the stale set without waiting on another fetch's timeout. const FETCH_RETRY_AFTER_MS = 30 * 1000 -// jose's remote set, except that a failed refetch falls back to the last set auth served, for -// a bounded time: an auth outage must not turn every still-valid token into a 4401. +// Bounded stale-key fallback prevents auth outages refusing otherwise valid tokens with 4401. export function createStaleIfErrorJWKSet( url: URL, options: { diff --git a/cloud/apps/relay/src/sticky-slot-statements.ts b/cloud/apps/relay/src/sticky-slot-statements.ts index a7981961c6c..f63bfdf87e3 100644 --- a/cloud/apps/relay/src/sticky-slot-statements.ts +++ b/cloud/apps/relay/src/sticky-slot-statements.ts @@ -11,8 +11,7 @@ export type StickySlotSample = { type StickySlotStatementTotals = Omit -// PostgreSQL round trips (all but ROLLBACK) made while one request holds the sticky -// slot. Closed at release, so work the request left running is not charged to it. +// Count PostgreSQL trips except ROLLBACK while sticky slot is held; exclude work still running after release. export class StickySlotStatements { private closed = false private poolWaitMs = 0 diff --git a/cloud/apps/relay/src/test-fixtures/delayed-postgres-database.ts b/cloud/apps/relay/src/test-fixtures/delayed-postgres-database.ts index eef94c52957..15345be7ba0 100644 --- a/cloud/apps/relay/src/test-fixtures/delayed-postgres-database.ts +++ b/cloud/apps/relay/src/test-fixtures/delayed-postgres-database.ts @@ -9,13 +9,11 @@ import { export type StatementDelay = { enabled: boolean delayMs: number - // Runs before each delayed statement leaves the client, so it sees the locks - // the transaction holds between round trips. + // Observe held locks before each delayed client round trip. beforeTrip?: (sql: string) => Promise } -// The serving pool's session settings with a fixed delay in front of every -// statement, BEGIN and COMMIT included: one delay per cross-region round trip. +// Inject cross-region delay before every statement, including BEGIN/COMMIT, with serving-pool settings. export function openDelayedPostgresDatabase( url: string, delay: StatementDelay, @@ -24,8 +22,7 @@ export function openDelayedPostgresDatabase( const pool = new pg.Pool({ connectionString: url, max: poolMax, - // Same bound as openRelayDatabase, so a saturated pool fails acquires the - // way a serving cell does instead of queueing forever. + // Use serving acquire bound so pool saturation fails instead of queueing forever. connectionTimeoutMillis: 2_000, statement_timeout: POSTGRES_STATEMENT_TIMEOUT_MS, lock_timeout: POSTGRES_LOCK_TIMEOUT_MS, diff --git a/cloud/apps/relay/src/test-fixtures/host-control-proof.ts b/cloud/apps/relay/src/test-fixtures/host-control-proof.ts index 55c8df9315b..fc21204d7f0 100644 --- a/cloud/apps/relay/src/test-fixtures/host-control-proof.ts +++ b/cloud/apps/relay/src/test-fixtures/host-control-proof.ts @@ -16,8 +16,7 @@ import { HostSessionRegistry } from '../host-session-registry.js' import type { RelayTokenClaims } from '../relay-token-verifier.js' import { ProcessQueuedByteBudget } from '../splice-forwarder.js' -// A cell's host session registry over a stubbed database, driven through the real host hello -// and proof with fake sockets. +// Drive real hello/proof against stubbed database and fake sockets. export class ProofSocket extends EventEmitter { readonly OPEN = 1 diff --git a/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts b/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts index 265e5bfaa61..a942e7050da 100644 --- a/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts +++ b/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts @@ -12,8 +12,7 @@ import type { RelayDatabase } from '../database.js' import type { AppliedControlFlags } from '../relay-control-flag-channel.js' import { createRelayServer } from '../relay-server.js' -// One reserve-mode cell over a real database, with desktops that complete the host proof. -// For the flip-back tests (E-mix), on SQLite and on a latency-injected PostgreSQL. +// Real reserve-cell host proof for SQLite and latency-injected PostgreSQL flip-back tests (E-mix). export const RESERVE_MODE_CELL_ID = 'production-gce-c3' @@ -42,7 +41,6 @@ export function nextMessage(socket: WebSocket): Promise> }) } -// The next frame of this type, skipping pings and anything else the socket carries first. export function nextFrame(socket: WebSocket, type: string): Promise> { return new Promise((resolve) => { const onMessage = (data: RawData) => { @@ -131,7 +129,6 @@ export async function startReserveModeCell(input: { return { keyPair, hostId, token } } - // The first control frame after the proof, and the socket's close code once it closes. const connect = async (identity: Awaited>, epoch: number) => { const socket = new WebSocket(`ws://127.0.0.1:${port}/v1/host/control`, { headers: { authorization: `Bearer ${identity.token}` }, @@ -212,7 +209,6 @@ export async function startReserveModeCell(input: { return { identity, ...reply } } - // A paired phone's device row, as an install would have written it. const installDevice = async ( relayHostId: string, resumeToken: string, @@ -230,7 +226,6 @@ export async function startReserveModeCell(input: { ] ) - // A phone's dial: its first relay-hello frame and its close. let phoneSource = 0 const dialPhone = async (relayHostId: string, credential: string) => { phoneSource += 1 @@ -251,7 +246,6 @@ export async function startReserveModeCell(input: { return { socket, hello, closed } } - // The desktop's data leg for a conn-open its control received. const attachData = async (connOpen: Record, generation: unknown) => { const socket = new WebSocket(`ws://127.0.0.1:${port}/v1/host/data/${String(connOpen.connId)}`, { perMessageDeflate: false diff --git a/cloud/apps/relay/src/test-fixtures/statement-recording-database.ts b/cloud/apps/relay/src/test-fixtures/statement-recording-database.ts index bd4290b4cff..d6cea027382 100644 --- a/cloud/apps/relay/src/test-fixtures/statement-recording-database.ts +++ b/cloud/apps/relay/src/test-fixtures/statement-recording-database.ts @@ -5,8 +5,7 @@ import type { SqlRow } from '../database.js' -// Every statement a relay path issues, in order, so its round trips are counted rather than -// described. BEGIN and COMMIT are recorded too: each is a trip to a remote database. +// Count real round trips including BEGIN/COMMIT rather than inferring from query counts. export class StatementRecordingDatabase implements RelayDatabase { readonly dialect: RelayDatabase['dialect'] // A statement this matches fails as a dropped connection would, before reaching the database. diff --git a/cloud/packages/postgres-schema/src/apply-postgres-schema.test.ts b/cloud/packages/postgres-schema/src/apply-postgres-schema.test.ts index 85aaf0f5b76..ab394d37d94 100644 --- a/cloud/packages/postgres-schema/src/apply-postgres-schema.test.ts +++ b/cloud/packages/postgres-schema/src/apply-postgres-schema.test.ts @@ -61,8 +61,7 @@ describe('applyPostgresSchema classification', () => { }) it('classifies a comment-prefixed CREATE TABLE by its own collision codes', async () => { - // pg_type_typname_nsp_index is reached only through the CREATE TABLE branch, so a statement - // misread as unknown would fail the boot on a benign concurrent create instead of retrying. + // CREATE TABLE catalog collision must retry; unknown classification would fail boot on concurrent creates. let calls = 0 const query = vi.fn(async () => { calls += 1 @@ -88,8 +87,7 @@ describe('applyPostgresSchema classification', () => { }) it('treats an already-applied constraint as skipped rather than an error', async () => { - // Still the answer for a caller with no pre-check, and for a constraint another director - // committed between this boot's pre-check and its ALTER TABLE. + // Covers callers without precheck and constraints concurrently committed between precheck and ALTER. const query = vi.fn(async () => { throw postgresError('42710') }) @@ -98,8 +96,7 @@ describe('applyPostgresSchema classification', () => { }) it('treats an index another director already dropped as skipped rather than an error', async () => { - // Every director boots at once on a deploy and all of them send the same DROP INDEX IF EXISTS. - // Only one can win; the losers must not fail their boot over a drop that already happened. + // Concurrent deploy boots race the same DROP; losing directors must not fail after another succeeds. const query = vi.fn(async () => { throw postgresError('42704') }) @@ -109,7 +106,6 @@ describe('applyPostgresSchema classification', () => { }) it('still propagates 42704 from a statement that is not a DROP IF EXISTS', async () => { - // Keeps the case above narrow: an undefined object anywhere else is a real boot failure. const query = vi.fn(async () => { throw postgresError('42704') }) @@ -119,9 +115,7 @@ describe('applyPostgresSchema classification', () => { }) it('leaves a deferrable statement unapplied on a lock timeout instead of failing the boot', async () => { - // The crash loop this prevents: 28 directors reach the same DROP INDEX at once on a table - // under continuous write, all of them time out, and every one restarts to re-queue the same - // DDL behind the same writers. + // 28 directors retrying the same contended DROP after restart would crash-loop and requeue behind writers. const warned: string[] = [] vi.spyOn(console, 'warn').mockImplementation((line: string) => { warned.push(line) @@ -142,8 +136,6 @@ describe('applyPostgresSchema classification', () => { }) it('runs the statements after a deferral, rather than abandoning the boot at that point', async () => { - // A deferral is not a failure, so nothing behind it may be skipped: the schema still has - // tables to create, and a boot that stopped here would come up against a partial schema. const sent: string[] = [] const query = vi.fn(async (statement: string) => { sent.push(statement) @@ -160,8 +152,7 @@ describe('applyPostgresSchema classification', () => { }) it('still fails the boot on a lock timeout for a statement that is not marked deferrable', async () => { - // Keeps the marker meaningful. An unmarked statement retains the old contract: fail once and - // loudly, because retrying parks every writer behind the same queue again. + // Unmarked DDL must fail once: retries queue writers behind the same relation lock again. vi.spyOn(console, 'error').mockImplementation(() => undefined) const query = vi.fn(async () => { throw postgresError('55P03') @@ -172,8 +163,6 @@ describe('applyPostgresSchema classification', () => { }) it('defers only on a lock timeout, not on any other error from a deferrable statement', async () => { - // A deferrable statement is not a statement whose failures stop mattering. A permission error - // is still a boot failure. const query = vi.fn(async () => { throw postgresError('42501') }) @@ -186,7 +175,6 @@ describe('applyPostgresSchema classification', () => { const query = vi.fn(async (_statement: string) => undefined) const { catalogQuery, asked } = catalogAnswers([]) const summary = await applyPostgresSchema(['DROP INDEX IF EXISTS i'], query, { catalogQuery }) - // One parameter, the index name: the statement names no table, and the query references no $2. expect(asked).toEqual([[expect.stringContaining("relkind = 'i'"), 'i']]) expect(query).not.toHaveBeenCalled() expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 }) @@ -242,8 +230,7 @@ describe('applyPostgresSchema lock timeouts', () => { }) it('still retries a lock timeout for a caller that opts in', async () => { - // A caller with no catalog pre-check learns nothing from a lock timeout about whether the - // object exists, so its old bounded retry is the correct behaviour there. + // Without catalog precheck, lock timeout does not prove existence; preserve bounded retry. vi.spyOn(console, 'warn').mockImplementation(() => undefined) let calls = 0 const query = vi.fn(async () => { @@ -275,8 +262,7 @@ describe('applyPostgresSchema catalog pre-check', () => { }) it('skips an index the catalog reports as invalid rather than rebuilding it', async () => { - // A cancelled CREATE INDEX CONCURRENTLY leaves exactly this state, and IF NOT EXISTS skips it - // too, so reading indisvalid as a condition would newly take the lock it used to avoid. + // IF NOT EXISTS skips invalid cancelled builds too; indisvalid must not trigger new table locks. const logged: { event?: string }[] = [] vi.spyOn(console, 'log').mockImplementation((line: string) => { logged.push(JSON.parse(line)) @@ -355,8 +341,7 @@ describe('applyPostgresSchema catalog pre-check', () => { }) it('skips a DROP CONSTRAINT IF EXISTS when the constraint is already gone', async () => { - // Inverse polarity: an absent constraint is what means there is nothing to drop. Sending it - // anyway takes ACCESS EXCLUSIVE to discover the same thing. + // Absent DROP target means skip, avoiding ACCESS EXCLUSIVE just to rediscover absence. const logged: { event?: string }[] = [] vi.spyOn(console, 'log').mockImplementation((line: string) => { logged.push(JSON.parse(line)) @@ -407,8 +392,7 @@ describe('applyPostgresSchema catalog pre-check', () => { describe('applyPostgresSchema concurrent creates', () => { it('re-asks the catalog on a collision instead of retrying the CREATE INDEX', async () => { - // Another director created the index between the pre-check and this statement. Retrying would - // take SHARE on the table again for an object that is already there. + // After concurrent creation, re-ask catalog instead of re-taking SHARE for an existing index. vi.spyOn(console, 'log').mockImplementation(() => undefined) const query = vi.fn(async (_statement: string) => { throw postgresError('42P07') @@ -476,8 +460,7 @@ describe('applyPostgresSchema unparseable statements', () => { describe('applyPostgresSchema statement text', () => { it('sends the original statement, comments included, not the classified form', async () => { - // Classification reads a comment-free copy. Rewriting what the server runs would change the - // DDL itself, and a comment inside a string literal or a quoted name is part of the statement. + // Classify a comment-free copy; server must receive original SQL, including quoted comment-like text. const statement = `ALTER TABLE t ADD /* note */ COLUMN c TEXT DEFAULT '-- keep'` const query = vi.fn(async (_sql: string) => undefined) const { catalogQuery, asked } = catalogAnswers([]) diff --git a/cloud/packages/postgres-schema/src/apply-postgres-schema.ts b/cloud/packages/postgres-schema/src/apply-postgres-schema.ts index 33490ab1147..b569c6d24b6 100644 --- a/cloud/packages/postgres-schema/src/apply-postgres-schema.ts +++ b/cloud/packages/postgres-schema/src/apply-postgres-schema.ts @@ -41,20 +41,15 @@ const CREATE_INDEX_IF_NOT_EXISTS = /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+E const ALTER_TABLE_ADD_CONSTRAINT = /^ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i const DROP_INDEX_IF_EXISTS = /^DROP\s+INDEX\s+(?:CONCURRENTLY\s+)?IF\s+EXISTS\b/i -// Marked in the schema text, beside the SQL it applies to, and read from the raw statement because -// classification strips comments. Says: this boot may leave the statement unapplied rather than -// fail. Only sound for a statement that is idempotent AND that nothing this boot goes on to do -// depends on, because the database is then simply as it was and the next boot re-sends it. +// Leading schema-deferrable marker permits deferral only for idempotent DDL with no later boot dependency. +// Read raw text because classification strips comments; next boot retries unapplied work. const DEFERRABLE = /^\s*--[^\n]*\bschema-deferrable\b/ export function schemaDeferrable(statement: string): boolean { return DEFERRABLE.test(statement) } -// `IF NOT EXISTS` only checks the name before the catalog inserts, so the loser of a concurrent -// CREATE can fail on the catalog unique index (23505) or, when the winner has already committed by -// the time the loser reaches TypeCreate/heap_create_with_catalog, on the name check those routines -// repeat (42710 duplicate type, 42P07 duplicate relation). Each is a no-op on the next attempt. +// Concurrent CREATE losers can hit catalog 23505 or repeated name-check 42710/42P07; the next attempt is a no-op. function concurrentCreateCollision( value: { code?: unknown; constraint?: unknown }, sql: string @@ -81,10 +76,7 @@ function constraintAlreadyApplied(error: unknown, sql: string): boolean { ) } -// `IF EXISTS` resolves the name, then locks; between those two steps another director's drop can -// commit and the loser raises 42704 instead of the notice it would have got a moment later. Every -// director boots at once on a deploy, so without this the losers fail their boot over a drop that -// already happened. +// IF EXISTS resolves before locking; another director’s DROP can commit between them and produce benign 42704. function dropAlreadyApplied(error: unknown, sql: string): boolean { return DROP_INDEX_IF_EXISTS.test(sql) && (error as { code?: unknown } | null)?.code === '42704' } @@ -94,8 +86,7 @@ function retryableSchemaError(error: unknown, sql: string): boolean { return RETRYABLE_SCHEMA_CODES.has(String(value.code)) || concurrentCreateCollision(value, sql) } -// Evaluated immediately before each statement, so a pre-check still sees the objects the statements -// ahead of it created in this same boot. +// Precheck immediately before each statement so prior creates in this boot are visible. async function nothingToDo( target: SchemaLockTarget | undefined, options: SchemaStartupOptions, @@ -150,14 +141,9 @@ export async function applyPostgresSchema( break } const code = String((error as { code?: unknown } | null)?.code) - // With the pre-check ahead of it a lock timeout means the object is genuinely missing and - // this boot lost the queue. Relation locks are granted in queue order, so each retry parks - // every writer behind it again for another timeout. Fail once, loudly. + // Prechecked missing objects lost the lock queue; retries would park writers for another timeout, so fail once. if (code === LOCK_NOT_AVAILABLE && !options.retryLockTimeout) { - // A deferrable statement yields the queue instead of crash-looping the instance. Every - // director boots at once on a migration, so a table under continuous write can hand the - // whole fleet a lock timeout on the one statement that has to win once; failing the boot - // for it restarts the instance, which re-queues the same DDL behind the same writers. + // Deferrable DDL yields instead of restarting every director into the same continuously written table’s lock queue. if (schemaDeferrable(statement)) { console.warn( JSON.stringify({ @@ -182,9 +168,7 @@ export async function applyPostgresSchema( ) throw error } - // The object was created between the pre-check and this statement. Re-asking the catalog - // is the cheap answer; retrying the CREATE INDEX would take SHARE on the table again for - // an object that is already there. + // Re-ask catalog after concurrent creation; retrying CREATE INDEX re-takes SHARE for an existing object. if ( concurrentCreateCollision((error as { code?: unknown; constraint?: unknown }) ?? {}, sql) && (await nothingToDo(target, options, eventPrefix)) diff --git a/cloud/packages/postgres-schema/src/catalog-object-precheck.ts b/cloud/packages/postgres-schema/src/catalog-object-precheck.ts index 6f923cb4cd2..60296e97574 100644 --- a/cloud/packages/postgres-schema/src/catalog-object-precheck.ts +++ b/cloud/packages/postgres-schema/src/catalog-object-precheck.ts @@ -8,11 +8,8 @@ export type SchemaCatalogQuery = ( params: unknown[] ) => Promise -// The index name is resolved inside the table's own namespace, and `i.indrelid = t.oid` ties it to -// this table: index names are unique per schema, not per table, so without that condition a -// same-named index on a sibling table answers yes and the real index is skipped forever. -// `to_regclass` returns NULL rather than erroring when the table does not exist yet, which is the -// whole of a cold start. +// Resolve index in table’s schema and bind indrelid or a sibling’s same-named index permanently suppresses creation. +// to_regclass returns NULL for absent cold-start tables. const INDEX_PRESENT = `SELECT i.indisvalid FROM pg_catalog.pg_class t JOIN pg_catalog.pg_class c ON c.relnamespace = t.relnamespace AND c.relname = $2 JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid AND i.indrelid = t.oid @@ -21,22 +18,15 @@ WHERE t.oid = to_regclass($1)` const COLUMN_PRESENT = `SELECT 1 FROM pg_catalog.pg_attribute WHERE attrelid = to_regclass($1) AND attname = $2 AND attnum > 0 AND NOT attisdropped` -// Name only. The CHECK body is generated from RELAY_REGIONS, so comparing it would re-run the swap -// on every region change, and an ADD CONSTRAINT is the one statement here that scans the table. +// Compare constraint name only; region-derived CHECK changes must not trigger table scans on every region update. const CONSTRAINT_PRESENT = `SELECT 1 FROM pg_catalog.pg_constraint WHERE conrelid = to_regclass($1) AND conname = $2` -// By name through the search_path, with no table condition, because a DROP INDEX has no table to -// condition on and does not need one: a name that resolves to no visible index is nothing to drop. -// `relkind = 'i'` keeps a same-named table or view from answering for an index. Partitioned indexes -// are 'I', which this deliberately does not match - relay has none, and dropping one is not a -// boot-time operation. +// DROP INDEX resolves through search_path without a table; match relkind i to exclude tables/views and unsupported partitioned indexes (I). const INDEX_BY_NAME_PRESENT = `SELECT 1 FROM pg_catalog.pg_class c WHERE c.relname = $1 AND c.relkind = 'i' AND pg_catalog.pg_table_is_visible(c.oid)` -// reloptions is a text[] of `name=value` pairs, absent entirely while the option is at its -// default. Comparing the whole pair is what makes a changed value re-run: `@>` on a different -// value answers no, and the statement runs and overwrites it. +// Compare complete reloption name=value pair so changed values reapply; defaults have no reloptions entry. const RELOPTION_PRESENT = `SELECT 1 FROM pg_catalog.pg_class WHERE oid = to_regclass($1) AND reloptions @> ARRAY[$2]` @@ -50,16 +40,13 @@ const PRESENCE_SQL = { export type SchemaCatalogPresence = { present: boolean; indisvalid: unknown } -// Row presence is the answer, whatever the row says. An index left invalid by a cancelled -// concurrent build is skipped by `IF NOT EXISTS` today as well, so reading `indisvalid` as a -// condition would newly take the lock for exactly the indexes a failed build left behind. +// Existence alone matches IF NOT EXISTS, even for invalid cancelled builds; checking validity would take new locks. export async function catalogObjectPresence( query: SchemaCatalogQuery, target: SchemaLockTarget ): Promise { const sql = PRESENCE_SQL[target.kind] - // The name-only lookup binds one parameter; every other shape binds the table first. Passing a - // parameter the SQL never references is a bind error, not a harmless extra. + // Bind only referenced parameters; the table-less lookup has one, and unused extras cause bind errors. const params = target.kind === 'index-by-name' ? [target.name] : [target.table, target.name] const rows = await query(sql, params) const row = rows[0] diff --git a/cloud/packages/postgres-schema/src/schema-lock-target.test.ts b/cloud/packages/postgres-schema/src/schema-lock-target.test.ts index 80d2714bd2c..e3a95034645 100644 --- a/cloud/packages/postgres-schema/src/schema-lock-target.test.ts +++ b/cloud/packages/postgres-schema/src/schema-lock-target.test.ts @@ -7,8 +7,7 @@ import { takesRelationLock } from './schema-lock-target.js' -// The shape a schema string split on ';' actually produces: the comment written above a statement -// arrives glued to the front of it. +// Semicolon splitting glues preceding SQL comments onto the next statement. const COMMENTED_INDEX = `-- Why: the maintenance sweep matches (active, deadline) while inactive -- bases accumulate unboundedly. CREATE INDEX IF NOT EXISTS relay_connection_bases_active_deadline @@ -61,9 +60,7 @@ describe('sqlWithoutComments', () => { }) describe('comments between keywords', () => { - // Before this, the classification regexes and the must-parse shapes both needed `ADD COLUMN` - // contiguous, so this statement derived NO target and threw NOTHING: the DDL ran with no - // pre-check, taking ACCESS EXCLUSIVE on every boot. + // Inter-keyword comments once bypassed classification and must-parse checks, taking ACCESS EXCLUSIVE every boot. it('derives a column target through a comment between ADD and COLUMN', () => { expect(requireSchemaLockTarget('ALTER TABLE t ADD /* note */ COLUMN c TEXT')).toEqual({ kind: 'column', @@ -252,7 +249,6 @@ describe('schemaLockTarget', () => { }) it('skips a DROP CONSTRAINT IF EXISTS when the constraint is already gone', () => { - // The inverse polarity: nothing to drop is nothing to do. expect(schemaLockTarget('ALTER TABLE t DROP CONSTRAINT IF EXISTS c')).toEqual({ kind: 'constraint', table: 't', @@ -279,8 +275,7 @@ describe('schemaLockTarget', () => { }) }) -// Each of these reads as an index or column statement and each fails to yield a target. Letting any -// of them through would send an unchecked lock-taking statement on every boot. +// Malformed lock-taking shapes must fail instead of silently joining the lock queue every boot. const MALFORMED = [ ['an index with no ON clause', 'CREATE INDEX IF NOT EXISTS i'], ['an auto-named index', 'CREATE INDEX ON t(c)'], @@ -321,8 +316,7 @@ describe('requireSchemaLockTarget', () => { describe('multi-action ALTER TABLE', () => { it('throws rather than deriving only the first subcommand', () => { - // Deriving `a` and skipping on it would drop `b` for the life of the database, and the first - // subcommand parses fine, so nothing else here would catch it. + // Skipping on the first subcommand would permanently drop every later action. const statement = 'ALTER TABLE t ADD COLUMN IF NOT EXISTS a TEXT, ADD COLUMN IF NOT EXISTS b TEXT' expect(schemaLockTarget(statement)).toEqual({ @@ -434,24 +428,21 @@ describe('schemaLockTarget storage parameters', () => { }) it('makes a changed value a different target, so it re-runs instead of skipping', () => { - // The failure this prevents: matching on the option name alone would read `fillfactor=100` as - // already satisfying `fillfactor = 70` and skip the statement for the life of the database. + // Comparing name alone would treat fillfactor=100 as satisfying 70 forever. const seventy = schemaLockTarget('ALTER TABLE t SET (fillfactor = 70)') const eighty = schemaLockTarget('ALTER TABLE t SET (fillfactor = 80)') expect(seventy?.name).not.toBe(eighty?.name) }) it('refuses a multi-option SET rather than skipping on only the first option', () => { - // Same reason a multi-action ALTER TABLE is refused: skipping on one option would silently - // drop the others for good. + // Skipping on one option would permanently lose every other change. expect(() => requireSchemaLockTarget('ALTER TABLE t SET (fillfactor = 70, autovacuum_enabled = false)') ).toThrow(/unparsed_schema_lock_target/) }) it('fails the boot on a SET whose shape it cannot read, rather than sending it unchecked', () => { - // A storage parameter takes a relation lock, so no target means the lock is taken on every - // boot. RESET has no value to compare and is not supported. + // Unparsed SET would take a relation lock every boot; RESET has no comparable value and is outside precheck support. expect(() => requireSchemaLockTarget('ALTER TABLE t RESET (fillfactor)')).not.toThrow() expect(() => requireSchemaLockTarget('ALTER TABLE t SET (fillfactor)')).toThrow( /unparsed_schema_lock_target/ @@ -486,8 +477,7 @@ describe('schemaLockTarget dropped indexes', () => { }) it('requires IF EXISTS, so a bare DROP fails the boot instead of running unchecked', () => { - // Same contract as DROP CONSTRAINT: a bare DROP on a missing index is an error the server is - // supposed to raise, and a pre-check that skipped it would swallow that. + // Skipping a bare DROP would swallow the server’s required missing-object error. expect(() => requireSchemaLockTarget('DROP INDEX i')).toThrow(/unparsed_schema_lock_target/) }) @@ -499,8 +489,7 @@ describe('schemaLockTarget dropped indexes', () => { }) it('derives the target through a leading deferrable marker', () => { - // The real shape in relay's schema: the marker is a comment, so classification must see past - // it or the statement would reach the server with no pre-check at all. + // Real schema markers lead statements; classifier must strip them without losing target checks. const statement = '-- schema-deferrable: reason\nDROP INDEX IF EXISTS i' expect(sqlWithoutComments(statement)).toBe('DROP INDEX IF EXISTS i') expect(schemaLockTarget(statement)?.name).toBe('i') diff --git a/cloud/packages/postgres-schema/src/schema-lock-target.ts b/cloud/packages/postgres-schema/src/schema-lock-target.ts index 8a96dc5fb1d..e85d4a1309e 100644 --- a/cloud/packages/postgres-schema/src/schema-lock-target.ts +++ b/cloud/packages/postgres-schema/src/schema-lock-target.ts @@ -1,35 +1,21 @@ -// The object a boot-time DDL statement locks on Postgres, so the catalog can be asked whether it -// already exists before the statement joins the lock queue. `table` is kept exactly as the -// statement wrote it, schema qualification and quoting included, because it is fed to -// `to_regclass`; `name` is the bare identifier the catalog stores in `relname`/`attname`. +// Precheck catalog before boot DDL joins lock queue; table retains schema/quoting for to_regclass, name is the bare catalog identifier. export type SchemaLockTarget = | { kind: 'index' | 'column' | 'constraint' | 'reloption' table: string name: string - // The catalog answer that means this statement has nothing left to do. Creating statements - // skip on present; `DROP CONSTRAINT IF EXISTS` is the inverse, because nothing to drop is - // done. + // Creates skip when present; IF EXISTS drops skip when absent. skipWhen: 'present' | 'absent' } - // A `DROP INDEX` names no table, and needs none: an index name that resolves to nothing is - // nothing to drop, whatever table it used to belong to. Resolution is by name through the - // search_path, which is how the DROP itself would resolve it. + // DROP INDEX has no table: resolve the name through search_path as PostgreSQL does. | { kind: 'index-by-name'; name: string; skipWhen: 'absent' } -// Keywords that sit in an identifier position when the optional clause before them is absent. -// Without this, `CREATE UNIQUE INDEX CONCURRENTLY ON t(c)` reads CONCURRENTLY as the index name and -// `ADD COLUMN IF NOT EXISTS` with no column reads IF as the column: a silently wrong target, which -// is worse than no target. Excluding them makes both throw instead. A column genuinely named `if` -// has to be quoted to be derivable, which is the safe direction to fail in. +// Reject keywords masquerading as absent identifiers (CONCURRENTLY, IF); genuine keyword names must be quoted to fail safely. const NOT_KEYWORD = '(?!(?:CONCURRENTLY|IF|NOT|EXISTS|ON|ONLY)\\b)' const IDENTIFIER = `"(?:[^"]|"")*"|${NOT_KEYWORD}[A-Za-z_][A-Za-z0-9_$]*` const QUALIFIED = `((?:${IDENTIFIER})(?:\\.(?:${IDENTIFIER}))?)` -// `$$...$$` and `$tag$...$tag$` are opaque: a comment marker, comma, parenthesis or bracket inside -// one is text. The closing delimiter must match the opening tag exactly, so an inner `$$` inside a -// `$tag$` body is more text rather than the end. The tag cannot start with a digit, which is what -// keeps a `$1` placeholder from reading as an opener. +// Dollar-quoted bodies are opaque; exact matching tags keep inner $$ as text, and nonnumeric starts exclude $1 placeholders. const DOLLAR_QUOTE = /\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/y function dollarQuoteEnd(sql: string, index: number): number | undefined { @@ -40,12 +26,8 @@ function dollarQuoteEnd(sql: string, index: number): number | undefined { return close === -1 ? sql.length : close + opener.length } -// Every comment, not only the block a ';'-split schema glues above a statement. A comment between -// two keywords (`ADD /* note */ COLUMN`) is invisible to the classification regexes AND to the -// must-parse shapes, so it used to yield no target and no throw: the statement ran with no -// pre-check at all, which is the one direction this must never fail in. Postgres treats a comment -// as whitespace, so each becomes a single space. Only classification reads this; the server is -// always sent the original text. +// Replace all comments with whitespace for classification/must-parse checks, including between keywords. +// Otherwise DDL can silently bypass prechecks; send original text to the server. export function sqlWithoutComments(statement: string): string { let stripped = '' let quote: string | undefined @@ -116,43 +98,32 @@ const ADD_CONSTRAINT = new RegExp( `ADD\\s+CONSTRAINT\\s+${QUALIFIED}`, 'i' ) -// `IF EXISTS` is required, not optional. A bare `DROP CONSTRAINT` on a missing constraint is an -// error the server is supposed to raise, and skipping it would swallow that. Without a target the -// statement throws at boot instead, which tells the author to write `IF EXISTS`. +// Require IF EXISTS or a precheck skip would swallow the missing-constraint error; bare DROP fails boot. const DROP_CONSTRAINT = new RegExp( `^ALTER\\s+TABLE\\s+(?:IF\\s+EXISTS\\s+)?(?:ONLY\\s+)?${QUALIFIED}\\s+` + `DROP\\s+CONSTRAINT\\s+IF\\s+EXISTS\\s+${QUALIFIED}`, 'i' ) -// `IF EXISTS` is required for the same reason it is on DROP CONSTRAINT: a bare `DROP INDEX` on a -// missing index is an error the server is supposed to raise. Without a target the statement throws -// at boot instead, which tells the author to write `IF EXISTS`. +// Require IF EXISTS to preserve missing-index errors; bare DROP fails boot instead of being prechecked away. const DROP_INDEX = new RegExp(`^DROP\\s+INDEX\\s+(?:CONCURRENTLY\\s+)?IF\\s+EXISTS\\s+${QUALIFIED}\\s*$`, 'i') -// One option per statement, and a literal value: the catalog stores reloptions as `name=value` -// text, so the pre-check compares the written pair against that array verbatim. A list of options -// is refused by `hasTopLevelComma` before it reaches here, the same as a multi-action ALTER TABLE. +// Compare one literal reloption as catalog name=value; multi-option statements cannot be safely skipped together. const SET_RELOPTION = new RegExp( `^ALTER\\s+TABLE\\s+(?:IF\\s+EXISTS\\s+)?(?:ONLY\\s+)?${QUALIFIED}\\s+` + `SET\\s+\\(\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*=\\s*([A-Za-z0-9_.]+)\\s*\\)\\s*$`, 'i' ) -// Every statement shape that takes a relation lock before Postgres evaluates its existence test. -// `CREATE TABLE IF NOT EXISTS` is absent on purpose: it resolves a name against the schema and -// takes no lock on an existing table. -// `DROP INDEX` is here because it takes ACCESS EXCLUSIVE on the index's table whenever the index is -// actually there, which is every boot until the first one wins. That it takes no lock once the -// index is gone is what the pre-check turns into the steady state, not a reason to omit it. +// CREATE TABLE takes no lock on existing tables; ALTER/CREATE INDEX/DROP INDEX can. +// Include DROP while its index exists; catalog precheck supplies the no-lock steady state once absent. const TAKES_RELATION_LOCK = /^(?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX|ALTER\s+TABLE)\b/i export function takesRelationLock(statement: string): boolean { return TAKES_RELATION_LOCK.test(sqlWithoutComments(statement)) } -// Splitting on '.' is not enough: `"a.b"` is one identifier containing a dot, not two parts. Each -// part is read quote-aware, with a doubled quote unescaped to one. +// Parse quoted dotted names whole; doubled quotes escape literal quotes. function qualifiedParts(written: string): { text: string; quoted: boolean }[] { const parts: { text: string; quoted: boolean }[] = [] let text = '' @@ -184,18 +155,14 @@ function qualifiedParts(written: string): { text: string; quoted: boolean }[] { return parts } -// Postgres folds an unquoted identifier to lower case before storing it, so `Foo` is `foo` in -// relname, attname and conname. Comparing the written case would miss the row and rebuild the -// object on every boot. +// Fold unquoted names like PostgreSQL or written case misses catalog rows and rebuilds every boot. function catalogName(written: string): string { const last = qualifiedParts(written).pop() if (!last) return written return last.quoted ? last.text : last.text.toLowerCase() } -// Shapes whose lock target the pre-check must be able to derive. Deliberately looser than the -// regexes that parse them, so a statement that reads as one of these but does not parse is caught -// rather than falling through to the lock path. +// Broader than target parsers so malformed lock-taking shapes fail rather than bypass precheck. const MUST_PARSE = [ /^CREATE\s+(?:UNIQUE\s+)?INDEX\b/i, /^ALTER\s+TABLE\b[\s\S]*\bADD\s+COLUMN\b/i, @@ -240,9 +207,7 @@ export function schemaLockTarget(statement: string): SchemaLockTarget | undefine } const option = SET_RELOPTION.exec(sql) if (option?.[1] && option[2] && option[3]) { - // Option names are always folded, but the value is stored as written, so only the name goes - // through catalogName. `fillfactor=70` and `fillfactor=80` are different targets, which is - // what makes a changed value re-run rather than skip. + // Fold option name but preserve value; changed name=value pairs must reapply. return { kind: 'reloption', table: option[1], @@ -255,9 +220,7 @@ export function schemaLockTarget(statement: string): SchemaLockTarget | undefine const ALTER_TABLE = /^ALTER\s+TABLE\b/i -// A comma that separates ALTER TABLE subcommands rather than sitting inside a type, a default, a -// CHECK body or a dollar-quoted body. Takes comment-free SQL. Square brackets count as depth too, -// or an array type or `DEFAULT ARRAY[1, 2]` reads as a second subcommand and fails the boot. +// Detect only top-level commas in comment-free SQL; types/defaults/CHECK/dollar bodies and ARRAY brackets can contain nested commas. function hasTopLevelComma(sql: string): boolean { let depth = 0 let quote: string | undefined @@ -284,15 +247,10 @@ function hasTopLevelComma(sql: string): boolean { return false } -// An index or column statement whose target cannot be read is the dangerous case: it would be sent -// unchecked and take the lock the pre-check exists to avoid, silently and on every boot. An -// auto-named `CREATE INDEX ON t(c)` lands here too, because nothing in the text says what the -// catalog will call it. Fail the boot with the statement instead. +// Unparsed lock targets, including auto-named indexes, must fail boot rather than lock unchecked every time. export function requireSchemaLockTarget(statement: string): SchemaLockTarget | undefined { const sql = sqlWithoutComments(statement) - // A multi-action ALTER TABLE parses to its FIRST subcommand's target only, so skipping on that - // one object would silently drop every later action for the life of the database. One action per - // statement, or no pre-check is possible. + // One ALTER action per statement: skipping on first target would permanently lose later actions. if (ALTER_TABLE.test(sql) && hasTopLevelComma(sql)) { throw new Error(`unparsed_schema_lock_target: ${sql}`) } diff --git a/cloud/packages/push-contract/src/push-limits.ts b/cloud/packages/push-contract/src/push-limits.ts index 0a4b9b25866..c68298b5a2a 100644 --- a/cloud/packages/push-contract/src/push-limits.ts +++ b/cloud/packages/push-contract/src/push-limits.ts @@ -2,8 +2,7 @@ export const PUSH_LIMITS = { titleMaxChars: 80, bodyMaxChars: 180, maxRegistrationIdsPerSend: 20, - // A host pairs phones, not a fleet. The cap bounds what one session can write - // through a caller-chosen deviceId. + // Bound writes through caller-chosen deviceId to phone-scale use per host. maxDevicesPerHost: 64, maxHttpBodyBytes: 16 * 1024, hostEventsPerWindow: 300, @@ -13,8 +12,7 @@ export const PUSH_LIMITS = { clockSkewToleranceMs: 30_000, sessionTtlMs: 24 * 60 * 60 * 1000, notificationTtlSeconds: 5 * 60, - // The challenge and session routes are the only unauthenticated writes, so - // they are capped per client IP before any key material is generated. + // Rate-limit unauthenticated challenge/session writes before generating keys. unauthenticatedRequestsPerMinutePerIp: 30, authenticatedRequestsPerMinutePerIp: 6_000, authenticatedRequestsPerMinutePerHost: 600 diff --git a/cloud/packages/push-contract/src/send-messages.ts b/cloud/packages/push-contract/src/send-messages.ts index 61e4c1b9166..6114cdf68d0 100644 --- a/cloud/packages/push-contract/src/send-messages.ts +++ b/cloud/packages/push-contract/src/send-messages.ts @@ -8,9 +8,7 @@ import { OpaqueIdSchema, SequenceSchema } from './wire-scalars.js' export const PushNotificationSchema = z .object({ - // Absent for terminal-bell, which the desktop raises without a notification record. - // Printable ASCII only: the id becomes the APNs collapse header, and the - // desktop builds it from URL-encoded parts, so anything else is not Orca's. + // Absent for terminal-bell; printable ASCII only because desktop URL-encoded IDs become APNs collapse headers. notificationId: z .string() .min(1) diff --git a/cloud/packages/push-contract/src/wire-scalars.ts b/cloud/packages/push-contract/src/wire-scalars.ts index f190aef8354..2d6f63f1c89 100644 --- a/cloud/packages/push-contract/src/wire-scalars.ts +++ b/cloud/packages/push-contract/src/wire-scalars.ts @@ -1,7 +1,6 @@ import { z } from 'zod' -// Copied from relay-contract rather than imported: the push gateway ships as a -// standalone image and must not pull the relay wire contract into its closure. +// Copy relay scalars to keep standalone push images independent of the relay wire contract. export const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/) export const Base6432ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){10}[A-Za-z0-9+/]{3}=$/) export const Base64Raw24ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){8}$/) diff --git a/cloud/packages/relay-contract/src/admission-budgets.ts b/cloud/packages/relay-contract/src/admission-budgets.ts index 6dcf18ec3af..e3c87bf7b1f 100644 --- a/cloud/packages/relay-contract/src/admission-budgets.ts +++ b/cloud/packages/relay-contract/src/admission-budgets.ts @@ -8,8 +8,7 @@ export const RELAY_ADMISSION_BUDGETS = { maxProcessQueuedBytes: 64 * 1024 * 1024, spliceLowWaterBytes: 64 * 1024, spliceHighWaterBytes: 256 * 1024, - // Why: must admit at least one maxFrameBytes frame for a backpressured - // peer, or large catalog responses close the splice with LIMIT_EXCEEDED. + // Queue at least one maxFrameBytes frame or large catalogs close backpressured splices with LIMIT_EXCEEDED. spliceHardQueuedBytes: 8 * 1024 * 1024 + 256 * 1024, spliceWedgedTimeoutMs: 10 * 1000 } as const diff --git a/cloud/packages/relay-contract/src/close-codes.ts b/cloud/packages/relay-contract/src/close-codes.ts index 12c125e3fca..4044f395a72 100644 --- a/cloud/packages/relay-contract/src/close-codes.ts +++ b/cloud/packages/relay-contract/src/close-codes.ts @@ -9,12 +9,9 @@ export const RELAY_CLOSE_CODE = { export type RelayCloseCode = (typeof RELAY_CLOSE_CODE)[keyof typeof RELAY_CLOSE_CODE] -// Mirror of MOBILE_RELAY_PHONE_CLOSE_REASON in src/shared/mobile-relay-close-codes.ts -// in the Orca app repo half. Close reasons on a phone socket that refine its close -// code; not RELAY_HOST_CLOSE_REASON, which names host control-socket closes. +// Mirror src/shared/mobile-relay-close-codes.ts; these refine phone close codes, while RELAY_HOST_CLOSE_REASON describes host controls. export const RELAY_PHONE_CLOSE_REASON = { - // A HOST_OFFLINE sent while the host's control looked alive but its data link - // missed hostAttachDeadlineMs; the phone redials the same cell once, quickly. + // Live control with missed hostAttachDeadlineMs: phone retries same cell once quickly. HOST_ATTACH_TIMEOUT: 'attach-timeout' } as const diff --git a/cloud/packages/relay-contract/src/contract.test.ts b/cloud/packages/relay-contract/src/contract.test.ts index 60a99c043c1..767f30a2ee3 100644 --- a/cloud/packages/relay-contract/src/contract.test.ts +++ b/cloud/packages/relay-contract/src/contract.test.ts @@ -337,7 +337,6 @@ describe('pending connection details capability', () => { pendingConns: [{ ...identifiers, kind: 'resume', relayDeviceId: 'device-1' }] }).success ).toBe(true) - // Still strict otherwise: an unannounced key must not slip through as data. expect( HostHelloAckSchema.safeParse({ ...ack, @@ -350,7 +349,6 @@ describe('pending connection details capability', () => { expect(parseRelayHostCapabilities(' pending-conn-details , future-thing')).toEqual( new Set(['pending-conn-details', 'future-thing']) ) - // A host that predates the header sends nothing; absence is never capable. expect(parseRelayHostCapabilities(undefined).size).toBe(0) expect(parseRelayHostCapabilities('').size).toBe(0) expect(parseRelayHostCapabilities('x'.repeat(65)).size).toBe(0) diff --git a/cloud/packages/relay-contract/src/control-messages.ts b/cloud/packages/relay-contract/src/control-messages.ts index 976f069f5e4..c7b74de118a 100644 --- a/cloud/packages/relay-contract/src/control-messages.ts +++ b/cloud/packages/relay-contract/src/control-messages.ts @@ -44,14 +44,11 @@ export const HostChallengeAckSchema = z .object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema }) .strict() -// Advertised on the control upgrade rather than in host-hello: HostHelloSchema -// is strict, so a new hello key is refused by every already-deployed cell. +// Advertise via upgrade header: strict deployed HostHelloSchema rejects new hello keys. export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities' -// The director's `/v1/assign` lease, echoed on the control upgrade. Optional: cells -// that predate it ignore unknown headers, and today a cell only shadow-checks it. +// Echo director assignment lease on upgrade; older cells ignore unknown headers, current cells shadow-check. export const RELAY_ASSIGNMENT_LEASE_HEADER = 'x-orca-relay-assignment-lease' -// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does -// not advertise this parses those entries strictly and would drop the whole ack. +// Advertise before sending pendingConns details: legacy strict parsers would reject the entire ack. export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details' export const RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME = 'idle-regional-rehome-v1' @@ -68,8 +65,7 @@ export function parseRelayHostCapabilities( ) } -// kind/relayDeviceId are optional so an entry stays readable by a host that -// predates them; the cell only emits them to a host that advertised support. +// Optional details support older cells; emit only to hosts advertising support. const PendingConnectionSchema = z .object({ connId: OpaqueIdSchema, diff --git a/cloud/packages/relay-contract/src/director-messages.ts b/cloud/packages/relay-contract/src/director-messages.ts index 0f57081014a..9b44ccb4c07 100644 --- a/cloud/packages/relay-contract/src/director-messages.ts +++ b/cloud/packages/relay-contract/src/director-messages.ts @@ -21,8 +21,7 @@ export const AssignmentRequestSchema = z .object({ v: z.literal(1), relayHostId: RelayHostIdSchema, - // Client-declared reconnection; the director verifies it against the - // durable assignment before granting fast-lane admission. + // Verify client-declared reconnect against durable assignment before fast-lane admission. reconnect: z.boolean().optional(), preferredRegion: RelayRegionSchema.optional(), regionCorrection: RegionCorrectionRequestSchema.optional() diff --git a/cloud/packages/relay-contract/src/host-close-reason.ts b/cloud/packages/relay-contract/src/host-close-reason.ts index 3a5abde3f00..3258d943790 100644 --- a/cloud/packages/relay-contract/src/host-close-reason.ts +++ b/cloud/packages/relay-contract/src/host-close-reason.ts @@ -1,8 +1,5 @@ -// Mirror of src/shared/relay-host-close-reason.ts in the Orca app repo half. -// A host control socket may close with one of these as its WebSocket close -// reason; the cell records it so a later phone rejection can name the cause. -// Anything else (including the empty reason of an abrupt 1006) means "unknown", -// which is what every peer that predates this file sends. +// Mirror src/shared/relay-host-close-reason.ts; record host socket reasons for later phone refusals. +// Unknown/empty reasons, including abrupt 1006 and older peers, mean unknown. export const RELAY_HOST_CLOSE_REASON = { SIGNED_OUT: 'signed-out' } as const diff --git a/cloud/packages/relay-contract/src/idle-regional-rehome.ts b/cloud/packages/relay-contract/src/idle-regional-rehome.ts index cb148e06b1c..c8d6c51f637 100644 --- a/cloud/packages/relay-contract/src/idle-regional-rehome.ts +++ b/cloud/packages/relay-contract/src/idle-regional-rehome.ts @@ -15,9 +15,7 @@ export const IdleRegionalRehomeRequestSchema = z }) .strict() -// Deferrals no other candidate in the same poll can get past: the source -// re-reads the same durable row for every request, so the next POST takes the -// same branch. The director stops walking its list on one of these. +// Global deferrals repeat the same durable row verdict for every candidate; stop this poll’s POST walk. export const GLOBAL_IDLE_REGIONAL_REHOME_DEFER_REASONS = [ 'control-closed', 'budget-closed', @@ -37,13 +35,10 @@ export const IdleRegionalRehomeResponseSchema = z .object({ v: z.literal(1), outcome: z.enum(['busy', 'committed', 'deferred', 'stale']), - // Optional both ways: a source cell on an older image omits it and the - // director keeps its walk-the-whole-list behaviour, and a reason a newer - // cell adds later reads as absent instead of failing the whole response. + // Older cells omit reasons; newer unknown reasons decode absent so mixed versions retain the full walk. reason: z.enum(IDLE_REGIONAL_REHOME_DEFER_REASONS).optional().catch(undefined) }) - // Unknown keys are dropped rather than rejected, so the next optional field - // on this response does not have to wait for every director to redeploy. + // Strip unknown fields so optional additions need no synchronized director rollout. .strip() export type IdleRegionalRehomeRequest = z.infer diff --git a/cloud/packages/relay-contract/src/protocol-limits.ts b/cloud/packages/relay-contract/src/protocol-limits.ts index 6257f9cecb7..390e7224d93 100644 --- a/cloud/packages/relay-contract/src/protocol-limits.ts +++ b/cloud/packages/relay-contract/src/protocol-limits.ts @@ -1,11 +1,8 @@ export const RELAY_PROTOCOL_LIMITS = { firstFrameDeadlineMs: 2_000, maxHttpBodyBytes: 4 * 1024, - // Why: the splice is an opaque E2EE stream; the desktop's worktree catalog - // response already exceeds 1MiB on large workspaces (~775KiB at 415 - // worktrees, growing), and an oversized frame kills the session on every - // reconnect. 8MiB buys years of headroom; catalog pagination is the - // long-term fix on the desktop side. + // Opaque E2EE catalogs exceed 1MiB on large workspaces; oversized frames kill every reconnect. + // 8MiB adds headroom until desktop catalog pagination bounds responses. maxFrameBytes: 8 * 1024 * 1024, maxConnectionsPerHost: 8, idleTimeoutMs: 10 * 60 * 1000, diff --git a/cloud/packages/relay-contract/src/relay-regions.ts b/cloud/packages/relay-contract/src/relay-regions.ts index 6b8837829df..94be52c6f08 100644 --- a/cloud/packages/relay-contract/src/relay-regions.ts +++ b/cloud/packages/relay-contract/src/relay-regions.ts @@ -8,10 +8,7 @@ export type RelayRegion = z.infer export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1' -// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so -// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a -// new region a compile error here, which is the point: a region with no segment would silently -// drop out of the region-skew alert's denominators. +// Explicit segments must match Terraform; satisfies forces new regions to define metrics or skew-alert denominators silently omit them. export const RELAY_REGION_METRIC_SEGMENTS = { 'us-central1': 'UsCentral1', 'asia-east2': 'AsiaEast2' diff --git a/config/build-plugins/bootstrap-fatal-exit-banner.ts b/config/build-plugins/bootstrap-fatal-exit-banner.ts index 9233d4327e1..8a8359da9db 100644 --- a/config/build-plugins/bootstrap-fatal-exit-banner.ts +++ b/config/build-plugins/bootstrap-fatal-exit-banner.ts @@ -4,10 +4,7 @@ export const BOOTSTRAP_FATAL_LOG_FILE_NAME = 'bootstrap-fatal.log' const BOOTSTRAP_FATAL_LOG_MAX_BYTES = 262_144 export function createBootstrapFatalExitBanner(): string { - // Why: Electron's pre-import error dialog can leave main resident and block NSIS replacement. - // Suppressing that dialog also removes the only account of the failure, so record one first: - // a partially copied resources tree otherwise exits silently on every launch, with nothing - // for the user or a support log to name the module that was missing. + // Electron’s pre-import dialog can leave main resident and block NSIS replacement; log missing-module failures before suppressing it or partial installs exit silently. return ` ;(() => { const guardKey = ${JSON.stringify(BOOTSTRAP_FATAL_EXIT_GUARD_KEY)} diff --git a/config/build-plugins/main-build-entry-points.ts b/config/build-plugins/main-build-entry-points.ts index fdf7639d9f2..f92b757e6f1 100644 --- a/config/build-plugins/main-build-entry-points.ts +++ b/config/build-plugins/main-build-entry-points.ts @@ -20,17 +20,13 @@ export const MAIN_BUILD_ENTRY_POINTS: Record = { 'wsl-transcript-fs-process-entry': resolve( 'src/main/native-chat/wsl-transcript-fs-process-entry.ts' ), - // Why: libuv spawns processes inline on the calling loop, so the port - // scan's probe commands run on a worker thread instead of the UI one. + // libuv spawns inline; run port probes on a worker to keep the UI loop free. 'port-scan-command-worker-entry': resolve('src/main/ports/port-scan-command-worker-entry.ts'), - // Why (#27362): the Codex usage probe spawns on window focus; a cold - // CreateProcessW must not block the UI loop. + // Focus-triggered usage probes must not block UI on cold CreateProcessW (#27362). 'worker-thread-child-process-entry': resolve( 'src/main/worker-thread-child-process/worker-thread-child-process-entry.ts' ), - // Why: the Claude/Codex/OpenCode usage scans walk whole history - // corpora and read SQLite synchronously; a worker thread keeps that - // off the main-process event loop. + // Usage scans synchronously walk histories/SQLite; keep them off the main loop. 'usage-scan-worker-entry': resolve('src/main/usage/usage-scan-worker-entry.ts'), 'workspace-space-scan-worker-entry': resolve('src/main/workspace-space-scan-worker-entry.ts'), // Why: a first account setup can merge a large history tree with sync fs calls. @@ -43,8 +39,7 @@ export const MAIN_BUILD_ENTRY_POINTS: Record = { 'profile-state-writer-worker-entry': resolve( 'src/main/persistence/profile-state/profile-state-writer-worker-entry.ts' ), - // Why: forked with ELECTRON_RUN_AS_NODE so @parcel/watcher faults - // can't take down the main process (issue #7547). + // Isolate native @parcel/watcher faults from main (#7547). 'parcel-watcher-process-entry': resolve( 'src/main/ipc/filesystem/watcher-process/parcel-watcher-process-entry.ts' ), @@ -52,8 +47,7 @@ export const MAIN_BUILD_ENTRY_POINTS: Record = { [ORCAD_LOCAL_SERVE_SELECTION_ENTRY]: resolve( 'src/main/orcad/orcad-local-serve-selection-entry.ts' ), - // Why: a worker thread survives the macOS 26 AppKit main-thread deadlock - // without paying for another Electron process. + // Worker survives macOS 26 AppKit main-thread deadlock without another Electron process. 'main-thread-hang-watchdog-entry': resolve( 'src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts' ), diff --git a/config/build-plugins/plain-node-entry-guard.ts b/config/build-plugins/plain-node-entry-guard.ts index 6fe5d0a2e2f..00b01098a28 100644 --- a/config/build-plugins/plain-node-entry-guard.ts +++ b/config/build-plugins/plain-node-entry-guard.ts @@ -8,14 +8,8 @@ type NormalizedOutputOptions = Rollup.NormalizedOutputOptions type OutputBundle = Rollup.OutputBundle type OutputChunk = Rollup.OutputChunk -// Why: v1.4.129-rc.1 shipped a dead terminal daemon because a shared main -// chunk gained `require("electron")` (an import edge added in #7642), and the -// daemon is forked as a plain-Node process where electron cannot be required. -// Nothing in CI executes the built daemon-entry under plain Node, so the leak -// stayed invisible until an adopted old daemon died. This guard fails the -// build when any chunk reachable from a plain-Node fork entry requires -// electron, and smoke-loads daemon-entry under plain Node to prove its module -// graph still resolves. +// A shared-chunk Electron import killed plain-Node daemons in v1.4.129-rc.1 (#7642). +// Scan every reachable fork/worker chunk for Electron requires; smoke-load daemon-entry to verify module resolution. // The CLI loads these paths after electron-vite replaces out/main. export const CLI_MAIN_ENTRY_NAMES = [ @@ -52,13 +46,7 @@ const PLAIN_NODE_ENTRY_NAMES = [ ...CLI_MAIN_ENTRY_NAMES ] as const -// Entries executed as worker threads of the main process. Electron's module is -// not registered on worker threads, so require("electron") throws -// "Cannot find module 'electron'" there too (verified on Electron 43) and kills -// the worker at startup. These carry hand-written "must stay electron-free" -// comments, which is convention, not enforcement — and the port-scan worker in -// particular sits one import away from a client module that deliberately does -// require electron. +// Electron is unavailable in main-process worker threads too (verified Electron 43); scan their graphs rather than trusting entry comments. const WORKER_THREAD_ENTRY_NAMES = [ 'stt-worker', 'warp-theme-parser-worker', @@ -83,9 +71,7 @@ type EntryRuntime = 'plain-Node process' | 'worker thread' // Subpaths (electron/main) are as unloadable as the bare module under plain Node. const ELECTRON_REQUIRE_RE = /require\(\s*["'`]electron(?:\/[^"'`]+)?["'`]\s*\)/ -// Why: writeBundle skips any name missing from the bundle, so a renamed or -// removed rollup input would silently drop that entry from the guard and let the -// regression back in. Pin the lists to the input keys at build start instead. +// Pin guarded names to Rollup inputs or renames silently remove entries from the guard. function assertEntryNamesAreRollupInputs(input: NormalizedInputOptions['input']): void { if (typeof input === 'string' || Array.isArray(input)) { return @@ -151,10 +137,7 @@ function assertNoElectronRequire( } } -// Why: on packaged Windows the daemon runs from a relocated daemon-host copy that mirrors only -// these packages (src/main/daemon/daemon-host-manifest.ts). Any other bare require in its graph -// resolves in the install dir but throws MODULE_NOT_FOUND in the copy, and the desktop silently -// falls back to an unrelocated daemon that the next update kills. +// Relocated Windows daemon copies only these packages; other bare requires cause silent fallback to an unrelocated daemon that updates kill. const DAEMON_HOST_MIRRORED_PACKAGES = [ '@orca/process-host', 'node-pty', @@ -212,9 +195,7 @@ type SmokeResult = { timedOut: boolean } -// Why not spawnSync({ timeout }): its timeout only sends killSignal and then -// keeps blocking until the child exits, so a child that traps SIGTERM hangs the -// build forever. Escalate to SIGKILL instead. +// spawnSync timeout waits after SIGTERM; trapped signals can hang builds, so escalate asynchronously to SIGKILL. function runDaemonEntry(entryPath: string, timings: SmokeTimings): Promise { return new Promise((resolve) => { const child = spawn(process.execPath, [entryPath], { stdio: ['ignore', 'ignore', 'pipe'] }) @@ -256,10 +237,7 @@ function runDaemonEntry(entryPath: string, timings: SmokeTimings): Promise { const entryPath = join(outputDir, 'daemon-entry.js') const result = await runDaemonEntry(entryPath, timings) @@ -308,8 +286,7 @@ export function createPlainNodeEntryGuardPlugin( assertEntryNamesAreRollupInputs(options.input) }, writeBundle(options: NormalizedOutputOptions, bundle: OutputBundle) { - // Why: skip in `electron-vite dev` watch mode — the smoke would respawn on - // every rebuild, and the guard only needs to gate produced builds. + // Skip dev-watch smoke or every rebuild respawns daemon-entry; gate produced builds only. if (this.meta.watchMode) { return } diff --git a/config/build-plugins/process-host-dev-rebuild.ts b/config/build-plugins/process-host-dev-rebuild.ts index 5daf6a686bd..26237140cd3 100644 --- a/config/build-plugins/process-host-dev-rebuild.ts +++ b/config/build-plugins/process-host-dev-rebuild.ts @@ -142,8 +142,7 @@ export function createProcessHostDevRebuildPlugin(root = process.cwd()): Plugin if (!packageDirty) { return } - // The external package must finish emitting before electron-vite restarts the app. The - // package's own build keeps its compiler and per-file atomic dist sync identical to build:packages. + // Finish external-package atomic dist emission before electron-vite restarts the app. const result = runProcessSync({ program: process.execPath, args: [ diff --git a/config/build-plugins/renderer-bundled-dev.ts b/config/build-plugins/renderer-bundled-dev.ts index 0c230dd89f1..d427d083946 100644 --- a/config/build-plugins/renderer-bundled-dev.ts +++ b/config/build-plugins/renderer-bundled-dev.ts @@ -1,9 +1,6 @@ import type { UserConfig } from 'electron-vite' -/** - * Default in dev (opt out with `ORCA_DEV_BUNDLED=0`): Vite's bundled dev mode serves the renderer as - * bundled chunks instead of thousands of per-module requests, which shortens dev startup and reloads. - */ +/** Default to bundled dev chunks to reduce renderer requests and startup/reload time; opt out with ORCA_DEV_BUNDLED=0. */ export function withBundledRendererDev( config: UserConfig, env: NodeJS.ProcessEnv = process.env diff --git a/config/bundled-ripgrep-resources.cjs b/config/bundled-ripgrep-resources.cjs index b1689fdf9dd..2acad200d25 100644 --- a/config/bundled-ripgrep-resources.cjs +++ b/config/bundled-ripgrep-resources.cjs @@ -1,8 +1,7 @@ const { chmodSync, existsSync } = require('node:fs') const { join } = require('node:path') -// Why every relay platform in every artifact: SSH deploys upload the remote host's rg from the -// local bundle, and WSL runs the Linux build from the Windows install directory. +// Ship every relay platform: SSH uploads remote-host rg from local bundle; WSL uses Linux rg from Windows installs. const BUNDLED_RIPGREP_PLATFORMS = [ 'linux-x64', 'linux-arm64', @@ -24,8 +23,7 @@ const bundledRipgrepExtraResources = [ to: RIPGREP_RESOURCE_DIR, filter: BUNDLED_RIPGREP_PLATFORMS.map((platform) => `${platform}/**`) }, - // Why: the binaries statically link PCRE2, and on Linux musl, jemalloc and LLVM libunwind -- - // all of which require their notice on binary redistribution. Whole dir, so notices can be added. + // Static PCRE2/musl/jemalloc/LLVM libunwind require redistribution notices; copy the whole directory to include future notices. { from: 'resources/licenses/ripgrep', to: `${RIPGREP_RESOURCE_DIR}/licenses` } ] diff --git a/config/ci/windows-ssh-provider/invoke-windows-host-cells.ps1 b/config/ci/windows-ssh-provider/invoke-windows-host-cells.ps1 index 039b934e784..94e17ff2e15 100644 --- a/config/ci/windows-ssh-provider/invoke-windows-host-cells.ps1 +++ b/config/ci/windows-ssh-provider/invoke-windows-host-cells.ps1 @@ -1,6 +1,4 @@ -# Ephemeral CI only. Runs each Windows host cell (src/main/ssh/ssh-windows-host-cells.ts) against the -# private sshd prove-preview-openssh.ps1 provisioned: one account, one DefaultShell and one vitest -# process per cell, so every cell starts from an empty runtime store. +# Ephemeral CI only: one private-sshd account, DefaultShell and vitest process per host cell for an empty runtime store. param( [Parameter(Mandatory=$true)][string]$SourceRoot, [Parameter(Mandatory=$true)][hashtable]$Context, @@ -48,18 +46,14 @@ function Set-PrivateDefaultShell([string]$Shell) { } } -# Diagnostic only: Orca must launch orcad without WMI, which refuses a standard user's SSH -# (network) logon unless an administrator grants Remote Enable on root\cimv2. The cells run with no -# such grant, so a launch that still needs WMI fails its cell. Prints ORCA_WMI= -# or ORCA_WMI=denied. +# Diagnostic ORCA_WMI=|denied: standard SSH logons lack root\cimv2 Remote Enable; Orca must launch without WMI. function Test-PrivateWmiLaunch([string]$Account) { $out=Invoke-PrivateSsh $Account 'powershell.exe -NoProfile -NonInteractive -Command "try{$r=Invoke-CimMethod -ClassName Win32_Process -MethodName Create -Arguments @{CommandLine=''cmd.exe /d /c exit 0''} -ErrorAction Stop;''ORCA_WMI=''+$r.ReturnValue}catch{''ORCA_WMI=denied''}"' $match=[regex]::Match($out,'ORCA_WMI=(\S+)') if($match.Success){return $match.Groups[1].Value}else{return 'no-output'} } -# App-level cells (tests/e2e/ssh-orcad-auto-convert.spec.ts, ssh-orcad-windows-cli-matrix.spec.ts). Last -# in the run: they switch native modules to Electron's ABI, which the vitest cells cannot load. +# Run app-level cells last: the e2e build switches natives to Electron’s ABI, which vitest cannot load. function Invoke-AppCell($Account,[string]$Descriptor,[string]$Log,[string]$Spec,[string]$Grep) { $ready=Invoke-PrivateSsh $Account.name 'git init -q orca-convert-repo && git -C orca-convert-repo -c user.name=orca -c user.email=orca@example.invalid commit -q --allow-empty -m init && echo ORCA_REPO_READY' if($ready -notmatch 'ORCA_REPO_READY'){throw 'Could not create the convert cell repository as the account'} diff --git a/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 b/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 index 56961bbd78c..ba6fa4c14d3 100644 --- a/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 +++ b/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 @@ -1,7 +1,5 @@ -# Ephemeral CI only. Preview ZIP or inbox capability binaries, always under a private service and accounts. -# -HiddenTools: the private accounts are denied every machine PATH directory holding one of these -# executables, and their own PATH carries logging shims for them, so SSH sessions have no host toolchain. -# -HostCellProbe receives a context hashtable (accounts, port, keys, shim log) once provisioning passes. +# Ephemeral CI only: preview/inbox sshd under private service/accounts. -HiddenTools denies machine tool PATHs and adds logging shims. +# -HostCellProbe receives accounts, port, keys and shim log after provisioning. param([Parameter(Mandatory=$true)][string]$Receipt,[string]$Archive,[Parameter(Mandatory=$true)][ValidateSet('arm64','x64')][string]$Arch,[ValidateSet('preview','inbox')][string]$Server='preview',[scriptblock]$ProductionRouteProbe,[ValidateRange(1,6)][int]$Accounts=1,[ValidateRange(0,6)][int]$ForwardingAccounts=0,[string[]]$HiddenTools=@(),[scriptblock]$HostCellProbe,[string]$InboxPreparationReceipt) $ErrorActionPreference = 'Stop' . (Join-Path $PSScriptRoot 'windows-ssh-capability.ps1') @@ -123,8 +121,7 @@ function Record-PrivateServiceDiagnostics([switch]$AfterStop) { Write-Stage 'private-service-diagnostics-complete' } -# Drops every PATH entry holding one of the tools; entries are expanded, as a session sees them. -# Why .NET and not Join-Path: Join-Path throws on an entry naming a drive this runner lacks. +# Expand PATH entries as the session does; .NET tolerates missing drives where Join-Path throws. function Split-HostToolchainPath([string]$PathValue,[string[]]$Tools) { $kept=[Collections.Generic.List[string]]::new();$hidden=[Collections.Generic.List[string]]::new() foreach($entry in @($PathValue -split ';' | Where-Object {$_})){ @@ -274,8 +271,7 @@ $(@($accountNames | Select-Object -Last $ForwardingAccounts | ForEach-Object {"M $machinePath=(Get-Item -LiteralPath 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment').GetValue('Path','',[Microsoft.Win32.RegistryValueOptions]::DoNotExpandEnvironmentNames) $split=Split-HostToolchainPath $machinePath ($HiddenTools+@('node')) $kept=$split.kept;$hidden=$split.hidden - # Why ACLs, not PATH: Win32-OpenSSH builds the session PATH from the machine and user registry - # values, ignoring the service environment and SetEnv. Deny ACEs name only the private accounts. + # OpenSSH rebuilds PATH from registry, ignoring service env/SetEnv; deny ACEs affect only private accounts. foreach($directory in $hidden){ Invoke-Bounded icacls.exe (@($directory.TrimEnd('\'),'/deny')+@($ownedAccounts | ForEach-Object {"*$($_.sid):(OI)(CI)RX"})) 120 | Out-Null $deniedToolDirs.Add($directory) diff --git a/config/dark-build-switches.cjs b/config/dark-build-switches.cjs index b46f40cb686..019b2e18137 100644 --- a/config/dark-build-switches.cjs +++ b/config/dark-build-switches.cjs @@ -1,10 +1,8 @@ const { existsSync } = require('node:fs') const { join } = require('node:path') -// Dark build switches: a build compiles a switch's path in only when its variable is 'main'. Such a -// main build leaves the switch's marker in out/main, the phone page bundle (out/mobile-web), out/orcad -// and a --dark-test-template orcad template; packaging refuses any of them, and packaging and a -// release orcad template also refuse the variable itself. +// Only main-valued switches compile dark paths; marked main/mobile-web/orcad/template outputs cannot ship. +// Packaging and release templates also reject enabled switch environment variables. const DARK_BUILD_SWITCHES = [ // src/shared/layout-owner-switch.ts { variable: 'ORCA_LAYOUT_OWNER', off: 'window' }, @@ -38,12 +36,10 @@ function turnedOnDarkBuildDefines(env = process.env) { ) } -/** Every marker a switched-on build can leave, so a bundle check accounts for them. */ function darkBuildMarkerFileNames() { return DARK_BUILD_SWITCHES.map(markerFileName) } -/** The marker each switched-on build leaves in its output. */ function darkBuildMarkers(env = process.env) { return turnedOn(env).map((darkSwitch) => ({ fileName: markerFileName(darkSwitch), diff --git a/config/docker/daemon-scope-lifetime/terminal-scope-cases.cjs b/config/docker/daemon-scope-lifetime/terminal-scope-cases.cjs index 0cf2d7288b8..8b10ce3978b 100644 --- a/config/docker/daemon-scope-lifetime/terminal-scope-cases.cjs +++ b/config/docker/daemon-scope-lifetime/terminal-scope-cases.cjs @@ -1,6 +1,4 @@ -// Per-terminal scopes against the real user manager: an OOM kill or a fork limit in one -// terminal leaves the daemon and the other terminals running, and the daemon scope's end -// takes the terminal slice with it. The baseline only reports what one OOM kill does today. +// Exercise terminal isolation against real systemd: OOM/fork limits spare daemon/neighbors, while daemon exit ends the terminal slice. Baseline reports shared-scope OOM behavior. const assert = require('node:assert/strict') const fs = require('node:fs') const { join } = require('node:path') @@ -101,7 +99,7 @@ async function oomCase(bundlePath, prefix, systemdVersion, mode) { const [term1, term2, term3] = daemon.terminals await until(() => pidFrom(`${r}.sleeper`) && pidFrom(`${r}.stubborn`), 'terminals not ready') const sleeper = pidFrom(`${r}.sleeper`) - // Today every terminal shares the daemon's scope, so its limit is the only one to hit. + // Baseline terminals share the daemon scope, so only its limit can be hit. const [limited, memoryMax] = mode === 'baseline' ? [daemon.unit, '192M'] : [daemon.scopes[1].split('/').at(-1), '64M'] systemctl('set-property', '--runtime', limited, `MemoryMax=${memoryMax}`, 'MemorySwapMax=0') diff --git a/config/docker/daemon-shutdown-descendants/run-case.sh b/config/docker/daemon-shutdown-descendants/run-case.sh index a8ce8414d74..d55b5ea848e 100755 --- a/config/docker/daemon-shutdown-descendants/run-case.sh +++ b/config/docker/daemon-shutdown-descendants/run-case.sh @@ -6,8 +6,7 @@ mode=${2:?mode is required} result_dir=/tmp/daemon-shutdown-descendants mkdir -p "$result_dir" -# The fixture exits like daemon-entry after dispose(). This shell remains as a -# supervisor so a detached descendant can be inspected after the daemon exits. +# Keep this supervisor alive after the daemon-like exit to inspect detached descendants. timeout --kill-after=2s 20s node /opt/daemon-shutdown-descendants/fixture.cjs "$bundle" "$result_dir/fixture.json" child_pid=$(node -e "const r=require(process.argv[1]); process.stdout.write(String(r.childPid))" "$result_dir/fixture.json") diff --git a/config/docker/headless-serve-shutdown/run-signal-case.sh b/config/docker/headless-serve-shutdown/run-signal-case.sh index 5d1979462aa..47f281a1aec 100755 --- a/config/docker/headless-serve-shutdown/run-signal-case.sh +++ b/config/docker/headless-serve-shutdown/run-signal-case.sh @@ -6,8 +6,7 @@ app_root=${ORCA_TEST_APP_ROOT:-/artifacts/root} signal_target_kind=${ORCA_SIGNAL_TARGET:-app} entrypoint_kind=${ORCA_TEST_ENTRYPOINT:-app} int_delivery=${ORCA_INT_DELIVERY:-foreground-process-group} -# Packaged Electron startup can approach 90s on a cold CI runner; leave room -# for the readiness line to reach the log before the observer deadline. +# Allow cold CI’s ~90s Electron startup plus log delivery before the observer deadline. startup_timeout_seconds=${ORCA_STARTUP_TIMEOUT_SECONDS:-180} if ((EUID == 0)); then @@ -57,9 +56,7 @@ setsid env -u DISPLAY "${entrypoint[@]}" serve --port 0 --pairing-address 127.0. app_pid=$! app_start_ticks=$(awk '{print $22}' "/proc/$app_pid/stat") -# jq's `inputs` waits for EOF even when wrapped in `first`, so a tail -F -# observer can outlive the timeout and leak into the next signal case. Poll -# finite snapshots instead; each parser invocation has a definite EOF. +# Poll finite snapshots: jq inputs waits for EOF under first(), leaking tail -F observers past the timeout. read_ready_line() { sed -u -n 's/^[^{]*//p' "$stdout_log" \ | jq --unbuffered -Rnc 'first(inputs | fromjson? | select(.type == "orca_server_ready" and .schemaVersion == 1))' @@ -161,8 +158,7 @@ fi signal_target_pid=$app_pid if [[ "$signal_target_kind" == serving-electron ]]; then - # The ready socket identifies the serving Electron even when AppImage's - # extraction wrapper rewrites the command line before it reaches Chromium. + # AppImage’s extraction wrapper rewrites argv; the ready socket identifies the serving Electron. signal_target_pid=$(head -n1 <<<"$listener_before_pids") [[ -n "$signal_target_pid" ]] || { echo "FAIL: serving Electron process not found" >&2; exit 1; } if [[ -z "${tree_start_ticks[$signal_target_pid]+present}" ]]; then diff --git a/config/electron-builder.config.cjs b/config/electron-builder.config.cjs index bb1f6d9618b..2785ceda272 100644 --- a/config/electron-builder.config.cjs +++ b/config/electron-builder.config.cjs @@ -71,17 +71,11 @@ const { const { signWindowsUninstallerViaSignPath } = require('./scripts/windows-uninstaller-signing.cjs') const { buildMacShipItScheduling } = require('./scripts/macos-shipit-scheduling.cjs') -// Why: dev-channel builds must carry the *release* identity — same bundle id, -// Developer ID signature, and notarization ticket — or Squirrel.Mac refuses to -// swap them over an installed Orca and macOS treats each build as a new app. +// Dev channels need release bundle ID/signature/notarization so Squirrel.Mac accepts updates and macOS retains app identity. const isMacHourly = process.env.ORCA_MAC_HOURLY === '1' const isMacDaily = process.env.ORCA_MAC_DAILY === '1' const isMacAdhoc = process.env.ORCA_MAC_ADHOC === '1' -// Why a second set of variables rather than making the mac ones platform-neutral: -// the mac ones gate `isMacRelease` below, which turns on hardened runtime, -// notarization, and root-level `forceCodeSigning`. A Windows dev build that -// reused them would fail packaging outright for want of a cert it is -// deliberately not using. +// Keep Windows dev flags separate: mac flags enable hardened runtime/notarization/global signing, which would demand an unavailable Windows cert. const isWinHourly = process.env.ORCA_WIN_HOURLY === '1' const isWinDaily = process.env.ORCA_WIN_DAILY === '1' const isWinAdhoc = process.env.ORCA_WIN_ADHOC === '1' @@ -100,12 +94,7 @@ const devChannelBuildVersion = isHourlyChannel : isAdhocChannel ? process.env.ORCA_ADHOC_BUILD_VERSION : undefined -// Why each dev channel gets its own repo rather than tagging into the main one: -// the releases atom feed exposes only the 10 newest entries, so 24 hourly tags a -// day would evict every stable/RC entry and strand users on a feed with nothing -// to install. Keeping adhoc/daily separate from hourly too means a branch build -// or a once-a-day cut cannot be picked up by someone who only meant to ride -// main's hourlies. +// Separate dev-channel repos: 24 hourly tags/day would evict stable/RC from the ten-entry release feed; isolate adhoc/daily from hourly subscribers too. const devChannelRepo = isHourlyChannel ? 'orca-hourly' : isDailyChannel @@ -118,38 +107,28 @@ const featureWallResources = { from: 'resources/onboarding/feature-wall', to: 'onboarding/feature-wall' } -// Why: freshness detection needs immutable identity metadata from this exact -// app build, but never needs the skill package bytes or a runtime network read. +// Freshness checks need immutable build identity, not skill bytes or runtime network access. const skillFreshnessResources = { from: 'resources/skills', to: 'skills' } -// Why a real directory: native-chat agents load this skill plugin by path (Claude --plugin-dir, -// Codex skill roots), and neither can read inside app.asar. +// Native agents load plugins by path and cannot read inside app.asar. const nativeChatVisualsResource = { from: 'resources/native-chat-visuals', to: 'native-chat-visuals' } -// Why: SSH relay deploy resolves bundles from process.resourcesPath in packaged -// apps. Keeping relay assets as extraResources makes them real directories -// instead of paths hidden inside app.asar. +// SSH deploy resolves process.resourcesPath/relay; extraResources keeps bundles on real disk. const relayExtraResource = { from: 'out/relay', to: 'relay' } -// Why: bundled plugins are immutable install inputs and must remain ordinary -// directories so the startup bootstrap can verify and publish exact bytes. +// Bootstrap verifies/publishes immutable plugin bytes from ordinary directories. const bundledPluginResources = { from: 'resources/plugins/launch', to: 'plugins/launch' } -// Why: the main bundle, packaged CLI, SSH paths, and speech worker all execute -// from package directories where pnpm's symlink farm is absent. Copy the exact -// runtime dependency closure to Resources/node_modules so bare require() calls -// do not fall through to a developer checkout's node_modules. -// Why the single file rather than the package root: app.asar carries no node_modules, so main's -// lazy require in deferred-emoji-shortcode-dataset.ts resolves only out of Resources/node_modules, -// but emojibase-data is 49 MB of locale datasets and worktree naming reads exactly this 166 KB file. +// Packaged main/CLI/SSH/speech paths lack pnpm links; copy runtime closure so bare requires cannot fall through to checkout dependencies. +// Ship only the 166KB emoji dataset used by worktree naming, not all 49MB of emojibase-data. const emojiShortcodeDatasetResource = { from: 'node_modules/emojibase-data/en/shortcodes/emojibase.json', to: 'node_modules/emojibase-data/en/shortcodes/emojibase.json' @@ -164,8 +143,7 @@ const commonExtraResources = [ nativeChatVisualsResource, emojiShortcodeDatasetResource ] -// Why: native speech addons must be real files outside app.asar; copy only the -// package matching the artifact target instead of every optional variant. +// Native speech addons need real disk paths; copy only the artifact target’s variant. const macSpeechNativeResource = { from: 'node_modules/sherpa-onnx-darwin-${arch}', to: 'node_modules/sherpa-onnx-darwin-${arch}' @@ -202,16 +180,11 @@ const rpmElectronRuntimeDependencies = [ '(libuuid or libuuid1)' ] -// Why mirrored, not imported: this config is CJS loaded by electron-builder outside the TS build. -// Keep in sync with isOsOpenedDocumentName() in src/main/startup/os-opened-documents.ts and with -// config/nsis/orca-installer-hooks.nsh, which registers the same set on Windows. +// Mirror OS document extensions: CJS packager cannot import TS; keep startup/os-opened-documents.ts and NSIS hooks aligned. const MARKDOWN_FILE_EXTENSIONS = ['md', 'markdown', 'mdx'] const TABULAR_FILE_EXTENSIONS = ['csv', 'tsv'] -// Why: the config must load on a host-only install without resolving unused Windows addons. -// This is load-time tolerance only; beforePack enforces that the target's natives are installed. -// Why one package: @vscode/windows-process-tree is the only os: win32 npm addon; -// the @orca/windows-* addons are workspace links present on every host, so their presence proves nothing. +// Config loads without unused Windows addons; beforePack enforces target natives. Only @vscode/windows-process-tree has os:win32; workspace links prove no binary exists. const windowsRuntimeResources = existsSync( join(__dirname, '..', 'node_modules', '@vscode', 'windows-process-tree', 'package.json') ) @@ -256,8 +229,7 @@ module.exports = { }, files: [ '!**/.vscode/*', - // Why: these repo-only inputs are either bundled into out/ or copied via - // extraResources. Shipping them in app.asar bloats the desktop bundle. + // Repo inputs already bundled/copied through extraResources would bloat app.asar. '!src{,/**/*}', '!out/orcad{,/**/*}', // Never in app.asar: the template ships via orcadTemplateExtraResource; prebuilds are build inputs. @@ -277,24 +249,17 @@ module.exports = { '!mobile{,/**/*}', '!native{,/**/*}', '!skills{,/**/*}', - // Why: guide/stub authoring sources are compiled into runtime artifacts; shipping - // either source tree would duplicate content without a runtime consumer. + // Authoring guides/stubs compile into runtime artifacts; source trees only duplicate content. '!skill-guides{,/**/*}', '!skill-stubs{,/**/*}', '!tests{,/**/*}', - // Why: examples/ is plugin authoring documentation with no runtime consumer — - // bundled plugins ship via extraResources from resources/plugins/launch/. It also - // carries hostile-panel, the adversarial fixture the containment tests point at, - // which must never reach a user's install. + // Examples have no runtime consumer and include adversarial hostile-panel; ship only resources/plugins/launch. '!examples{,/**/*}', - // Why: pr-evidence/ is a local e2e screenshot output (ORCA_CAPTURE_EVIDENCE); - // it is gitignored, but exclude it defensively so a stray local capture at - // package time never bloats app.asar. + // Exclude gitignored local screenshot evidence defensively so packaging cannot pick up stray captures. '!pr-evidence{,/**/*}', // Local build logs and rollback copies are never application resources. '!notes{,/**/*}', - // Why: local agent/tooling directories may contain worktree symlink loops; - // they are never runtime inputs and must not be traversed by electron-builder. + // Local tooling directories are not runtime inputs and may contain symlink loops. '!{.claude,.grok,.agents,.codex}{,/**/*}', '!Casks{,/**/*}', '!{AGENTS.md,CLAUDE.md,DEVELOPING.md,bundle-size-progress.md,ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md,ORCHESTRATION_STRUCTURED_OUTPUT_DESIGN.md}', @@ -307,63 +272,29 @@ module.exports = { '!out/renderer/.vite{,/**/*}', // The installed browser serves its allowlisted assets from the unpacked renderer. '!out/web{,/**/*}', - // Why: out/electron-dev caches `pnpm dev`'s per-branch Electron.app copies (~270MB each). - // CI never creates it, but packaging on a machine that has run dev would pack them all. + // Local dev Electron.app caches (~270MB/branch) must not enter packaged builds. '!out/electron-dev{,/**/*}', - // Why: relayExtraResource already ships out/relay to resources/relay, which is - // the only path a packaged build resolves. Packing it again added 14MB and put - // relay.js inside app.asar, so a script-heuristic verdict on relay.js took the - // whole asar with it as a compound object and gutted the install (#20966, #20972). + // extraResources already ships relay; duplicating it in asar added 14MB and let script-heuristic quarantine gut the whole archive (#20966/#20972). '!out/relay{,/**/*}', '!electron.vite.config.{js,ts,mjs,cjs}', '!{.oxlintrc.json,.oxfmtrc.json,components.json,opencode.json,orca.yaml,pnpm-workspace.yaml,vite.web.config.ts}', '!{.eslintcache,eslint.config.mjs,.prettierignore,.prettierrc.yaml,CHANGELOG.md,README.md}', '!{.env,.env.*,.npmrc,pnpm-lock.yaml}', '!tsconfig.json', - // Why: feature-wall media is copied via extraResources so runtime can read - // it from process.resourcesPath; exclude the source copy from app.asar. + // Feature media is read from process.resourcesPath via extraResources; exclude duplicate sources. '!resources/onboarding/feature-wall/**', '!resources/skills/**', '!resources/native-chat-visuals/**', - // Why: bundled plugins ship via extraResources to resources/plugins/launch; - // packing the source tree into app.asar would duplicate those exact bytes. + // extraResources already ships plugin bytes; exclude source duplicates. '!resources/plugins/launch/**', - // Why: speech packages are copied selectively through the platform - // extraResources entry below; keeping them in app.asar would ship every - // native variant (and duplicate the selected one). + // Platform extraResources selects speech natives; asar would duplicate it and ship other variants. '!node_modules/sherpa-onnx*{,/**/*}', - // Why: the Windows CLI shim ships via extraResources to resources/bin/orca.cmd - // (beside the native resources/bin/orca.exe). Packing the source tree into - // app.asar too lets asarUnpack:['resources/**'] extract a second copy at - // app.asar.unpacked/resources/win32/bin/orca.cmd with no adjacent orca.exe, - // which fails to launch the CLI (#7351). + // CLI shim belongs beside resources/bin/orca.exe; unpacking source creates a second .cmd without its executable (#7351). '!resources/win32{,/**/*}' ], - // Why: the CLI entry-point lives in out/cli/ but imports shared modules - // from out/shared/ and local hook mutators from out/main/. These paths must be - // unpacked so that Node's require() can resolve the cross-directory imports - // when the CLI runs outside the asar archive. - // Why: daemon-entry.js is forked as a separate Node.js process and must be - // accessible on disk (not inside the asar archive) for child_process.fork(). - // Why: the CLI is compiled by tsc (not bundled), so its runtime imports - // resolve at runtime via Node's normal module lookup. The shim launches - // the CLI with ELECTRON_RUN_AS_NODE, which bypasses Electron's asar - // integration — dependencies inside the asar archive are invisible to - // require(). Unpack CLI runtime deps so they resolve from - // app.asar.unpacked/node_modules/. - // Why: remote runtime connections use WebSocket + E2EE from the packaged CLI - // before the GUI process starts, so those deps need the same treatment. - // Why: out/package.json pins compiled output to CommonJS so parent - // package.json files with type=module cannot change the packaged CLI loader. - // Why: the foreign SQLite reader entry is also spawned by the scanner - // service, which runs under ELECTRON_RUN_AS_NODE and so cannot see into - // app.asar. Left packed, that spawn fails closed and every OpenCode session - // disappears from Agent Session History in packaged builds only. Worker - // entries reached solely from the Electron main process stay packed, since - // asar redirects their app.asar paths. - // Why: `orca serve` runs the orcad selection entry under ELECTRON_RUN_AS_NODE too (#27307). - // Why out/renderer and out/mobile-web: their assets are opened by path, and opening a packed file - // reads a /tmp extract Electron makes once and never remakes, so a /tmp sweep broke them (#27976). + // Node-mode CLI/daemon/foreign SQLite reader/serve entries cannot read asar; unpack their shared/main dependencies too. + // Keep out/package.json’s CommonJS boundary and remote CLI WebSocket/E2EE deps. Electron-only workers may stay packed. + // Renderer/mobile-web assets need real paths: swept temporary asar extracts are not recreated (#27976). asarUnpack: [ 'out/package.json', 'out/cli/**', @@ -468,12 +399,7 @@ module.exports = { stampPackagedCliVersion(resourcesDir, context.packager.appInfo.version) prunePackagedRuntimeNodeModules(resourcesDir, context.electronPlatformName, context.arch) assertNoWindowsExecutablesInRuntimeNodeModules(resourcesDir, context.electronPlatformName) - // Why: a Linux runner-image glibc bump silently shipped a node-pty pty.node - // requiring GLIBC_2.34, crashing the app on startup on Ubuntu 20.04 (#9902). - // Fail packaging if any bundled native binary exceeds the supported floor. - // Why after the prune: `pnpm install:release` widens the CPU set for cross-builds, - // so an arm64 slice can still carry the x64 @parcel/watcher until - // prunePackagedRuntimeNodeModules drops it. + // Gate native glibc floor after pruning foreign CPU variants; GLIBC_2.34 once broke Ubuntu 20.04 startup (#9902). let appImageNodeRuntime = null if (context.electronPlatformName === 'linux') { const targetArch = { 1: 'x64', 3: 'arm64' }[context.arch] @@ -485,11 +411,7 @@ module.exports = { verifyLinuxGlibcFloor(context.appOutDir, { targetArch }) } verifyPackagedMainRuntimeDeps(resourcesDir) - // Why: boot the packaged daemon-entry under plain Node, but only for the - // slice matching the packaging host's arch — daemon-entry.js is JS, yet it - // require()s the native (N-API) node-pty for the TARGET arch, which the host - // Node cannot load cross-arch. `Arch` enum: ia32=0, x64=1, armv7l=2, - // arm64=3, universal=4 (universal contains the host slice, so run it). + // Smoke only host-runnable slices: daemon JS loads target-arch node-pty. Arch enum ia32=0, x64=1, armv7l=2, arm64=3, universal=4. const archEnumByNodeArch = { ia32: 0, x64: 1, armv7l: 2, arm64: 3 } const hostArchEnum = archEnumByNodeArch[process.arch] const canExecuteTargetArch = context.arch === hostArchEnum || context.arch === 4 @@ -511,16 +433,13 @@ module.exports = { await verifyAppImageNodeRuntimeServesPty(resourcesDir, appImageNodeRuntime) } } else { - // Why: a cross-arch slice can't be booted by the host Node, but the - // unpacked entry must still exist — its absence is a layout regression - // regardless of arch, so only the boot is skipped, not the check. + // Cross-arch smoke cannot run, but missing unpacked entry is always a packaging error. assertPackagedDaemonEntryExists(resourcesDir) console.log( `[verify-packaged-daemon-entry] skipped boot on cross-arch slice (target ${context.arch}, host ${process.arch})` ) } - // Why: inspect electron-builder's real output so a broken extraResources - // mapping fails packaging before bundled content reaches users. + // Validate real electron-builder output so broken extraResources mappings fail before shipping. verifyPackagedPluginResources(resourcesDir) assertPackagedWebClient(resourcesDir) assertPackagedServeSelectionEntry(resourcesDir) @@ -535,9 +454,7 @@ module.exports = { if (!filename.startsWith('agent-browser-')) { continue } - // Why: the upstream package has inconsistent executable bits across - // platform binaries (notably darwin-x64). child_process.execFile needs - // the copied binary to be executable in packaged apps. + // Upstream executable bits vary (notably darwin-x64); restore them so packaged execFile works. chmodSync(join(resourcesDir, filename), 0o755) } if (context.electronPlatformName === 'darwin') { @@ -557,23 +474,9 @@ module.exports = { }, win: { executableName: 'Orca', - // Why: Windows installers are signed after electron-builder packaging by - // SignPath, so the packager cannot infer the updater publisherName. - // - // Why dev channels drop it instead: they ship unsigned, because SignPath's - // approval waits are budgeted in hours and cannot fit an hourly cadence. - // electron-updater Authenticode-verifies every installer it downloads - // against the publisherName baked into the *installed* app's app-update.yml - // (NsisUpdater.verifySignature), and skips verification entirely when that - // name is absent. An unsigned build that still claimed 'SignPath Foundation' - // would therefore reject its own channel's next build — and its way back to - // stable with it. Dropping it is what makes dev→dev and dev→stable work. - // Why a sign hook on a build that does not sign: it is the only moment - // electron-builder exposes the NSIS uninstaller (built in its own makensis - // pass, embedded, then deleted). The hook signs nothing — it relays the file - // to and from the CI SignPath request, and is inert when the relay env vars - // are unset, so local and dev builds are unaffected. publisherName stays on - // its existing channel split above. + // SignPath signs Windows installers after packaging, so set release publisherName explicitly. + // Unsigned dev channels omit it or updater signature checks reject dev updates and return to stable. + // Sign hook signs nothing: it relays transient NSIS uninstaller files to CI SignPath; unset relay env leaves local/dev builds inert. signtoolOptions: { sign: signWindowsUninstallerViaSignPath, ...(isWinDevChannel ? {} : { publisherName: 'SignPath Foundation' }) @@ -617,17 +520,12 @@ module.exports = { shortcutName: '${productName}', uninstallDisplayName: '${productName}', createDesktopShortcut: 'always', - // Why: electron-builder allows one include, so both Windows installer hooks live in it - - // the relocated-daemon uninstall sweep (guarded by ${isUpdated} so it never runs during an - // update's uninstallOldVersion) and the additive markdown "Open with" registration. - // Windows markdown association is deliberately NOT done via `fileAssociations`; see the - // header comment in that file for why that would steal the user's default .md handler. + // One NSIS include hosts update-guarded daemon cleanup and additive markdown Open With registration. + // Avoid fileAssociations: it would steal default .md ownership; see the include’s header. include: resolve(__dirname, 'nsis', 'orca-installer-hooks.nsh') }, mac: { - // Why rank Alternate: Orca joins Finder's "Open With" list without claiming - // LSHandlerRank ownership, so whichever editor the user already prefers stays the default. - // Why one entry per extension: app-builder-lib globs `*.${ext}`, which an array would break. + // Alternate rank preserves Finder’s default; one entry per extension because app-builder globs the value as a string. fileAssociations: [ ...MARKDOWN_FILE_EXTENSIONS.map((ext) => ({ ext, @@ -654,18 +552,9 @@ module.exports = { ...macTerminalHostSignIgnore ], extendInfo: macUsageDescriptions, - // Why: local macOS validation builds should launch without Apple release - // credentials. Hardened runtime + notarization stay enabled only on the - // explicit release path so production artifacts remain strict while dev - // artifacts do not fail with broken ad-hoc launch behavior. + // Local macOS builds need no release credentials; hardening/notarization stay on explicit release paths. hardenedRuntime: isMacRelease, - // Why dev builds notarize too, despite the ~10min notary round trip: TCC - // anchors a notarized Developer ID app's permission grants on identifier + - // team, which is cdhash-independent and so survives an update. Without a - // ticket there is no such stable identity, so every build reads as a - // different client — the grant row stays but stops matching, and file access - // under Documents/Desktop/Downloads fails with EPERM and no re-prompt. At 24 - // builds a day that revokes the user's grants faster than they can re-grant. + // Dev notarization preserves TCC identifier/team identity across cdhash changes; unticketed hourly updates otherwise lose folder grants with EPERM and no prompt. notarize: isMacRelease, extraResources: [ ...commonExtraResources, @@ -685,9 +574,7 @@ module.exports = { }, featureWallResources ], - // Why: the notification-status helper must execute from Contents/MacOS — - // on macOS 26 UNUserNotificationCenter aborts (bundleProxyForCurrentProcess - // is nil) for executables launched out of Contents/Resources (#7929). + // macOS 26 UNUserNotificationCenter aborts helpers launched from Resources; execute from Contents/MacOS (#7929). extraFiles: [ { from: 'native/notification-status-macos/.build/release/orca-notification-status', @@ -716,17 +603,12 @@ module.exports = { artifactName: 'orca-macos-${arch}.${ext}' }, linux: { - // Why mimeTypes and not fileAssociations: shared-mime-info already maps *.md/*.markdown to - // text/markdown, so reusing that type puts Orca in the Open With list without shipping a glob - // override. A desktop entry's MimeType only adds a handler - mimeapps.list still owns the - // default. .mdx is deliberately absent: Ubuntu 24.04's mime database maps it to - // application/x-genesis-32x-rom, so claiming it here would need a glob override. + // shared-mime-info already maps md/markdown: advertise handler only, leaving mimeapps.list defaults intact. + // Omit mdx because Ubuntu 24.04 maps it to application/x-genesis-32x-rom; claiming it would need a glob override. mimeTypes: ['text/markdown', 'text/csv', 'text/tab-separated-values'], - // Why: Ubuntu desktop ships GNOME Orca as the `orca` package and /usr/bin/orca. - // The Linux installer should not claim those system package/file names. + // Avoid GNOME Orca’s orca package and /usr/bin/orca names. executableName: 'orca-ide', - // Why: the icns source lets electron-builder emit standard hicolor PNG - // sizes; a single 1024px PNG is ignored by some Linux docks/launchers. + // Use icns to generate hicolor sizes; some Linux docks ignore a lone 1024px PNG. icon: 'resources/build/icon.icns', desktop: { entry: { @@ -768,9 +650,7 @@ module.exports = { deb: { packageName: 'orca-ide', artifactName: 'orca-ide_${version}_${arch}.${ext}', - // Why: xvfb lets the bundled `orca serve` CLI run browser panes on a headless - // Linux host — Chromium needs a display server even for offscreen rendering, - // and serve starts Xvfb itself when present (see ensure-virtual-display.ts). + // Headless Chromium still needs display service; orca serve starts packaged Xvfb when available (ensure-virtual-display.ts). depends: [ ...debElectronRuntimeDependencies, 'python3', @@ -781,10 +661,7 @@ module.exports = { 'xclip', 'xvfb' ], - // Why: symlink the bundled CLI onto PATH at install time so `orca-ide serve` - // works on a headless host. The in-app CLI registration (CliInstaller) is - // GUI-triggered and can never run on a server, so without this the CLI is - // unreachable from the shell on exactly the hosts that need it. + // Install CLI onto PATH for headless hosts where GUI-triggered CliInstaller can never run. afterInstall: 'resources/linux/packaging/after-install.sh', afterRemove: 'resources/linux/packaging/after-remove.sh', // Why: only an AppImage's files can vanish under a running daemon (see appimage-node-runtime.cjs). @@ -810,21 +687,14 @@ module.exports = { fpm: appImageNodeRuntimeFpmArgs }, beforeBuild: electronBuilderNativeRebuild, - // Why: must be true so that electron-builder rebuilds native modules - // (node-pty) for each target architecture when producing dual-arch macOS - // builds (x64 + arm64). With npmRebuild disabled, CI on an arm64 runner - // packages arm64 binaries into the x64 DMG, causing "posix_spawnp failed" - // on Intel Macs. The beforeBuild hook performs Orca's targeted rebuild and - // returns false so electron-builder does not rebuild optional cpu-features. + // Dual-arch builds need target node-pty rebuilds or Intel Macs get arm64 binaries and posix_spawnp failure. + // beforeBuild performs targeted rebuild then returns false to skip optional cpu-features. npmRebuild: true, publish: { provider: 'github', owner: 'stablyai', repo: devChannelRepo ?? 'orca', - // Why draft on the main repo: `--publish always` otherwise creates a - // public GitHub release as soon as the first platform uploads, and - // /releases/latest serves a missing Windows exe. release-cut undrafts - // only after every required asset exists. + // Keep main-repo releases draft until release-cut verifies all assets; early --publish always would expose missing Windows installers. releaseType: devChannelRepo ? 'prerelease' : 'draft' } } @@ -848,8 +718,7 @@ function chmodUnixCliLaunchers(resourcesDir, electronPlatformName) { if (!existsSync(launcherPath)) { continue } - // Why: packaged Unix installs expose these extraResources as public shell - // commands, and source/packager mode drift must not ship a non-executable CLI. + // Public shell commands need executable extraResources modes despite source/packager drift. chmodSync(launcherPath, 0o755) } } diff --git a/config/flaky-test-pattern-baseline.txt b/config/flaky-test-pattern-baseline.txt index 465f6801c65..6a2e686d585 100644 --- a/config/flaky-test-pattern-baseline.txt +++ b/config/flaky-test-pattern-baseline.txt @@ -30,7 +30,6 @@ 2 test-no-polling-loop:src/main/daemon/repro-12101-mouse-tracking-survives-agent-death.test.ts 2 test-no-polling-loop:src/main/git/repo-ref-maintenance-real-git.test.ts 1 test-no-polling-loop:src/main/git/worktree-removal-large-tree.bench.test.ts -1 test-no-polling-loop:src/main/ipc/filesystem-watcher-real.test.ts 1 test-no-polling-loop:src/main/ipc/folder-repo-git-upgrade.test.ts 1 test-no-polling-loop:src/main/ipc/native-chat.test.ts 1 test-no-polling-loop:src/main/ipc/runtime-environments/runtime-environment-request-connections.test.ts @@ -271,9 +270,6 @@ 1 test-no-real-time-wait:src/main/ipc/created-worktree-reconciliation.test.ts 1 test-no-real-time-wait:src/main/ipc/filesystem-auth.test.ts 1 test-no-real-time-wait:src/main/ipc/filesystem/filesystem-content-search-cancellation.test.ts -1 test-no-real-time-wait:src/main/ipc/filesystem-watcher-local-events.test.ts -1 test-no-real-time-wait:src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts -1 test-no-real-time-wait:src/main/ipc/filesystem-watcher-real.test.ts 5 test-no-real-time-wait:src/main/ipc/filesystem/local-disk-write-precondition.test.ts 2 test-no-real-time-wait:src/main/ipc/folder-repo-git-upgrade.test.ts 2 test-no-real-time-wait:src/main/ipc/native-chat.test.ts @@ -576,7 +572,7 @@ 1 test-no-real-time-wait:tests/e2e/native-chat-history-prepend-anchor.spec.ts 3 test-no-real-time-wait:tests/e2e/notification-sound-replay.spec.ts 1 test-no-real-time-wait:tests/e2e/onboarding.spec.ts -7 test-no-real-time-wait:tests/e2e/orchestration-idle-mail-delivery.spec.ts +6 test-no-real-time-wait:tests/e2e/orchestration-idle-mail-delivery.spec.ts 1 test-no-real-time-wait:tests/e2e/orchestration-idle-mail-restore.spec.ts 4 test-no-real-time-wait:tests/e2e/paced-terminal-typing.ts 1 test-no-real-time-wait:tests/e2e/paired-client-hosted-browser-double-restart.spec.ts @@ -639,7 +635,7 @@ 6 test-no-real-time-wait:tests/e2e/terminal-codex-cursor-jitter-repro.spec.ts 7 test-no-real-time-wait:tests/e2e/terminal-codex-local-typing-latency.spec.ts 8 test-no-real-time-wait:tests/e2e/terminal-codex-skill-preview-artifact-repro.spec.ts -11 test-no-real-time-wait:tests/e2e/terminal-column-desync-repro.spec.ts +9 test-no-real-time-wait:tests/e2e/terminal-column-desync-repro.spec.ts 2 test-no-real-time-wait:tests/e2e/terminal-column-probes.ts 2 test-no-real-time-wait:tests/e2e/terminal-document-visibility-webgl-recovery.spec.ts 2 test-no-real-time-wait:tests/e2e/terminal-hangul-terminating-digit-native.spec.ts @@ -835,7 +831,6 @@ 2 test-no-wall-clock:src/main/hook-archive-timeout-observation.test.ts 1 test-no-wall-clock:src/main/hooks-runner.test.ts 1 test-no-wall-clock:src/main/ipc/ai-vault.test.ts -2 test-no-wall-clock:src/main/ipc/filesystem-watcher-real.test.ts 3 test-no-wall-clock:src/main/ipc/folder-repo-git-upgrade.test.ts 2 test-no-wall-clock:src/main/ipc/native-chat-subscribe-lifecycle.test.ts 2 test-no-wall-clock:src/main/ipc/native-chat.test.ts @@ -1273,7 +1268,6 @@ 2 test-timeout-needs-reason:src/main/hook-archive-timeout-observation.test.ts 2 test-timeout-needs-reason:src/main/hook-termination-real-process.test.ts 2 test-timeout-needs-reason:src/main/ipc/filesystem/filesystem-quick-open-options.integration.test.ts -1 test-timeout-needs-reason:src/main/ipc/filesystem-watcher-lifecycle-model.test.ts 1 test-timeout-needs-reason:src/main/ipc/preflight-runnable-local-cli.test.ts 1 test-timeout-needs-reason:src/main/ipc/runtime-environments/runtime-environment-federated-read-transport.bench.test.ts 1 test-timeout-needs-reason:src/main/ipc/runtime-environments/runtime-environment-session-tabs-delta.integration.test.ts diff --git a/config/focus-setting-read-baseline.txt b/config/focus-setting-read-baseline.txt index 4082fa759ec..4abe37d5f59 100644 --- a/config/focus-setting-read-baseline.txt +++ b/config/focus-setting-read-baseline.txt @@ -69,7 +69,7 @@ 1 src/renderer/src/components/new-workspace/use-smart-workspace-secondary-searches.ts 2 src/renderer/src/components/new-workspace/use-work-item-source-foundation.ts 1 src/renderer/src/components/new-workspace/use-work-item-source-search.ts -3 src/renderer/src/components/right-sidebar/checks-panel-hosted-review-click-routing.ts +2 src/renderer/src/components/right-sidebar/checks-panel-hosted-review-click-routing.ts 1 src/renderer/src/components/right-sidebar/checks-panel/use-checks-panel-create-review.tsx 1 src/renderer/src/components/right-sidebar/FolderWorkspacePrChecksRow.tsx 1 src/renderer/src/components/right-sidebar/source-control/sync/git-history-commit-context-menu.tsx @@ -212,7 +212,7 @@ 1 src/renderer/src/hooks/ipc-events/settings-sidebar-ipc-bridge.ts 1 src/renderer/src/hooks/ipc-events/tab-lifecycle-ipc-bridge.ts 1 src/renderer/src/hooks/ipc-events/terminal-request-ipc-bridge.ts -2 src/renderer/src/hooks/use-active-skill-discovery-runtime-target.ts +1 src/renderer/src/hooks/use-active-skill-discovery-runtime-target.ts 1 src/renderer/src/hooks/useActiveProjectSkillRuntime.ts 1 src/renderer/src/hooks/useEditorExternalWatch.ts 1 src/renderer/src/hooks/useInstalledAgentSkills.ts @@ -334,7 +334,7 @@ 1 src/renderer/src/store/terminals/terminal-tab-close-providers.ts 1 src/renderer/src/web/preload-api/web-app-api.ts 3 src/renderer/src/web/preload-api/web-preference-normalization.ts -22 src/renderer/src/web/preload-api/web-preferences-store.ts +21 src/renderer/src/web/preload-api/web-preferences-store.ts 13 src/renderer/src/web/preload-api/web-settings-api.ts 9 src/renderer/src/web/preload-api/web-ui-api.ts 3 src/renderer/src/web/preload-api/web-workspace-session-api.ts diff --git a/config/nsis/orca-installer-hooks.nsh b/config/nsis/orca-installer-hooks.nsh index bcb94911a2f..b2ed925004f 100644 --- a/config/nsis/orca-installer-hooks.nsh +++ b/config/nsis/orca-installer-hooks.nsh @@ -1,28 +1,10 @@ -; electron-builder NSIS hooks for the Orca Windows installer. -; -; electron-builder accepts exactly ONE `nsis.include` file, so every customInstall / -; customUnInstall hook Orca needs lives here. +; electron-builder accepts one nsis.include; keep every customInstall/customUnInstall hook here. !include "${__FILEDIR__}\orca-process-check.nsh" -; --------------------------------------------------------------------------- -; Markdown and CSV/TSV "Open with Orca" (issues #10138, #23225) -; -; Why hand-rolled instead of electron-builder's `fileAssociations` on Windows: -; app-builder-lib emits !insertmacro APP_ASSOCIATE, whose first line is -; WriteRegStr SHELL_CONTEXT "Software\Classes\.md" "" "" -; That overwrites whichever editor currently owns .md, with no backup, for every -; existing user on their next UPDATE - and APP_UNASSOCIATE never restores it, so -; uninstalling Orca would leave .md pointing at a deleted ProgID. -; -; These writes are additive only. Registering a ProgID plus an OpenWithProgids -; hint and an Applications\\SupportedTypes entry puts Orca in Explorer's -; "Open with" list and in "Choose another app", while the default handler stays -; exactly where the user left it. Never add a `Software\Classes\.` default -; value here. -; -; Keep the extension list in sync with isOsOpenedDocumentName(). -; --------------------------------------------------------------------------- +; #10138/#23225: electron-builder’s APP_ASSOCIATE overwrites the extension’s default without backup, +; and APP_UNASSOCIATE never restores it. Register only additive ProgID/OpenWithProgids/SupportedTypes +; entries; never write the extension’s default handler. Sync extensions with isOsOpenedDocumentName(). !define MARKDOWN_PROGID "Orca.Markdown" !define TABULAR_PROGID "Orca.Tabular" @@ -55,40 +37,20 @@ System::Call "shell32::SHChangeNotify(i,i,i,i) (0x08000000, 0x1000, 0, 0)" !macroend -; --------------------------------------------------------------------------- -; Clean up the relocated terminal daemon on a REAL uninstall. -; -; Why: the daemon host is deliberately copied OUT of the install dir into -; %LOCALAPPDATA%\Orca\daemon-host-profiles so that app UPDATES cannot kill it — -; electron-builder's kill sweep selects processes whose image path is under -; $INSTDIR, and that relocation is what keeps terminals alive across updates. -; The same design means a normal uninstall's process sweep and file removal both -; miss it, leaving an orphaned daemon plus its runtime copy behind. -; -; The ${isUpdated} guard is essential: electron-builder runs this uninstaller as -; part of uninstallOldVersion on EVERY update, and killing the daemon there would -; defeat the whole feature. Only clean up on a genuine uninstall. -; -; The LOCALAPPDATA folder name must stay in sync with LOCAL_HOST_ROOT_NAME in -; src/main/daemon/daemon-host-relocation.ts. See -; docs/reference/windows-daemon-host-relocation.md. +; Relocated hosts escape $INSTDIR’s process/file sweep, surviving updates but requiring real-uninstall cleanup. +; Keep ${isUpdated}: uninstallOldVersion runs this hook on every update and must preserve terminals. +; Sync LOCALAPPDATA root with LOCAL_HOST_ROOT_NAME in src/main/daemon/daemon-host-relocation.ts; +; see docs/reference/windows-daemon-host-relocation.md. !macro customUnInstall ${ifNot} ${isUpdated} Push $0 Push $1 Push $2 - ; The host exe is a verbatim copy of the app exe, so the app's own image name - ; reaches it; the second name covers hosts left by builds that renamed the copy. - ; Filtered to the current user like upstream's per-user KILL_PROCESS, so an - ; elevated machine-wide uninstall cannot reach another logged-on user's session. - ; NSIS expands USERNAME itself: routing through cmd.exe only to get %USERNAME% - ; would add two interpreter spawns to the uninstall path for nothing. + ; The verbatim app exe and an older renamed host both need cleanup, scoped to the current user + ; like upstream KILL_PROCESS. NSIS expands USERNAME directly, avoiding two cmd.exe spawns. ReadEnvStr $1 USERNAME ${if} $1 == "" - ; Measured: taskkill rejects an empty filter value outright ("The search filter - ; cannot be recognized") and kills nothing, so with no USERNAME to scope by, - ; kill unfiltered rather than not at all. USERNAME is set in every session an - ; uninstaller runs in, so this is a backstop, not the expected path. + ; taskkill rejects an empty user filter and kills nothing. Fall back unfiltered if the normally set USERNAME is absent. StrCpy $2 "" ${else} StrCpy $2 '/FI "USERNAME eq $1"' @@ -105,8 +67,7 @@ RMDir /r "$LOCALAPPDATA\Orca\daemon-host" RMDir /r "$LOCALAPPDATA\Orca\daemon-host-profiles" ${endIf} - ; Why outside the ${isUpdated} guard: customInstall rewrites these on every update, so - ; dropping them during uninstallOldVersion is correct and keeps the pair symmetric. + ; customInstall rewrites associations on every update, so remove them during uninstallOldVersion too. DeleteRegKey SHELL_CONTEXT "Software\Classes\${MARKDOWN_PROGID}" DeleteRegKey SHELL_CONTEXT "Software\Classes\${TABULAR_PROGID}" !insertmacro ORCA_UNREGISTER_DOCUMENT_OPEN_WITH ".md" "${MARKDOWN_PROGID}" diff --git a/config/oxlint-plugins/app-store-performance.mjs b/config/oxlint-plugins/app-store-performance.mjs index 263e8b04ac0..b46ca7c15bf 100644 --- a/config/oxlint-plugins/app-store-performance.mjs +++ b/config/oxlint-plugins/app-store-performance.mjs @@ -99,12 +99,7 @@ function isIdentitySelector(selector) { ) } -/** - * `everyBranch` decides how a conditional counts. An inline selector is flagged - * when ANY branch allocates; a helper the selector delegates to must allocate on - * EVERY branch, so the `cache.get(k) ?? build(state)` identity-caching shape is - * not a false positive. - */ +// Inline selectors flag ANY allocating branch; delegated helpers require EVERY branch so cache.get(k) ?? build(state) remains valid. function allocates(expression, everyBranch) { const branches = expression?.type === 'ConditionalExpression' @@ -190,11 +185,7 @@ function resolveSelector(argument, state) { : null } -/** - * One hop: a selector that delegates to a module-scope helper is the idiomatic - * shape here, and neither the inline-body check nor a reviewer reading the call - * site can see what that helper returns. An unresolvable helper is left alone. - */ +// Inspect one delegated module helper hop to catch hidden allocations; leave unresolved helpers alone. function expandThroughNamedHelper(expression, state) { const helper = expression?.type === 'CallExpression' && expression.callee.type === 'Identifier' @@ -267,10 +258,7 @@ function requireSelectorRule(sourceCode) { } } -/** - * Selector arguments are collected during traversal and judged at Program:exit so a - * selector hoisted below its call site still resolves. - */ +// Judge at Program:exit so selectors declared after call sites resolve. function deferredSelectorRule(sourceCode, inspect) { const state = createRuleState(sourceCode) return { diff --git a/config/oxlint-plugins/ime-enter-guard.mjs b/config/oxlint-plugins/ime-enter-guard.mjs index 21c19ee2a71..87eebe96d8d 100644 --- a/config/oxlint-plugins/ime-enter-guard.mjs +++ b/config/oxlint-plugins/ime-enter-guard.mjs @@ -1,6 +1,4 @@ -// Flags Enter handling on raw text-entry surfaces that skips the shared IME guard, so the Enter -// that confirms a CJK candidate cannot submit half-composed text. /