Merge main and preserve registered SSH endpoint detection

This commit is contained in:
vam
2026-10-19 10:36:25 +08:00
32854 changed files with 4434225 additions and 943965 deletions
+88 -9
View File
@@ -1,24 +1,103 @@
/config/scripts/create-draft-release.mjs text eol=lf
/config/scripts/orca-dev.mjs text eol=lf
/config/scripts/latest-stable-release.mjs text eol=lf
/config/scripts/publish-complete-draft-releases.mjs text eol=lf
/config/scripts/release-rc-history.mjs text eol=lf
/config/scripts/run-internal-dev-setup.mjs text eol=lf
/config/scripts/verify-cli-bin.mjs text eol=lf
/config/scripts/verify-release-required-assets.mjs text eol=lf
# A shebang plus CRLF makes vite's SSR transform emit a literal `#!` mid-module,
# so any suite importing the script dies at load with a SyntaxError. Pin the whole
# directory rather than the scripts that happen to have a test today.
/config/scripts/**/*.mjs text eol=lf
/skill-guides/*.md text eol=lf
/skill-stubs/*.md text eol=lf
/skill-stubs/_shared/*.md text eol=lf
/skills/*/SKILL.md text eol=lf
/src/cli/bundled-skill-guides.ts text eol=lf
# SVG bytes feed embedded data URLs and asset hashes on every build host.
*.svg text eol=lf
# Compiler input line breaks must not depend on the checkout OS.
*.ts text eol=lf
*.tsx text eol=lf
*.html text eol=lf
# Bundled plugin trees are byte-hashed; CRLF checkout would break the pinned hash.
/resources/plugins/** text eol=lf
# pnpm hashes every patch byte-for-byte, so a CRLF checkout breaks the install.
# Relay assets are copied verbatim into the bundle and hashed byte-for-byte into
# .version, which names the immutable remote install dir. A CRLF checkout makes a
# Windows-built client disagree with a mac/Linux-built one on the same release,
# so one host ends up with two relay trees (#17886 review).
# Pin the bytes so a patch reads and diffs identically on every host. It is NOT
# what makes the hash right: pnpm hashes a patch LF-normalized, so a CRLF checkout
# cannot change it. Believing otherwise put a hand-computed raw digest in the
# lockfile twice and broke every install (#17886).
# These files are stored LF, which is not always the encoding they were written
# against -- @vscode/windows-process-tree ships CRLF sources -- so any code that
# runs `git apply` on one must force `-c core.autocrlf=input` rather than trust
# the host's setting. See config/scripts/windows-process-tree-gyp-rebuild.mjs.
/config/patches/*.patch -text
# Same reason, and pnpm parses these too: a CRLF checkout makes the mobile
# patches unparseable, so Windows packaging dies on ERR_PNPM_INVALID_PATCH.
/mobile/patches/*.patch -text
# The xterm bundle hunks also make a diff nobody can read; review the hand-written
# source patch under xterm-src/ instead. The sibling patches stay diffable.
/config/patches/@xterm__xterm@*.patch -diff
/config/patches/xterm-src/*.patch text eol=lf
# pnpm parses these unified diffs during Windows installs; keep checkout bytes stable.
/mobile/patches/*.patch -text
# Generated wrapper fixtures: collapse them in the PR diff so they stop drowning
# the reviewable change, and pin LF because they are compared byte-for-byte.
# Not -diff: the shell diff is the review surface when a wrapper does change.
/src/main/__fixtures__/shell-wrapper-snapshots/*.txt linguist-generated=true text eol=lf
# Captured agent PTY transcripts. -text, not `text eol=lf` like the wrapper snapshots above:
# these carry real CR and CRLF bytes as the terminal emitted them, and line-ending
# normalisation on a Windows checkout would rewrite the evidence the fixture exists to be.
/src/main/runtime/__fixtures__/*.txt -text
/src/main/daemon/__fixtures__/pty-transcripts/*.txt -text
/src/renderer/src/components/terminal-pane/terminal-link-transcripts/*.txt -text whitespace=cr-at-eol
# Generated runtime English subset: compared byte-for-byte by
# verify:localization-runtime-catalog, so a CRLF checkout would fail the gate.
/src/renderer/src/i18n/en-runtime-required.json linguist-generated=true text eol=lf
# Generated method->params catalog: compared byte-for-byte by
# verify:rpc-params-catalog, so a CRLF checkout would fail the gate.
/src/shared/rpc-contract/rpc-params-catalog.generated.ts linguist-generated=true text eol=lf
# Mobile RPC recording goldens: replay requires the file text to equal what rpc:record writes
# (LF), so a CRLF checkout would fail every golden.
/mobile/rpc-foundation/goldens/*.json text eol=lf
# Mobile web page source. Every text byte here is hashed into an asset digest and from
# there into buildId, so a CRLF Windows checkout would ship a different bundle id for
# identical source (91af2897 vs 9d78435e, measured on the bundle this replaced).
/mobile/src/** text eol=lf
/mobile/app/** text eol=lf
/mobile/web-entry/** text eol=lf
# The blanket pin above would mark a future binary as text; exempt the asset types an
# RN page actually carries.
/mobile/src/**/*.png -text
/mobile/src/**/*.jpg -text
/mobile/src/**/*.jpeg -text
/mobile/src/**/*.gif -text
/mobile/src/**/*.ico -text
/mobile/src/**/*.webp -text
/mobile/src/**/*.ttf -text
/mobile/src/**/*.otf -text
/mobile/src/**/*.woff -text
/mobile/src/**/*.woff2 -text
/mobile/app/**/*.png -text
/mobile/app/**/*.jpg -text
/mobile/app/**/*.jpeg -text
/mobile/app/**/*.gif -text
/mobile/app/**/*.ico -text
/mobile/app/**/*.webp -text
/mobile/app/**/*.ttf -text
/mobile/app/**/*.otf -text
/mobile/app/**/*.woff -text
/mobile/app/**/*.woff2 -text
/mobile/web-entry/**/*.png -text
/mobile/web-entry/**/*.jpg -text
/mobile/web-entry/**/*.jpeg -text
/mobile/web-entry/**/*.gif -text
/mobile/web-entry/**/*.ico -text
/mobile/web-entry/**/*.webp -text
/mobile/web-entry/**/*.ttf -text
/mobile/web-entry/**/*.otf -text
/mobile/web-entry/**/*.woff -text
/mobile/web-entry/**/*.woff2 -text
# Generated ACP schemas are checked byte-for-byte against formatted output.
/src/main/acp/generated/*.generated.ts linguist-generated=true text eol=lf
# Generated skill bundle artifacts are verified byte-for-byte against fresh output.
/resources/skills/*.json linguist-generated=true text eol=lf
# Vitest writes snapshots with LF, so checkout bytes must match on every host.
**/__snapshots__/*.snap text eol=lf
+5
View File
@@ -3,3 +3,8 @@
/config/scripts/*localization*.mjs @brennanb2025
/config/scripts/*locale*.mjs @brennanb2025
/config/i18next.config.ts @brennanb2025
# The relay's deploy and operate surface: workspace, workflows, and the rollout lease action.
/cloud/ @Jinwoo-H
/.github/workflows/cloud-*.yml @Jinwoo-H
/.github/actions/cloud-sql-rollout-lease/ @Jinwoo-H
+39 -3
View File
@@ -16,11 +16,38 @@ Thanks for contributing to Orca.
## Local Setup
Install Node 24, pnpm, and the Bun version in [`config/.bun-version`](../config/.bun-version).
`pnpm test` runs Vitest on Bun, with Node workers for runtime contracts such as SQLite,
native PTYs, socket liveness, and V8 memory behavior. `pnpm test:node` runs the same suites
entirely on Node. Builds and dependency installation still use Node and pnpm.
```bash
pnpm install
pnpm dev
```
To reduce main-process rebuild CPU and memory when you do not need its source maps,
set `ORCA_DEV_MAIN_SOURCEMAPS=0` before `pnpm dev`. In Bash/zsh:
```bash
ORCA_DEV_MAIN_SOURCEMAPS=0 pnpm dev
```
In PowerShell, use `$env:ORCA_DEV_MAIN_SOURCEMAPS='0'` before `pnpm dev`.
Remove the variable when debugging the main process with source maps. Renderer and
preload settings are unaffected; production builds always keep their hidden main maps.
Ordinary installs include native optional dependencies for the current OS and CPU only.
Before a cross-architecture build (including `pnpm build:mac`, which produces both x64 and
arm64 artifacts by default), run `pnpm install:release` to add the other CPU's variants.
Linux `pnpm build`, `pnpm build:native`, and Linux packaging require a running Docker
daemon to build the clipboard helper against Ubuntu 20.04 rather than the host's
newer libraries. A verified cached helper can be reused without Docker. For native
Wayland development, run `pnpm build:native` before `pnpm dev` to make text copying
available. See the [clipboard build guide](../native/wayland-clipboard/README.md#building-and-testing).
macOS and Windows builds do not use this helper or require Docker for it.
## Branch Naming
Use a clear, descriptive branch name that reflects the change.
@@ -44,6 +71,15 @@ pnpm test
pnpm build
```
For repeated local typechecks, use `pnpm tc` (or `pnpm tc:node`, `pnpm tc:web`,
and `pnpm tc:cli`). These run `bun check` from the pinned `bun` devDependency —
about a third of tsc's CPU and an eighth of its memory, so several agents can
typecheck at once. Pass `--compiler=native` to use tsc and its incremental
cache instead. CI uses `pnpm typecheck:ci`, the same Bun check with CI's
concurrency. Shared tests are checked by the
Node project; shared production code is checked in all three environments.
Run `pnpm tc:node` after editing shared tests, or `pnpm tc` to check everything.
Add high-quality tests for behavior changes and bug fixes. Prefer tests that would actually catch a regression, not shallow coverage that only exercises the happy path.
If your change affects UI or interaction behavior, verify it on the platforms it could impact.
@@ -58,9 +94,10 @@ CI enforces this for `src/preload/` and `src/shared/`.
Each pull request should follow [`.github/pull_request_template.md`](./pull_request_template.md). In particular:
- if you are an outside contributor, link the issue your PR addresses
- open with an ELI5 of the change (plain language paragraph; the PR title is the one-liner)
- explain what changed and why, and stay focused on a single topic when possible
- for any UI or interaction change, attach **before and after** screenshots (or short videos); if there is no visual change, say `No visual change` and why
- for any UI or interaction change, attach **before and after** screenshots (or short videos); if there is no visual or interaction change, write `N/A` and briefly explain why
- include high-quality tests when behavior changes or bug fixes warrant them
- include a brief code review summary from your AI coding agent that explicitly checks cross-platform compatibility, SSH/remote/local compatibility, supported agent and integration compatibility, performance risk, UI quality when applicable, and basic security risk
- mention any platform-specific, remote/SSH-specific, agent-specific, integration-specific, or git-provider-specific behavior and testing notes
@@ -101,7 +138,7 @@ All stable kinds (`patch`, `minor`, `major`) are computed off the latest _stable
- If the latest RC tag exists but is still draft-only or missing its GitHub Release, the workflow resumes that tag only when it was built from the current release ref. Otherwise the next RC number is cut.
- RC numbering also considers release commits on `main`, so deleting a stale tag does not let a later cut reuse the same RC number.
- Off-main releases (when `ref` is not the tip of `main`) only push the tag. `main` is never mutated from a non-main ref, so you can safely release an older commit without polluting history.
- When `ref` is the tip of `main`, the version-bump commit is fast-forwarded onto `main` so local `package.json` stays in sync with what's shipped.
- When `ref` is the tip of `main`, the workflow tries to fast-forward the version-bump commit onto `main`. If main requires pull requests, it leaves main unchanged and pushes only the release tag. Other push errors still stop the cut.
**Common scenarios:**
@@ -112,7 +149,6 @@ All stable kinds (`patch`, `minor`, `major`) are computed off the latest _stable
The scheduled 2x/day RC cron in [`release-rc.yml`](../../actions/workflows/release-rc.yml) is independent and continues to run automatically from `main`.
## Release Channels
The public Homebrew cask tracks stable desktop releases:
@@ -0,0 +1,11 @@
name: Build the mobile web bundle
description: Builds out/mobile-web, the page phones load, which every orcad build ships and refuses to build without.
runs:
using: composite
steps:
# Why: the bundle build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
- name: Build the mobile web bundle
shell: bash
run: pnpm run build:mobile-web
@@ -0,0 +1,152 @@
# Cloud SQL rollout lease
A compare-and-swap lease on one Cloud Storage object, used to serialize Cloud SQL
**connection-budget** rollouts across two repositories.
`concurrency.group: production-cloud-sql-rollout` only serializes runs inside a single repository.
Once the relay workflows live in `stablyai/orca` and the app workflows stay in
`stablyai/orca-cloud`, there are two independent queues pointed at one shared Cloud SQL instance.
`relay-cloud-sql-connection-budget.mjs` computes `rolloutOverlap` as a `Math.max` over the relay
director, api, auth and relay-cell candidates, which is only sound when exactly one rollout is in
flight. This lease is what keeps that assumption true. Keep the per-repo concurrency groups **and**
the lease; they solve different halves of the problem.
## What it protects
No workflow runs a Cloud SQL schema migration. Every locked workflow either deploys a Cloud Run
revision or applies a GCE instance template against the shared instance, so the lease must cover
**all rollouts**, not just migrations.
## Usage
The lease step must run **after** `google-github-actions/setup-gcloud`, and after
`actions/checkout` — `uses: ./.github/actions/...` resolves against the checked-out workspace.
It belongs in the first job of the workflow that holds a GCP credential, which is not always the
gate job: `deploy-relay-production-same-cap`'s gate runs no `gcloud`, so its first acquire happens
in the first cell job.
```yaml
- uses: google-github-actions/auth@v2
with: { workload_identity_provider: ..., service_account: ... }
- uses: google-github-actions/setup-gcloud@v2
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
```
Buckets and objects in use:
| Environment | Bucket | Object |
| ----------- | -------------------------------------- | --------------------------------------------------- |
| production | `onorca-cloud-terraform-state` | `terraform/state/cloud-sql-rollout/production.lock` |
| staging | `onorca-cloud-staging-terraform-state` | `terraform/state/cloud-sql-rollout/staging.lock` |
Workflows that serve both environments (`deploy-relay-asia-topology`,
`operate-relay-asia-admission`) select the pair with an `inputs.environment == 'production'`
ternary on both `bucket` and `object`. `deploy-staging` keeps its own `deploy-artifacts-staging`
concurrency group but takes the staging lease, because it rolls the staging API revision.
The object sits beside `terraform/state/relay-fence-broker/<env>.lock`. The IAM grant names both the
relay and app service accounts, so it is a **foundation-root** resource: `roles/storage.objectAdmin`
conditioned on the `terraform/state/cloud-sql-rollout/` prefix, **plus** an unconditioned
`roles/storage.legacyBucketReader`. Without the second role the generation-matched write fails in a
way that looks like a permissions flake.
## One lease per run, not per job
`deploy-relay-production-capacity` calls its reusable job six times and
`deploy-relay-production-same-cap` four times. Each call is a separate job on a separate runner, so
a naive per-job acquire/release would leave the object free between waves — for runs that have taken
up to 85 minutes.
The lease is therefore keyed to the **run**, not the job. `holder-key` defaults to
`${{ github.repository }}/${{ github.run_id }}`, and a job that finds its own holder key on a live
lease **re-enters** it: the record is refreshed, not rejected. Every job in the chain acquires; only
the last one releases.
```yaml
jobs:
gate:
steps:
- uses: ./.github/actions/cloud-sql-rollout-lease
with: { bucket: ..., object: ..., release: 'false' } # intermediate
wave-1: # ... release: 'false' on every wave job
release_lease:
needs: [gate, wave-1, wave-2, wave-3, wave-4]
if: always()
steps:
- uses: google-github-actions/auth@v2
- uses: google-github-actions/setup-gcloud@v2
- uses: ./.github/actions/cloud-sql-rollout-lease
with: { bucket: ..., object: ..., release: 'true' } # final
```
`release: 'false'` still acquires and still runs its `post` step; `post` only skips the delete. A
single-job workflow leaves `release` at its `true` default and needs no extra job. So does a
workflow whose several jobs can never hold the lease at once: `prove-relay-staging-capacity`'s two
lease-holding jobs are guarded by complementary `inputs.mode` conditions, and the contract test
checks that exclusivity rather than assuming it.
If the final job never runs (runner killed, run cancelled hard), the lease expires on its TTL.
## Timing
- **TTL 35 minutes**, matching `apps/relay-fence-broker/src/mutation-lease.ts`.
- **Renewal every 5 minutes.** `main` spawns a detached background Node process that rewrites
`expires_at` on the same generation-matched path; `post` kills it by pid read back from
`$GITHUB_STATE`. The renewer stops on its own the moment the object stops being ours, and has a
six-hour backstop in case `post` never runs. Its log is written to
`$RUNNER_TEMP/cloud-sql-rollout-lease-renewer.log` and echoed by `post`.
- Renewal is mandatory, not optional: capacity runs have taken 85 minutes, well past any sane TTL.
## Failure behaviour
| Situation | Behaviour |
| -------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
| Object absent | Acquire with `ifGenerationMatch: 0`. |
| Live lease, our own holder key | Re-enter. Refresh `expires_at`, keep `acquired_at`. Never fails. |
| Live lease, another holder | **Fail the job immediately**, printing the holder's repository, workflow and run URL. Never queues, never steals. |
| Expired lease | Take over with the observed generation and emit `::warning::` naming the stale holder. |
| `412` on write | Someone raced us. Fail as a conflict. |
| Bucket unreachable, `403`, `5xx` | **Fail closed.** |
| Record present but unparseable | **Fail closed.** A record we cannot read is never treated as free; an operator must inspect and delete it. |
| Release finds a foreign holder | Warn and leave it alone. Our lease had already expired. |
| Release fails | Warn only. `post` never fails a job over a release; the TTL bounds the damage. |
## Why `monitor-relay-production` must not use this
`monitor-relay-production` is in the `production-cloud-sql-rollout` concurrency group but is
**read-only**: its identity holds only monitoring, logging, Cloud SQL and compute _viewer_ roles,
and it runs `gcloud sql instances describe`, never a mutation. It consumes no connection budget.
Putting it on the durable lease would let a monitoring run block a real rollout, and a rollout block
monitoring exactly when an operator most needs it. Keep its same-repo concurrency group; keep it off
the lease. The lock census contract test records it in the not-a-candidate map with this reason.
## Token acquisition
`gcloud auth print-access-token`, not a hand-rolled exchange of the `external_account` credentials
file. Every consuming workflow already runs `setup-gcloud`, gcloud already handles every ADC flavour
including the service-account impersonation leg, and this action must stay zero-dependency because
it is duplicated by hand into the public repo. The GCE metadata server that
`apps/relay-fence-broker/src/google-metadata.ts` uses does **not** exist on GitHub or Blacksmith
runners; only the compare-and-swap algorithm is shared with the fence broker.
## Duplication
This directory is copied verbatim into `stablyai/orca`. It has no `package.json`, no
`node_modules`, and imports nothing outside itself — `action-contract.test.mjs` enforces all three.
Cross-repo consumption via `uses: stablyai/orca/.github/actions/...@<sha>` was rejected: it would
put public-repo code inside private app deploys that hold a production credential, and neither
repository protects `main` today.
## Tests
```
node --test .github/actions/cloud-sql-rollout-lease/
```
`storage-lease.test.mjs` drives the real compare-and-swap path against an in-memory Cloud Storage
fake that enforces generations. No network.
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict'
import { readFileSync, readdirSync } from 'node:fs'
import { test } from 'node:test'
const here = new URL('./', import.meta.url)
const action = readFileSync(new URL('action.yml', here), 'utf8')
const modules = readdirSync(here).filter((name) => name.endsWith('.mjs'))
const shipped = modules.filter((name) => !name.endsWith('.test.mjs'))
test('is a node24 JavaScript action with an always-run post step', () => {
// A composite action has no `post:`, so the lease could never be released on cancel or failure.
assert.match(action, /^ {2}using: node24$/m)
assert.doesNotMatch(action, /using: composite/)
assert.match(action, /^ {2}main: main\.mjs$/m)
assert.match(action, /^ {2}post: post\.mjs$/m)
assert.match(action, /^ {2}post-if: always\(\)$/m)
})
test('declares the inputs the wave-chain callers depend on', () => {
for (const input of ['bucket:', 'object:', 'holder-key:', 'release:']) {
assert.match(action, new RegExp(`^ {2}${input}$`, 'm'), input)
}
assert.match(action, /default: \$\{\{ github\.repository \}\}\/\$\{\{ github\.run_id \}\}/)
assert.match(action, /default: 'true'/)
})
test('stays self-contained so it can be duplicated into the public repo', () => {
assert.deepEqual(
readdirSync(here).filter((name) => name === 'package.json' || name === 'node_modules'),
[],
'the action must run with zero installed dependencies'
)
for (const name of modules) {
const source = readFileSync(new URL(name, here), 'utf8')
for (const match of source.matchAll(/^import\b[\s\S]*?from '([^']+)'/gm)) {
const specifier = match[1]
const local = specifier.startsWith('.')
assert.ok(
specifier.startsWith('node:') || (local && !specifier.includes('..')),
`${name} imports ${specifier}; only node: builtins and same-directory modules are allowed`
)
}
}
})
test('never reaches for the GCE metadata server', () => {
// Runners have no metadata.google.internal; the fence broker's token path must not be copied.
for (const name of shipped) {
const source = readFileSync(new URL(name, here), 'utf8')
assert.doesNotMatch(source, /metadata\.google\.internal/, name)
}
})
@@ -0,0 +1,41 @@
name: Cloud SQL rollout lease
description: >-
Serialize Cloud SQL connection-budget rollouts across repositories with a compare-and-swap lease
on a Cloud Storage object. Fails immediately when another run holds the lease; never queues,
never steals.
inputs:
bucket:
description: Terraform state bucket that holds the lease object.
required: true
object:
description: Lease object name, e.g. terraform/state/cloud-sql-rollout/production.lock
required: true
holder-key:
description: >-
Identity that owns the lease. Every job in one run must pass the same value; a job that finds
its own holder key on a live lease re-enters it instead of failing.
required: false
default: ${{ github.repository }}/${{ github.run_id }}
release:
description: >-
Release the lease in the post step. Set to "false" on every job of a multi-job wave except the
final always() job, which sets "true".
required: false
default: 'true'
outputs:
holder-key:
description: The holder key written to the lease object.
generation:
description: Cloud Storage generation of the lease object after acquisition.
expires-at:
description: ISO-8601 instant at which the lease expires without renewal.
reentrant:
description: '"true" when this job re-entered a lease its own run already held.'
runs:
using: node24
main: main.mjs
post: post.mjs
post-if: always()
@@ -0,0 +1,46 @@
import { execFileSync } from 'node:child_process'
// DESIGN CHOICE: shell out to `gcloud auth print-access-token` instead of exchanging the
// external_account credentials file that google-github-actions/auth writes.
//
// Every workflow on the Cloud SQL rollout lease already runs google-github-actions/setup-gcloud
// right after auth (verified across all 11 mutating members), so gcloud is on PATH and already
// bound to the federated identity. Doing the exchange ourselves would mean reimplementing the STS
// token swap plus the service-account impersonation leg, in an action that must stay
// zero-dependency and is duplicated by hand into a second repo. gcloud already handles every ADC
// flavour and refreshes on its own. The metadata server is not an option: it does not exist on
// GitHub or Blacksmith runners.
//
// Consequence, documented in the README: the lease step MUST come after setup-gcloud.
const TOKEN_REUSE_MS = 40 * 60 * 1_000 // GCP access tokens live ~60 min; re-mint well before that.
export function createAccessTokenSource({ run = runGcloud, now = Date.now } = {}) {
let cached = null
return () => {
if (cached && cached.mintedAt + TOKEN_REUSE_MS > now()) {
return cached.token
}
const token = run()
if (!token) {
throw new Error('gcloud auth print-access-token returned an empty token')
}
cached = { token, mintedAt: now() }
return token
}
}
function runGcloud() {
const binary = process.platform === 'win32' ? 'gcloud.cmd' : 'gcloud'
try {
return execFileSync(binary, ['auth', 'print-access-token'], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 60_000
}).trim()
} catch (error) {
// Never surface stdout; it is the token on success and noise on failure.
const detail = String(error?.stderr ?? '').trim() || error?.message || 'unknown failure'
throw new Error(`could not mint a GCP access token via gcloud: ${detail}`)
}
}
@@ -0,0 +1,20 @@
// Who we claim to be on the lease object. Shared by main and the detached renewer so both agree
// on the holder key without re-deriving it from a different set of environment variables.
export function holderIdentity(explicitHolderKey) {
const repository = process.env.GITHUB_REPOSITORY ?? 'unknown'
const runId = process.env.GITHUB_RUN_ID ?? 'unknown'
const server = process.env.GITHUB_SERVER_URL ?? 'https://github.com'
const holderKey = explicitHolderKey || `${repository}/${runId}`
if (/[\r\n]/.test(holderKey)) {
throw new Error('holder-key must be single-line')
}
return {
holderKey,
repository,
workflow: process.env.GITHUB_WORKFLOW ?? 'unknown',
runId,
runUrl: `${server}/${repository}/actions/runs/${runId}`,
runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? 'unknown'
}
}
@@ -0,0 +1,81 @@
import { spawn } from 'node:child_process'
import { fileURLToPath } from 'node:url'
import { createAccessTokenSource } from './gcloud-access-token.mjs'
import { holderIdentity } from './holder-identity.mjs'
import { fail, input, notice, renewerLogPath, saveState, setOutput, warn } from './runner-state.mjs'
import { CloudSqlRolloutLease, LeaseConflict, describeHolder } from './storage-lease.mjs'
const bucket = input('bucket')
const objectName = input('object')
const release = input('release') !== 'false'
if (!bucket || !objectName) {
fail('cloud-sql-rollout-lease requires both `bucket` and `object`')
process.exit(1)
}
const holder = holderIdentity(input('holder-key'))
const lease = new CloudSqlRolloutLease({
bucket,
objectName,
accessToken: createAccessTokenSource()
})
let claim
try {
claim = await lease.acquire(holder)
} catch (error) {
if (error instanceof LeaseConflict) {
fail(
`${error.message}. Cloud SQL rollouts are serialized across repositories; this run will not queue or steal the lease. Wait for the holder to finish, then re-run.`
)
if (error.holder) {
console.log(`Lease holder repository: ${error.holder.repository}`)
console.log(`Lease holder workflow: ${error.holder.workflow}`)
console.log(`Lease holder run: ${error.holder.run_url}`)
}
} else {
// Bucket unreachable, permission denied, unreadable record: fail closed.
fail(`could not acquire ${lease.uri}: ${error.message}`)
}
process.exit(1)
}
// Persist before anything else can throw, so `post` always releases what we hold.
saveState('acquired', 'true')
saveState('bucket', bucket)
saveState('object', objectName)
saveState('holder_key', holder.holderKey)
saveState('release', release ? 'true' : 'false')
setOutput('holder-key', holder.holderKey)
setOutput('generation', claim.generation)
setOutput('expires-at', new Date(claim.record.expires_at).toISOString())
setOutput('reentrant', claim.state === 'reentrant' ? 'true' : 'false')
if (claim.state === 'reentrant') {
notice(
`Re-entered the Cloud SQL rollout lease on ${lease.uri} already held by this run; refreshed to ${new Date(claim.record.expires_at).toISOString()}.`
)
} else if (claim.state === 'takeover') {
notice(`Took over ${lease.uri} from ${describeHolder(claim.previous)}.`)
} else {
notice(
`Acquired ${lease.uri} until ${new Date(claim.record.expires_at).toISOString()} (holder ${holder.holderKey}).`
)
}
try {
const renewer = spawn(
process.execPath,
[fileURLToPath(new URL('./renew.mjs', import.meta.url)), bucket, objectName, holder.holderKey],
{ detached: true, stdio: 'ignore', env: process.env }
)
renewer.unref()
saveState('renewer_pid', String(renewer.pid))
const log = renewerLogPath()
notice(`Lease renewer running as pid ${renewer.pid}${log ? `, logging to ${log}` : ''}.`)
} catch (error) {
// A missing renewer is survivable for short jobs; the TTL still covers 35 minutes.
warn(`could not start the lease renewer: ${error.message}. The lease will expire on its TTL.`)
}
@@ -0,0 +1,81 @@
import { readFileSync } from 'node:fs'
import { createAccessTokenSource } from './gcloud-access-token.mjs'
import { notice, renewerLogPath, savedState, warn } from './runner-state.mjs'
import { CloudSqlRolloutLease, describeHolder } from './storage-lease.mjs'
stopRenewer()
printRenewerLog()
if (savedState('acquired') !== 'true') {
notice('No Cloud SQL rollout lease was acquired by this step; nothing to release.')
process.exit(0)
}
const bucket = savedState('bucket')
const objectName = savedState('object')
const holderKey = savedState('holder_key')
if (savedState('release') !== 'true') {
notice(
`Holding gs://${bucket}/${objectName} for the rest of run ${holderKey}; a later job with release=true must free it.`
)
process.exit(0)
}
const lease = new CloudSqlRolloutLease({
bucket,
objectName,
accessToken: createAccessTokenSource()
})
try {
const result = await lease.release(holderKey)
if (result.released) {
notice(`Released ${lease.uri} at generation ${result.generation}.`)
} else if (result.reason === 'absent') {
notice(`${lease.uri} was already gone; nothing to release.`)
} else if (result.reason === 'foreign') {
warn(
`${lease.uri} is now held by ${describeHolder(result.holder)}; leaving it alone. Our lease had already expired.`
)
} else {
warn(`${lease.uri} changed while releasing it; leaving it to expire on its TTL.`)
}
} catch (error) {
// Never fail a job in post over a release; the TTL bounds the damage to 35 minutes.
warn(`could not release ${lease.uri}: ${error.message}. It will expire on its TTL.`)
}
function stopRenewer() {
const pid = Number(savedState('renewer_pid'))
if (!Number.isInteger(pid) || pid <= 0) {
return
}
try {
process.kill(pid, 'SIGTERM')
notice(`Stopped the lease renewer (pid ${pid}).`)
} catch (error) {
if (error?.code !== 'ESRCH') {
warn(`could not stop the lease renewer ${pid}: ${error.message}`)
}
}
}
function printRenewerLog() {
const path = renewerLogPath()
if (!path) {
return
}
let text = ''
try {
text = readFileSync(path, 'utf8')
} catch {
return
}
if (!text.trim()) {
return
}
console.log('::group::Cloud SQL rollout lease renewer log')
console.log(text.trimEnd())
console.log('::endgroup::')
}
@@ -0,0 +1,73 @@
import { appendFileSync } from 'node:fs'
import { createAccessTokenSource } from './gcloud-access-token.mjs'
import { renewerLogPath } from './runner-state.mjs'
import { CloudSqlRolloutLease, RENEW_INTERVAL_MS } from './storage-lease.mjs'
// Detached renewer. `main` spawns it, `post` kills it. It rewrites expires_at on the same
// generation-matched path as acquisition, and stops the moment the object stops being ours.
const MAX_LIFETIME_MS = 6 * 60 * 60 * 1_000 // Backstop if post never runs (runner killed).
const [bucket, objectName, holderKey] = process.argv.slice(2)
const logPath = renewerLogPath()
const startedAt = Date.now()
function log(message) {
if (!logPath) {
return
}
try {
appendFileSync(logPath, `${new Date().toISOString()} ${message}\n`)
} catch {
// A renewer that cannot log must still renew.
}
}
if (!bucket || !objectName || !holderKey) {
log('renewer started without bucket/object/holder-key; exiting')
process.exit(1)
}
const lease = new CloudSqlRolloutLease({
bucket,
objectName,
accessToken: createAccessTokenSource(),
warn: (message) => log(`warning ${message}`)
})
let stopping = false
for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) {
process.on(signal, () => {
stopping = true
log(`received ${signal}; stopping`)
process.exit(0)
})
}
log(`renewer started for ${lease.uri} holder=${holderKey} interval=${RENEW_INTERVAL_MS}ms`)
while (!stopping) {
await new Promise((resolve) => {
setTimeout(resolve, RENEW_INTERVAL_MS)
})
if (stopping) {
break
}
if (Date.now() - startedAt > MAX_LIFETIME_MS) {
log('renewer hit its maximum lifetime; stopping so the lease can expire')
break
}
try {
const result = await lease.renew(holderKey)
if (!result.renewed) {
log(`lease is no longer ours (${result.reason}); stopping`)
break
}
log(
`renewed until ${new Date(result.record.expires_at).toISOString()} at generation ${result.generation}`
)
} catch (error) {
// Transient GCS or token failures are retried on the next tick; the TTL covers 7 misses.
log(`renewal attempt failed: ${error.message}`)
}
}
@@ -0,0 +1,55 @@
import { appendFileSync } from 'node:fs'
// Action-state and output plumbing via the runner's file protocol, so the action needs no
// @actions/core dependency. Values are single-line by construction; anything else is rejected.
/** The detached renewer's stdio is ignored, so it appends here instead and `post` echoes it. */
export function renewerLogPath() {
const dir = process.env.RUNNER_TEMP
return dir ? `${dir}/cloud-sql-rollout-lease-renewer.log` : null
}
export function input(name) {
return (process.env[`INPUT_${name.replace(/ /g, '_').toUpperCase()}`] ?? '').trim()
}
export function savedState(name) {
return (process.env[`STATE_${name}`] ?? '').trim()
}
export function saveState(name, value) {
appendToEnvFile('GITHUB_STATE', name, value)
}
export function setOutput(name, value) {
appendToEnvFile('GITHUB_OUTPUT', name, value)
}
export function notice(message) {
console.log(`::notice::${oneLine(message)}`)
}
export function warn(message) {
console.log(`::warning::${oneLine(message)}`)
}
export function fail(message) {
console.log(`::error::${oneLine(message)}`)
process.exitCode = 1
}
function appendToEnvFile(variable, name, value) {
const text = String(value)
if (/[\r\n]/.test(text)) {
throw new Error(`${name} must be single-line`)
}
const path = process.env[variable]
if (!path) {
return
} // Running outside a runner (local smoke run); nothing to persist.
appendFileSync(path, `${name}=${text}\n`)
}
function oneLine(message) {
return String(message).replace(/\r?\n/g, ' ')
}
@@ -0,0 +1,240 @@
// Compare-and-swap lease over a single Cloud Storage object.
//
// Ported from apps/relay-fence-broker/src/mutation-lease.ts rather than imported: this action is
// duplicated verbatim into stablyai/orca, so it must carry no repo-local imports. Only the
// algorithm is shared (read metadata -> write with ifGenerationMatch -> 412 is a conflict ->
// generation-matched delete -> an expired record is free). The broker's token path is NOT shared;
// it reads the GCE metadata server, which does not exist on Actions runners.
export const LEASE_TTL_MS = 35 * 60 * 1_000
export const RENEW_INTERVAL_MS = 5 * 60 * 1_000
const GENERATION = /^[1-9][0-9]{0,30}$/
export class LeaseConflict extends Error {
constructor(message, holder) {
super(message)
this.name = 'LeaseConflict'
this.holder = holder ?? null
}
}
/** A live record we cannot parse is never treated as free; wedging beats double-rollout. */
export class LeaseUnreadable extends Error {
constructor(message) {
super(message)
this.name = 'LeaseUnreadable'
}
}
function parseRecord(raw) {
if (!raw || typeof raw !== 'object') {
return null
}
if (typeof raw.holder_key !== 'string' || raw.holder_key.length === 0) {
return null
}
if (!Number.isSafeInteger(raw.acquired_at) || !Number.isSafeInteger(raw.expires_at)) {
return null
}
return {
repository: typeof raw.repository === 'string' ? raw.repository : 'unknown',
workflow: typeof raw.workflow === 'string' ? raw.workflow : 'unknown',
run_id: typeof raw.run_id === 'string' ? raw.run_id : 'unknown',
run_url: typeof raw.run_url === 'string' ? raw.run_url : 'unknown',
run_attempt: typeof raw.run_attempt === 'string' ? raw.run_attempt : 'unknown',
acquired_at: raw.acquired_at,
expires_at: raw.expires_at,
holder_key: raw.holder_key
}
}
function describe(record) {
return `${record.repository} / ${record.workflow} (run ${record.run_id}, attempt ${record.run_attempt}) ${record.run_url}`
}
export class CloudSqlRolloutLease {
#bucket
#objectName
#accessToken
#fetcher
#now
#warn
constructor({
bucket,
objectName,
accessToken,
fetcher = fetch,
now = Date.now,
warn = (message) => console.log(`::warning::${message}`)
}) {
this.#bucket = bucket
this.#objectName = objectName
this.#accessToken = accessToken
this.#fetcher = fetcher
this.#now = now
this.#warn = warn
}
get uri() {
return `gs://${this.#bucket}/${this.#objectName}`
}
async acquire(holder) {
const existing = await this.read()
const now = this.#now()
if (!existing) {
return this.#claim(holder, '0', now, now, 'created')
}
if (existing.record.holder_key === holder.holderKey) {
// Same run, another job in the wave chain. Refresh, never fail.
return this.#claim(
holder,
existing.generation,
existing.record.acquired_at,
now,
'reentrant',
existing.record
)
}
if (existing.record.expires_at > now) {
throw new LeaseConflict(
`${this.uri} is held by ${describe(existing.record)} until ${new Date(existing.record.expires_at).toISOString()}`,
existing.record
)
}
this.#warn(
`Taking over an expired Cloud SQL rollout lease on ${this.uri}. Stale holder: ${describe(existing.record)}, expired ${new Date(existing.record.expires_at).toISOString()}.`
)
return this.#claim(holder, existing.generation, now, now, 'takeover', existing.record)
}
async renew(holderKey) {
const existing = await this.read()
if (!existing) {
return { renewed: false, reason: 'absent' }
}
if (existing.record.holder_key !== holderKey) {
return { renewed: false, reason: 'foreign' }
}
const now = this.#now()
const record = { ...existing.record, expires_at: now + LEASE_TTL_MS }
const written = await this.#write(record, existing.generation)
return { renewed: true, generation: written.generation, record }
}
async release(holderKey) {
const existing = await this.read()
if (!existing) {
return { released: false, reason: 'absent' }
}
if (existing.record.holder_key !== holderKey) {
return { released: false, reason: 'foreign', holder: existing.record }
}
const response = await this.#fetcher(
`${this.#metadataUrl()}?ifGenerationMatch=${encodeURIComponent(existing.generation)}`,
{ method: 'DELETE', headers: { Authorization: `Bearer ${await this.#token()}` } }
)
if (response.status === 412) {
return { released: false, reason: 'conflict' }
}
if (!response.ok && response.status !== 404) {
throw new Error(`lease release failed: ${response.status}`)
}
return { released: true, generation: existing.generation }
}
async read() {
const token = await this.#token()
const metadataResponse = await this.#fetcher(this.#metadataUrl(), {
headers: { Authorization: `Bearer ${token}` }
})
if (metadataResponse.status === 404) {
return null
}
if (!metadataResponse.ok) {
throw new Error(`lease inspection failed: ${metadataResponse.status}`)
}
const metadata = await metadataResponse.json()
if (!GENERATION.test(metadata?.generation ?? '')) {
throw new LeaseUnreadable(`${this.uri} has no valid generation`)
}
const bodyResponse = await this.#fetcher(`${this.#metadataUrl()}?alt=media`, {
headers: { Authorization: `Bearer ${token}` }
})
if (bodyResponse.status === 404) {
return null
}
if (!bodyResponse.ok) {
throw new Error(`lease body read failed: ${bodyResponse.status}`)
}
let raw = null
try {
raw = await bodyResponse.json()
} catch {
raw = null
}
const record = parseRecord(raw)
if (!record) {
throw new LeaseUnreadable(
`${this.uri} holds an unreadable lease record; an operator must inspect and delete it before rollouts can resume`
)
}
return { generation: metadata.generation, record }
}
async #claim(holder, generation, acquiredAt, now, state, previous) {
const record = {
repository: holder.repository,
workflow: holder.workflow,
run_id: holder.runId,
run_url: holder.runUrl,
run_attempt: holder.runAttempt,
acquired_at: acquiredAt,
expires_at: now + LEASE_TTL_MS,
holder_key: holder.holderKey
}
const written = await this.#write(record, generation)
return { state, generation: written.generation, record, previous: previous ?? null }
}
async #write(record, generation) {
const response = await this.#fetcher(
`${this.#uploadUrl()}&ifGenerationMatch=${encodeURIComponent(generation)}`,
{
method: 'POST',
headers: {
Authorization: `Bearer ${await this.#token()}`,
'Content-Type': 'application/json'
},
body: JSON.stringify(record)
}
)
if (response.status === 412) {
throw new LeaseConflict(`${this.uri} changed concurrently while we were claiming it`)
}
if (!response.ok) {
throw new Error(`lease write failed: ${response.status}`)
}
const metadata = await response.json()
if (!GENERATION.test(metadata?.generation ?? '')) {
throw new LeaseUnreadable(`${this.uri} write returned no valid generation`)
}
return { generation: metadata.generation }
}
async #token() {
return typeof this.#accessToken === 'function' ? await this.#accessToken() : this.#accessToken
}
#metadataUrl() {
return `https://storage.googleapis.com/storage/v1/b/${encodeURIComponent(this.#bucket)}/o/${encodeURIComponent(this.#objectName)}`
}
#uploadUrl() {
return `https://storage.googleapis.com/upload/storage/v1/b/${encodeURIComponent(this.#bucket)}/o?uploadType=media&name=${encodeURIComponent(this.#objectName)}`
}
}
export const describeHolder = describe
@@ -0,0 +1,324 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { createAccessTokenSource } from './gcloud-access-token.mjs'
import {
CloudSqlRolloutLease,
LEASE_TTL_MS,
LeaseConflict,
LeaseUnreadable
} from './storage-lease.mjs'
const BUCKET = 'onorca-cloud-terraform-state'
const OBJECT = 'terraform/state/cloud-sql-rollout/production.lock'
const NOW = 1_756_000_000_000
/**
* Enough of the Cloud Storage JSON API to exercise real compare-and-swap semantics: generations
* increment, ifGenerationMatch is enforced, and a mismatch is a 412. `faults` injects failures.
*/
function fakeStorage({ object = null, faults = [] } = {}) {
const state = { object, requests: [] }
const fetcher = async (rawUrl, init = {}) => {
const url = new URL(rawUrl)
const method = init.method ?? 'GET'
const record = { method, url, path: url.pathname, search: url.searchParams }
state.requests.push(record)
const fault = faults.find((candidate) => candidate.when(record))
if (fault) {
return json(fault.status, fault.body ?? {})
}
if (url.pathname.startsWith('/upload/')) {
const want = url.searchParams.get('ifGenerationMatch')
const have = state.object ? state.object.generation : '0'
if (want !== have) {
return json(412, {})
}
const generation = String(Number(have === '0' ? '1000' : have) + 1)
state.object = { generation, body: JSON.parse(init.body) }
return json(200, { generation })
}
if (method === 'DELETE') {
if (!state.object) {
return json(404, {})
}
if (url.searchParams.get('ifGenerationMatch') !== state.object.generation) {
return json(412, {})
}
state.object = null
return json(204, {})
}
if (!state.object) {
return json(404, {})
}
if (url.searchParams.get('alt') === 'media') {
return json(200, state.object.body)
}
return json(200, { generation: state.object.generation })
}
return { state, fetcher }
}
function json(status, body) {
return {
status,
ok: status >= 200 && status < 300,
json: async () => body
}
}
function storedRecord({
holderKey,
expiresAt,
repository = 'stablyai/orca',
workflow = 'Deploy Relay Production'
}) {
return {
repository,
workflow,
run_id: '9001',
run_url: 'https://github.com/stablyai/orca/actions/runs/9001',
run_attempt: '1',
acquired_at: NOW - 60_000,
expires_at: expiresAt,
holder_key: holderKey
}
}
function leaseFor(storage, { warn = () => {} } = {}) {
return new CloudSqlRolloutLease({
bucket: BUCKET,
objectName: OBJECT,
accessToken: 'test-token',
fetcher: storage.fetcher,
now: () => NOW,
warn
})
}
const HOLDER = {
holderKey: 'stablyai/orca-cloud/42',
repository: 'stablyai/orca-cloud',
workflow: 'Deploy Relay Production Same-Cap',
runId: '42',
runUrl: 'https://github.com/stablyai/orca-cloud/actions/runs/42',
runAttempt: '1'
}
test('acquires a lease on an empty object with ifGenerationMatch=0', async () => {
const storage = fakeStorage()
const claim = await leaseFor(storage).acquire(HOLDER)
assert.equal(claim.state, 'created')
const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
assert.equal(upload.search.get('ifGenerationMatch'), '0')
assert.equal(upload.search.get('name'), OBJECT)
assert.deepEqual(storage.state.object.body, {
repository: 'stablyai/orca-cloud',
workflow: 'Deploy Relay Production Same-Cap',
run_id: '42',
run_url: 'https://github.com/stablyai/orca-cloud/actions/runs/42',
run_attempt: '1',
acquired_at: NOW,
expires_at: NOW + LEASE_TTL_MS,
holder_key: 'stablyai/orca-cloud/42'
})
})
test('refuses a live lease held by another run and never writes', async () => {
const storage = fakeStorage({
object: {
generation: '1500',
body: storedRecord({ holderKey: 'stablyai/orca/9001', expiresAt: NOW + 60_000 })
}
})
const error = await leaseFor(storage)
.acquire(HOLDER)
.catch((thrown) => thrown)
assert.ok(error instanceof LeaseConflict)
assert.equal(error.holder.repository, 'stablyai/orca')
assert.equal(error.holder.run_url, 'https://github.com/stablyai/orca/actions/runs/9001')
assert.equal(
storage.state.requests.filter((request) => request.method !== 'GET').length,
0,
'a foreign live lease must not be written'
)
assert.equal(storage.state.object.generation, '1500')
})
test('re-enters a live lease this run already holds and extends it', async () => {
const storage = fakeStorage({
object: {
generation: '1500',
body: storedRecord({ holderKey: HOLDER.holderKey, expiresAt: NOW + 60_000 })
}
})
const warnings = []
const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire(
HOLDER
)
assert.equal(claim.state, 'reentrant')
assert.deepEqual(warnings, [], 're-entering our own lease is not a takeover')
assert.equal(claim.record.acquired_at, NOW - 60_000, 'original acquisition time is preserved')
assert.equal(claim.record.expires_at, NOW + LEASE_TTL_MS)
const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
assert.equal(upload.search.get('ifGenerationMatch'), '1500')
assert.equal(storage.state.object.generation, '1501')
})
test('takes over an expired lease and warns naming the stale holder', async () => {
const storage = fakeStorage({
object: {
generation: '1500',
body: storedRecord({
holderKey: 'stablyai/orca/9001',
expiresAt: NOW - 1,
repository: 'stablyai/orca',
workflow: 'Deploy Relay Production Capacity'
})
}
})
const warnings = []
const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire(
HOLDER
)
assert.equal(claim.state, 'takeover')
assert.equal(warnings.length, 1)
assert.match(warnings[0], /stablyai\/orca/)
assert.match(warnings[0], /Deploy Relay Production Capacity/)
assert.match(warnings[0], /actions\/runs\/9001/)
const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
assert.equal(upload.search.get('ifGenerationMatch'), '1500')
assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey)
})
test('releases with a generation match and leaves the object gone', async () => {
const storage = fakeStorage()
const lease = leaseFor(storage)
const claim = await lease.acquire(HOLDER)
const released = await lease.release(HOLDER.holderKey)
assert.deepEqual(released, { released: true, generation: claim.generation })
const remove = storage.state.requests.find((request) => request.method === 'DELETE')
assert.equal(remove.search.get('ifGenerationMatch'), claim.generation)
assert.equal(storage.state.object, null)
})
test('refuses to release a lease another run now holds', async () => {
const storage = fakeStorage({
object: {
generation: '1500',
body: storedRecord({ holderKey: 'stablyai/orca/9001', expiresAt: NOW + 60_000 })
}
})
const released = await leaseFor(storage).release(HOLDER.holderKey)
assert.equal(released.released, false)
assert.equal(released.reason, 'foreign')
assert.equal(storage.state.object.generation, '1500')
})
test('fails closed when the bucket answers 5xx', async () => {
const storage = fakeStorage({
faults: [{ when: (request) => request.method === 'GET', status: 503 }]
})
const error = await leaseFor(storage)
.acquire(HOLDER)
.catch((thrown) => thrown)
assert.match(error.message, /lease inspection failed: 503/)
assert.equal(storage.state.requests.filter((request) => request.method !== 'GET').length, 0)
})
test('fails closed when permission is denied', async () => {
const storage = fakeStorage({
faults: [{ when: (request) => request.method === 'GET', status: 403 }]
})
const error = await leaseFor(storage)
.acquire(HOLDER)
.catch((thrown) => thrown)
assert.match(error.message, /lease inspection failed: 403/)
})
test('treats an unreadable record as held, not free', async () => {
const storage = fakeStorage({
object: { generation: '1500', body: { holder_key: 'stablyai/orca/9001' } }
})
const error = await leaseFor(storage)
.acquire(HOLDER)
.catch((thrown) => thrown)
assert.ok(error instanceof LeaseUnreadable)
assert.equal(storage.state.object.generation, '1500')
})
test('reports a 412 during acquisition as a conflict', async () => {
const storage = fakeStorage({
faults: [{ when: (request) => request.path.startsWith('/upload/'), status: 412 }]
})
const error = await leaseFor(storage)
.acquire(HOLDER)
.catch((thrown) => thrown)
assert.ok(error instanceof LeaseConflict)
assert.match(error.message, /changed concurrently/)
})
test('renewal rewrites only expires_at on the observed generation', async () => {
const storage = fakeStorage()
const lease = leaseFor(storage)
await lease.acquire(HOLDER)
storage.state.object.body.expires_at = NOW - 1
const renewed = await lease.renew(HOLDER.holderKey)
assert.equal(renewed.renewed, true)
assert.equal(storage.state.object.body.expires_at, NOW + LEASE_TTL_MS)
assert.equal(storage.state.object.body.acquired_at, NOW)
assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey)
})
test('renewal stops once the object belongs to someone else', async () => {
const storage = fakeStorage({
object: {
generation: '1500',
body: storedRecord({ holderKey: 'stablyai/orca/9001', expiresAt: NOW + 60_000 })
}
})
assert.deepEqual(await leaseFor(storage).renew(HOLDER.holderKey), {
renewed: false,
reason: 'foreign'
})
})
test('the access token source re-mints only after the reuse window', () => {
let clock = 0
let mints = 0
const source = createAccessTokenSource({
run: () => `token-${++mints}`,
now: () => clock
})
assert.equal(source(), 'token-1')
clock = 39 * 60 * 1_000
assert.equal(source(), 'token-1')
clock = 41 * 60 * 1_000
assert.equal(source(), 'token-2')
})
test('the access token source rejects an empty gcloud response', () => {
const source = createAccessTokenSource({ run: () => '' })
assert.throws(() => source(), /empty token/)
})
@@ -0,0 +1,42 @@
name: Download pinned Orca release
description: Downloads a released Linux .deb, checks its SHA-512, extracts it and points ORCA_CROSS_VERSION_PACKAGED_EXECUTABLE at its executable.
inputs:
version:
description: Release version without the leading v (e.g. 1.4.224).
required: true
sha512:
description: Base64 SHA-512 of orca-ide_<version>_amd64.deb.
required: true
github-token:
description: Token for gh release download.
required: true
runs:
using: composite
steps:
- name: Download and verify
shell: bash
env:
GH_TOKEN: ${{ inputs.github-token }}
ORCA_PINNED_VERSION: ${{ inputs.version }}
ORCA_PINNED_SHA512: ${{ inputs.sha512 }}
run: |
gh release download "v$ORCA_PINNED_VERSION" --repo stablyai/orca --pattern "orca-ide_${ORCA_PINNED_VERSION}_amd64.deb" --dir "$RUNNER_TEMP/old-orca"
python3 - <<'PYVERIFY'
import base64,hashlib,os,pathlib,subprocess
version=os.environ['ORCA_PINNED_VERSION']
root=pathlib.Path(os.environ['RUNNER_TEMP'])/'old-orca'
package=root/f'orca-ide_{version}_amd64.deb'
digest=hashlib.sha512()
with package.open('rb') as archive:
for block in iter(lambda: archive.read(1024*1024), b''):
digest.update(block)
assert base64.b64encode(digest.digest()).decode()==os.environ['ORCA_PINNED_SHA512']
extracted=root/'extracted'
subprocess.run(['dpkg-deb','-x',str(package),str(extracted)],check=True)
executable=extracted/'opt'/'Orca'/'orca-ide'
assert executable.is_file() and os.access(executable,os.X_OK)
with open(os.environ['GITHUB_ENV'],'a') as env: env.write('ORCA_CROSS_VERSION_PACKAGED_EXECUTABLE='+str(executable)+'\n')
print('Verified old package:',executable)
PYVERIFY
@@ -0,0 +1,22 @@
name: Install mobile dependencies
description: Frozen pnpm install for the mobile/ project, whose node_modules the mobile web bundle build and the mobile-aware lint passes resolve React Native and Expo from.
runs:
using: composite
steps:
# Why a separate install: mobile is its own pnpm project, so the root install leaves
# mobile/node_modules empty and every mobile import resolves to nothing.
# Why no --ignore-scripts, unlike the root install: mobile's postinstall generates the
# gitignored terminal/mermaid webview engine modules that tracked source imports.
# The drift guard mirrors the root install so a stale mobile lockfile fails by name --
# mobile's lockfile carries patchedDependencies that a silent rewrite would drop.
- name: Install mobile dependencies
shell: bash
working-directory: mobile
run: |
pnpm install --frozen-lockfile
# Job containers can run composite steps from a source mirror without .git.
if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
git -C "$GITHUB_WORKSPACE" diff --exit-code -- \
mobile/package.json mobile/pnpm-lock.yaml mobile/pnpm-workspace.yaml
fi
@@ -1,7 +1,19 @@
name: Install Node dependencies
description: Installs the Node toolchain and repository dependencies for Linux CI jobs.
description: Installs the Node toolchain and repository dependencies for CI jobs, with optional Electron archive caching.
inputs:
cache-pnpm-store:
description: Restore or save the pnpm download store; verification and native caches are independent.
required: false
default: 'true'
cache-pnpm-store-lookup-only:
description: Auto uses measured hosted Node 24 root installs; true forces lookup, false retains archive restoration.
required: false
default: auto
cache-pnpm-verification:
description: Restore pnpm's policy-checked lockfile verification record.
required: false
default: 'true'
native-runtime:
description: Native runtime to prepare after the script-free install (none, node, or electron).
required: false
@@ -10,23 +22,91 @@ inputs:
description: Node.js version override; defaults to the version declared in package.json.
required: false
default: ''
cache-dependency-path:
description: Lockfiles for the pnpm download store; include mobile/pnpm-lock.yaml only when the job installs mobile dependencies.
required: false
default: pnpm-lock.yaml
persist-native-cache:
description: Save restored native modules at job end. Set false when a later step overwrites the same path with a different ABI.
required: false
default: 'true'
cache-electron-package:
description: Cache the Electron package archive and export ELECTRON_CACHE for following steps.
required: false
default: 'false'
outputs:
pnpm-store-cache-hit:
description: Whether the requested download store matched an existing cache.
value: ${{ steps.pnpm-store-lookup.outputs.cache-hit || steps.pnpm-store-restore.outputs.cache-hit || steps.requested-node.outputs.cache-hit || steps.default-node.outputs.cache-hit }}
verification-cache-hit:
description: Whether pnpm's verification record was restored.
value: ${{ steps.verification-cache.outputs.cache-hit }}
verification-cache-path:
description: The small pnpm-owned verification record, without registry metadata.
value: ${{ steps.verification-cache.outputs.path }}
verification-cache-key:
description: Exact verification record key, also used by the isolated PR benchmark.
value: ${{ steps.verification-cache.outputs.key }}
node-version:
description: Resolved Node.js version used for the install.
value: ${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}
native-cache-scope:
description: Operating-system image scope used by the native module cache.
value: ${{ steps.native-runtime.outputs.cache-scope }}
native-cache-key:
description: Exact native cache key requested after installation.
value: ${{ steps.native-runtime.outputs.cache-key }}
native-cache-path:
description: Native build directories covered by the requested cache.
value: ${{ steps.native-runtime.outputs.cache-path }}
native-cache-hit:
description: Whether the compiled native module cache was restored.
value: ${{ steps.native-runtime.outputs.cache-hit }}
runs:
using: composite
steps:
- name: Resolve pnpm store mode
id: pnpm-store-mode
if: >-
github.event_name != 'pull_request' && inputs.cache-pnpm-store != 'false' &&
(inputs.cache-pnpm-store-lookup-only == 'true' ||
(inputs.cache-pnpm-store-lookup-only == 'auto' &&
inputs.cache-dependency-path == 'pnpm-lock.yaml' &&
runner.environment == 'github-hosted' && job.container.id == '' &&
(runner.os == 'Linux' || runner.os == 'macOS' || runner.os == 'Windows') &&
(runner.arch == 'X64' || runner.arch == 'ARM64') &&
(inputs.node-version == '' || inputs.node-version == '24')))
shell: bash
env:
LOOKUP_REQUEST: ${{ inputs.cache-pnpm-store-lookup-only }}
run: |
lookup_only=true
case "$LOOKUP_REQUEST" in
[aA][uU][tT][oO])
# Hosted runners have Node for this manifest-only check before toolchain setup.
lookup_only="$(node -p 'const p = require("./package.json"); p.engines?.node === "24" && typeof p.packageManager === "string" && p.packageManager.split("+")[0] === "pnpm@12.8.1"')"
;;
esac
printf 'lookup-only=%s\n' "$lookup_only" >> "$GITHUB_OUTPUT"
# setup-node needs pnpm on PATH to locate and restore its store.
- name: Setup pnpm
uses: pnpm/action-setup@v6
uses: pnpm/setup@v2
with:
run_install: false
install: false
# Desktop-only jobs should not miss their download cache when mobile dependencies change.
- name: Setup Node.js
id: default-node
if: inputs.node-version == ''
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
cache: ${{ github.event_name != 'pull_request' && inputs.cache-pnpm-store != 'false' && steps.pnpm-store-mode.outputs.lookup-only != 'true' && 'pnpm' || '' }}
cache-dependency-path: ${{ inputs.cache-dependency-path }}
package-manager-cache: false
- name: Setup requested Node.js
id: requested-node
@@ -34,7 +114,66 @@ runs:
uses: actions/setup-node@v6
with:
node-version: ${{ inputs.node-version }}
cache: pnpm
cache: ${{ github.event_name != 'pull_request' && inputs.cache-pnpm-store != 'false' && steps.pnpm-store-mode.outputs.lookup-only != 'true' && 'pnpm' || '' }}
cache-dependency-path: ${{ inputs.cache-dependency-path }}
package-manager-cache: false
# PR-local stores compete with reusable build caches for the repository quota.
- name: Resolve pnpm download store
id: pnpm-store
if: >-
github.event_name == 'pull_request' && inputs.cache-pnpm-store != 'false' &&
!((runner.os == 'Linux' || runner.os == 'macOS') && (runner.arch == 'X64' || runner.arch == 'ARM64') && inputs.cache-dependency-path == 'pnpm-lock.yaml') &&
(runner.os != 'Windows' ||
!(runner.arch == 'X64' && contains(inputs.cache-dependency-path, 'mobile/pnpm-lock.yaml')) &&
!((runner.arch == 'X64' || runner.arch == 'ARM64') && inputs.cache-dependency-path == 'pnpm-lock.yaml')) ||
(github.event_name != 'pull_request' && inputs.cache-pnpm-store != 'false' &&
steps.pnpm-store-mode.outputs.lookup-only == 'true')
shell: bash
env:
LOCKFILE_HASH: ${{ hashFiles(inputs.cache-dependency-path) }}
STORE_LOOKUP_ONLY: ${{ steps.pnpm-store-mode.outputs.lookup-only == 'true' }}
run: |
test -n "$LOCKFILE_HASH"
cache_path="$(pnpm store path --silent)"
test -n "$cache_path"
printf 'path=%s\n' "$cache_path" >> "$GITHUB_OUTPUT"
printf 'arch=%s\n' "$(node -p 'require("node:os").arch()')" >> "$GITHUB_OUTPUT"
if [ "$STORE_LOOKUP_ONLY" = 'true' ]; then
printf 'ORCA_PNPM_STORE_CACHE_PATH=%s\n' "$cache_path" >> "$GITHUB_ENV"
fi
# Match setup-node's key and path so existing default-branch stores remain reusable.
# Direct downloads beat store restoration for the measured Linux, macOS and Windows installs.
- name: Restore pnpm download store without saving
id: pnpm-store-restore
if: >-
github.event_name == 'pull_request' && inputs.cache-pnpm-store != 'false' &&
!((runner.os == 'Linux' || runner.os == 'macOS') && (runner.arch == 'X64' || runner.arch == 'ARM64') && inputs.cache-dependency-path == 'pnpm-lock.yaml') &&
(runner.os != 'Windows' ||
!(runner.arch == 'X64' && contains(inputs.cache-dependency-path, 'mobile/pnpm-lock.yaml')) &&
!((runner.arch == 'X64' || runner.arch == 'ARM64') && inputs.cache-dependency-path == 'pnpm-lock.yaml'))
uses: actions/cache/restore@v5
with:
path: ${{ steps.pnpm-store.outputs.path }}
key: node-cache-${{ runner.os }}-${{ steps.pnpm-store.outputs.arch }}-pnpm-${{ hashFiles(inputs.cache-dependency-path) }}
# Producers can refresh access and publish misses without downloading existing archives.
- name: Keep pnpm download store without restoring
id: pnpm-store-lookup
if: steps.pnpm-store-mode.outputs.lookup-only == 'true'
uses: actions/cache@v5
with:
# Twice-nested composite cleanup loses internal step outputs.
path: ${{ env.ORCA_PNPM_STORE_CACHE_PATH }}
key: node-cache-${{ runner.os }}-${{ steps.pnpm-store.outputs.arch }}-pnpm-${{ hashFiles(inputs.cache-dependency-path) }}
lookup-only: true
- name: Restore pnpm verification record
id: verification-cache
uses: ./.github/actions/restore-pnpm-verification
with:
enabled: ${{ inputs.cache-pnpm-verification }}
- name: Validate native runtime
shell: bash
@@ -49,14 +188,6 @@ runs:
;;
esac
# pnpm's bundled gyp_main.py is not executable on fresh Linux runners.
- name: Use external node-gyp
if: inputs.native-runtime != 'none'
shell: bash
run: |
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- name: Prepare dependency install
shell: bash
run: |
@@ -73,22 +204,71 @@ runs:
shell: bash
run: |
pnpm install --frozen-lockfile --ignore-scripts
git diff --exit-code package.json pnpm-lock.yaml
# Job containers can run composite steps from a source mirror without .git.
if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
git -C "$GITHUB_WORKSPACE" diff --exit-code -- package.json pnpm-lock.yaml pnpm-workspace.yaml
fi
# Why cached: `--ignore-scripts` leaves node-pty without build/Release, so
# ensure-native-runtime node-gyp-compiles it in every job that asks for a runtime.
# The artifacts are ABI-bound, so the key carries the target runtime, the resolved
# Node version, and the patch whose contents the build has to match.
- name: Restore compiled native modules
if: inputs.native-runtime != 'none'
# Why: the script-free install leaves node_modules/.bin/bun as a stub that shadows setup-bun under `pnpm run`.
- name: Link pinned Bun
shell: bash
run: |
if [ -d node_modules/bun ]; then
pnpm rebuild bun
fi
# Why --if-present: release cuts run main's workflows against older release tags that predate this script.
- name: Build internal packages
shell: bash
run: pnpm run --if-present build:packages
# pnpm checks the cached record's policy and validity; never bypass verification.
- name: Save pnpm verification record on main
if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' && steps.verification-cache.outputs.key != '' && steps.verification-cache.outputs.cache-hit != 'true'
continue-on-error: true
uses: actions/cache/save@v5
with:
path: ${{ steps.verification-cache.outputs.path }}
key: ${{ steps.verification-cache.outputs.key }}
- name: Resolve Electron package cache
id: electron-package-cache
if: inputs.native-runtime == 'electron' || inputs.cache-electron-package == 'true'
shell: bash
run: |
set -euo pipefail
case "$RUNNER_OS" in
Linux) cache_root="$HOME/.cache/electron" ;;
macOS) cache_root="$HOME/Library/Caches/electron" ;;
Windows) cache_root="${LOCALAPPDATA:-$HOME/AppData/Local}/electron/Cache" ;;
*)
echo "::error::Unsupported runner OS for Electron cache: $RUNNER_OS"
exit 2
;;
esac
printf 'cache-root=%s\n' "$cache_root" >> "$GITHUB_OUTPUT"
printf 'ELECTRON_CACHE=%s\n' "$cache_root" >> "$GITHUB_ENV"
printf 'version=%s\n' "$(node -p "require('./node_modules/electron/package.json').version")" >> "$GITHUB_OUTPUT"
- name: Cache Electron package archive
if: (github.event_name != 'pull_request' || runner.os != 'Linux') && steps.electron-package-cache.outputs.version != ''
uses: actions/cache@v5
with:
path: node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
key: native-modules-${{ runner.os }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.default-node.outputs.node-version }}${{ steps.requested-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', 'config/patches/node-pty@1.1.0.patch') }}
path: ${{ steps.electron-package-cache.outputs.cache-root }}
key: electron-package-${{ runner.os }}-${{ runner.arch }}-${{ steps.electron-package-cache.outputs.version }}
- name: Restore Electron package archive without saving
if: github.event_name == 'pull_request' && runner.os == 'Linux' && steps.electron-package-cache.outputs.version != ''
uses: actions/cache/restore@v5
with:
path: ${{ steps.electron-package-cache.outputs.cache-root }}
key: electron-package-${{ runner.os }}-${{ runner.arch }}-${{ steps.electron-package-cache.outputs.version }}
- name: Prepare native runtime
id: native-runtime
if: inputs.native-runtime != 'none'
shell: bash
env:
NATIVE_RUNTIME: ${{ inputs.native-runtime }}
run: node config/scripts/ensure-native-runtime.mjs --runtime="$NATIVE_RUNTIME"
uses: ./.github/actions/prepare-native-runtime
with:
native-runtime: ${{ inputs.native-runtime }}
node-version: ${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}
persist-native-cache: ${{ inputs.persist-native-cache }}
@@ -0,0 +1,35 @@
name: Prepare baseline Git compatibility binary
description: Restore or build the pinned Linux Git binary for the compatibility contract.
runs:
using: composite
steps:
# The default-branch warmer shares this key across PRs; a PR save is private to its merge ref.
# Cache eviction remains possible, so keep the checksum-verified cold build below.
- name: Cache baseline Git build
uses: actions/cache@v5
with:
path: ~/.cache/orca-git-compat/git-2.25.5
key: git-compat-baseline-${{ runner.os }}-${{ runner.arch }}-2.25.5-submodule
# Finish the CPU-heavy build before any timed compatibility lanes start.
- name: Build the baseline Git binary
shell: bash
run: |
archive="$RUNNER_TEMP/git-2.25.5.tar.gz"
source="$HOME/.cache/orca-git-compat/git-2.25.5"
if [ -x "$source/git" ] && [ -x "$source/git-submodule" ] \
&& [ -f "$source/git-sh-setup" ] && [ -f "$source/git-sh-i18n" ] \
&& [ -f "$source/git-parse-remote" ] && [ -x "$source/git-sh-i18n--envsubst" ]; then
exit 0
fi
curl -fsSL https://www.kernel.org/pub/software/scm/git/git-2.25.5.tar.gz -o "$archive"
echo "41662c52fc16fec4963bfc41075e71f8ead6b5e386797eb6f9a1111ff95a8ddf $archive" \
| sha256sum --check
mkdir -p "$source"
tar -xzf "$archive" -C "$source" --strip-components=1
make -C "$source" -j"$(nproc)" \
NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_PYTHON=YesPlease \
git git-submodule git-sh-setup git-sh-i18n git-parse-remote git-sh-i18n--envsubst
# Object files are no longer needed after linking the cached binary.
find "$source" -name '*.o' -delete
@@ -0,0 +1,45 @@
name: Prepare headless detector compiler
description: Reuse the policy-checked compiler from main; callers install normally on a miss.
inputs:
seed:
description: Pack an already installed compiler instead of activating a cached compiler.
default: 'false'
outputs:
available:
description: Whether the cached compiler was validated and activated.
value: ${{ steps.activate.outputs.available }}
runs:
using: composite
steps:
- id: identity
shell: bash
env:
COMPILER_POLICY_HASH: ${{ hashFiles('package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', '.pnpmfile.cjs', 'config/patches/**', '.github/actions/install-node-dependencies/**', '.github/actions/restore-pnpm-verification/**', 'config/scripts/headless-detector-compiler-cache.mjs', '.github/actions/prepare-headless-compiler/action.yml') }}
run: node config/scripts/headless-detector-compiler-cache.mjs identity
- id: cache
uses: actions/cache/restore@v5
continue-on-error: true
with:
path: ${{ steps.identity.outputs.path }}
key: ${{ steps.identity.outputs.key }}
- id: activate
if: inputs.seed != 'true' && steps.cache.outputs.cache-hit == 'true'
shell: bash
env:
COMPILER_CACHE_KEY: ${{ steps.identity.outputs.key }}
COMPILER_CACHE_PATH: ${{ steps.identity.outputs.path }}
run: node config/scripts/headless-detector-compiler-cache.mjs activate
- name: Pack installed compiler
if: inputs.seed == 'true' && steps.cache.outputs.cache-hit != 'true'
shell: bash
env:
COMPILER_CACHE_KEY: ${{ steps.identity.outputs.key }}
COMPILER_CACHE_PATH: ${{ steps.identity.outputs.path }}
run: node config/scripts/headless-detector-compiler-cache.mjs pack
- name: Save compiler only from main
if: inputs.seed == 'true' && steps.cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
uses: actions/cache/save@v5
continue-on-error: true
with:
path: ${{ steps.identity.outputs.path }}
key: ${{ steps.identity.outputs.key }}
@@ -0,0 +1,63 @@
name: Prepare cached Linux package fixture
description: Restore Docker build layers; only the main warmer builds and saves missing images.
inputs:
fixture:
description: Directory name below config/docker.
required: true
save-cache:
description: Build and save a missing fixture on the default branch.
default: 'false'
outputs:
image:
description: Loaded image to use as a Docker build cache, or empty on a cache miss.
value: ${{ steps.image.outputs.image }}
runs:
using: composite
steps:
- name: Restore fixture image
id: cache
uses: actions/cache/restore@v5
continue-on-error: true
with:
path: ${{ runner.temp }}/orca-package-fixtures/${{ inputs.fixture }}.tar
key: linux-package-fixture-v1-${{ runner.os }}-${{ runner.arch }}-linux-amd64-${{ inputs.fixture }}-${{ hashFiles(format('config/docker/{0}/**', inputs.fixture), '.github/actions/prepare-linux-package-fixture/action.yml') }}
- name: Load or seed fixture build cache
id: image
shell: bash
env:
FIXTURE: ${{ inputs.fixture }}
SAVE_CACHE: ${{ inputs.save-cache }}
CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
run: |
set -euo pipefail
case "$FIXTURE" in
headless-serve-shutdown|cli-launch-contract|daemon-shutdown-descendants) ;;
*) echo "Unsupported package fixture: $FIXTURE" >&2; exit 1 ;;
esac
archive="$RUNNER_TEMP/orca-package-fixtures/$FIXTURE.tar"
image="orca-package-fixture-$FIXTURE:cache"
if [[ "$CACHE_HIT" == true ]] && docker load --input "$archive"; then
if docker image inspect "$image" > /dev/null; then
echo "image=$image" >> "$GITHUB_OUTPUT"
exit 0
fi
fi
if [[ "$SAVE_CACHE" != true ]]; then
echo 'No usable fixture cache; the package runner will build normally.'
exit 0
fi
docker build --platform linux/amd64 --build-arg BUILDKIT_INLINE_CACHE=1 \
--tag "$image" --file "config/docker/$FIXTURE/Dockerfile" "config/docker/$FIXTURE"
mkdir -p "$(dirname "$archive")"
docker save --output "$archive" "$image"
echo "image=$image" >> "$GITHUB_OUTPUT"
echo 'built=true' >> "$GITHUB_OUTPUT"
- name: Save fixture image
if: inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && steps.image.outputs.built == 'true'
uses: actions/cache/save@v5
continue-on-error: true
with:
path: ${{ runner.temp }}/orca-package-fixtures/${{ inputs.fixture }}.tar
key: ${{ steps.cache.outputs.cache-primary-key }}
@@ -0,0 +1,128 @@
name: Prepare native runtime
description: Restore and probe source-built native modules for the requested runtime.
inputs:
native-runtime:
description: Native runtime to prepare (none, node, or electron).
required: false
default: none
node-version:
description: Resolved Node.js version provided by setup-node.
required: true
persist-native-cache:
description: Save native modules at job end unless a later step changes their ABI.
required: false
default: 'true'
outputs:
cache-scope:
description: Operating-system image scope used by the native module cache.
value: ${{ steps.native-cache-scope.outputs.scope }}
cache-key:
description: Exact key requested for this runtime, independent of whether restoration succeeds.
value: ${{ steps.native-cache-scope.outputs.key }}
cache-path:
description: Native build directories covered by the requested cache.
value: ${{ steps.native-cache-scope.outputs.path }}
cache-hit:
description: Whether compiled native modules were restored under the exact requested key.
value: ${{ steps.native-cache-restore.outputs.cache-hit || steps.native-cache-restore-only.outputs.cache-hit }}
runs:
using: composite
steps:
- name: Validate native runtime
shell: bash
env:
NATIVE_RUNTIME: ${{ inputs.native-runtime }}
NODE_VERSION: ${{ inputs.node-version }}
run: |
case "$NATIVE_RUNTIME" in
none) ;;
node|electron) test -n "$NODE_VERSION" ;;
*)
echo "::error::native-runtime must be none, node, or electron"
exit 2
;;
esac
# Exact keys separate runtime, architecture and host libc; every restored addon is probed.
- name: Resolve native cache scope
id: native-cache-scope
if: inputs.native-runtime != 'none'
shell: bash
env:
NATIVE_RUNTIME: ${{ inputs.native-runtime }}
NODE_VERSION: ${{ inputs.node-version }}
NATIVE_SOURCE_HASH: ${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', '.pnpmfile.cjs', '.github/actions/prepare-native-runtime/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/initialize-windows-native-toolchain.mjs', 'config/scripts/capture-windows-native-environment.cmd', 'config/scripts/rebuild-native-deps.mjs', 'config/scripts/node-pty-job-ownership.cjs', 'config/scripts/windows-pe-machine.cjs', 'config/scripts/windows-process-tree-gyp-rebuild.mjs', 'config/scripts/windows-process-tree-creation-time.cjs', 'config/scripts/install-electron-package-binary.mjs', 'config/scripts/electron-platform-path.mjs', 'config/scripts/zip-extractor-command.mjs', 'src/shared/zip-extractor-command.ts', 'config/scripts/shared-electron-dist-cache.mjs', 'config/scripts/space-sharing-copy.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json', 'native/windows-registry/index.js', 'native/windows-security/src/addon.cc', 'native/windows-security/binding.gyp', 'native/windows-security/package.json', 'native/windows-security/index.js') }}
run: |
scope="${ImageOS:-$RUNNER_OS}"
if [ -r /etc/os-release ]; then
. /etc/os-release
scope="${ID:-linux}-${VERSION_ID:-unknown}"
fi
pnpm_version="$(pnpm --version)"
test -n "$pnpm_version"
printf 'scope=%s\n' "$scope" >> "$GITHUB_OUTPUT"
printf 'key=native-modules-%s-%s-%s-%s-node%s-pnpm%s-%s\n' "$RUNNER_OS" "$scope" "$RUNNER_ARCH" "$NATIVE_RUNTIME" "$NODE_VERSION" "$pnpm_version" "$NATIVE_SOURCE_HASH" >> "$GITHUB_OUTPUT"
cat >> "$GITHUB_OUTPUT" <<'ORCA_NATIVE_CACHE_PATHS'
path<<ORCA_NATIVE_BUILD_DIRECTORIES
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
native/windows-registry/build
native/windows-security/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
ORCA_NATIVE_BUILD_DIRECTORIES
ORCA_NATIVE_CACHE_PATHS
- name: Restore compiled native modules
id: native-cache-restore
if: inputs.native-runtime != 'none' && inputs.persist-native-cache != 'false'
uses: actions/cache@v5
with:
# Post-job saving cannot resolve nested composite step outputs.
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
native/windows-registry/build
native/windows-security/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: ${{ steps.native-cache-scope.outputs.key }}
- name: Restore compiled native modules without saving
id: native-cache-restore-only
if: inputs.native-runtime != 'none' && inputs.persist-native-cache == 'false'
uses: actions/cache/restore@v5
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
native/windows-registry/build
native/windows-security/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: ${{ steps.native-cache-scope.outputs.key }}
# pnpm's bundled gyp_main.py is not executable; Electron rebuild uses its own node-gyp API.
- name: Use external node-gyp
if: runner.os == 'Linux' && inputs.native-runtime == 'node'
shell: bash
env:
NATIVE_RUNTIME: ${{ inputs.native-runtime }}
NATIVE_CACHE_HIT: ${{ steps.native-cache-restore.outputs.cache-hit || steps.native-cache-restore-only.outputs.cache-hit }}
run: |
# A cache hit can contain unusable addons; probe before skipping the rebuild toolchain.
if [ "$NATIVE_RUNTIME" = node ] && [ "$NATIVE_CACHE_HIT" = true ] &&
node config/scripts/ensure-native-runtime.mjs --check-only; then
exit 0
fi
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- name: Prepare native runtime
if: inputs.native-runtime != 'none'
shell: bash
env:
NATIVE_RUNTIME: ${{ inputs.native-runtime }}
run: |
if [ "$RUNNER_OS" = Windows ] && [ "$NATIVE_RUNTIME" = node ]; then
node config/scripts/initialize-windows-native-toolchain.mjs
else
node config/scripts/ensure-native-runtime.mjs --runtime="$NATIVE_RUNTIME"
fi
@@ -0,0 +1,67 @@
name: Prepare server native prebuilds
description: Builds and smokes the current server slot, optionally restoring a validated Windows slot.
inputs:
resolve-windows-cache:
description: Resolve the exact Windows cache identity for restoration or later qualified publication.
default: 'false'
restore-windows-cache:
description: Restore the exact Windows slot when its identity is available.
default: 'false'
outputs:
cache-key:
description: Exact Windows server cache key.
value: ${{ steps.orcad-prebuild-cache-identity.outputs.key }}
cache-path:
description: Directory containing the Windows slot and its manifest.
value: ${{ steps.orcad-prebuild-cache-identity.outputs.path }}
cache-identity-outcome:
description: Whether Windows cache identity resolution succeeded.
value: ${{ steps.orcad-prebuild-cache-identity.outcome }}
cache-hit:
description: Whether the exact Windows slot was restored.
value: ${{ steps.orcad-prebuild-cache-restore.outputs.cache-hit }}
runs:
using: composite
steps:
- name: Resolve this Windows server prebuild cache
id: orcad-prebuild-cache-identity
if: >-
inputs.resolve-windows-cache == 'true' && runner.os == 'Windows' &&
(runner.arch == 'X64' || runner.arch == 'ARM64')
continue-on-error: true
shell: bash
run: node config/scripts/orcad-windows-prebuild-cache.mjs --fingerprint
- name: Restore the exact Windows server prebuild
id: orcad-prebuild-cache-restore
if: >-
inputs.restore-windows-cache == 'true' &&
steps.orcad-prebuild-cache-identity.outcome == 'success' &&
steps.orcad-prebuild-cache-identity.outputs.key != ''
continue-on-error: true
uses: actions/cache/restore@v5
with:
path: ${{ steps.orcad-prebuild-cache-identity.outputs.path }}
key: ${{ steps.orcad-prebuild-cache-identity.outputs.key }}
- name: Build and smoke this runner's node-pty prebuild slot under the pinned Node
shell: bash
env:
WINDOWS_PREBUILD_CACHE_HIT: ${{ steps.orcad-prebuild-cache-restore.outputs.cache-hit }}
WINDOWS_PREBUILD_CACHE_RESTORE_OUTCOME: ${{ steps.orcad-prebuild-cache-restore.outcome }}
run: |
if [ "$RUNNER_OS" = Windows ] && [ "$WINDOWS_PREBUILD_CACHE_HIT" = true ] &&
[ "$WINDOWS_PREBUILD_CACHE_RESTORE_OUTCOME" = success ] &&
node config/scripts/orcad-windows-prebuild-cache.mjs --validate; then
echo 'Using the validated Windows server prebuild'
else
if [ "$RUNNER_OS" = Windows ]; then
rm -rf -- out/orcad-prebuilds
fi
pnpm build:orcad-prebuilds
fi
node config/scripts/build-orcad-prebuilds.mjs --require-slots "$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)"
node config/scripts/build-orcad-prebuilds.mjs --smoke
@@ -0,0 +1,53 @@
name: Prepare Wayland clipboard helper
description: Reuse the source-matched Ubuntu 20.04 helper, validating its architecture and glibc requirements every time.
inputs:
save-cache:
description: Publish a validated helper from the default branch.
default: 'false'
runs:
using: composite
steps:
- name: Resolve Wayland clipboard build identity
id: identity
shell: bash
run: |
node --input-type=module <<'NODE'
import { appendFileSync } from 'node:fs'
import { waylandClipboardFingerprint } from './config/scripts/build-wayland-clipboard.mjs'
const fingerprint = waylandClipboardFingerprint(process.cwd(), process.arch)
// Immutable invalid hits need a version bump here before publishing their repair.
appendFileSync(process.env.GITHUB_OUTPUT, `key=wayland-clipboard-v1-${process.arch}-${fingerprint}\n`)
NODE
- name: Restore Wayland clipboard helper
id: cache
uses: actions/cache/restore@v5
continue-on-error: true
with:
path: native/wayland-clipboard/.build
key: ${{ steps.identity.outputs.key }}
- name: Build or validate Wayland clipboard helper
shell: bash
env:
CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
run: |
if node config/scripts/build-wayland-clipboard.mjs; then
exit 0
fi
if [ "$CACHE_HIT" != true ]; then
exit 1
fi
echo 'Discarding an unusable Wayland clipboard cache and rebuilding from source.'
rm -rf -- native/wayland-clipboard/.build
node config/scripts/build-wayland-clipboard.mjs
- name: Save validated Wayland clipboard helper from main
if: inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
uses: actions/cache/save@v5
continue-on-error: true
with:
path: native/wayland-clipboard/.build
key: ${{ steps.identity.outputs.key }}
@@ -0,0 +1,27 @@
name: Prepare the Windows agent hook
description: Compiles native/windows-agent-hook into native/windows-agent-hook/.build/orca-agent-hook.exe, which electron-builder --win refuses to package without.
runs:
using: composite
steps:
- name: Cache the Windows agent hook build
if: runner.os == 'Windows'
uses: actions/cache@v5
with:
path: |
native/windows-agent-hook/.build
native/windows-agent-hook/target
~/.cargo/registry
key: windows-agent-hook-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('native/windows-agent-hook/**', 'config/scripts/build-windows-agent-hook.mjs', 'config/scripts/windows-rust-executable-build.mjs', 'resources/build/icon.ico', 'package.json') }}
# Why explicit: a runner image without cargo would otherwise fail inside electron-builder's verify.
- name: Build the Windows agent hook
if: runner.os == 'Windows'
shell: pwsh
run: |
if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
rustup toolchain install stable --profile minimal
rustup default stable
}
node config/scripts/build-windows-agent-hook.mjs
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
@@ -0,0 +1,39 @@
name: Prepare the orcad Windows CLI launcher
description: Compiles native/windows-cli-launcher for each requested arch into .build/windows-cli-launcher/<arch>/orca.exe.
inputs:
arches:
description: Space-separated Windows architectures (x64, arm64).
required: true
runs:
using: composite
steps:
- name: Cache the Windows CLI launcher builds
if: runner.os == 'Windows'
uses: actions/cache@v5
with:
path: |
.build/windows-cli-launcher
native/windows-cli-launcher/target
~/.cargo/registry
key: orcad-windows-cli-launcher-${{ runner.arch }}-${{ inputs.arches }}-${{ hashFiles('native/windows-cli-launcher/**', 'config/scripts/build-windows-cli-launcher.mjs', 'config/scripts/windows-rust-executable-build.mjs', 'resources/build/icon.ico', 'package.json') }}
# Why explicit: a runner image without cargo would otherwise fail inside the orcad build.
- name: Build the launcher for each architecture
if: runner.os == 'Windows'
shell: pwsh
env:
ARCHES: ${{ inputs.arches }}
run: |
if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
rustup toolchain install stable --profile minimal
rustup default stable
}
$triples = @{ x64 = 'x86_64-pc-windows-msvc'; arm64 = 'aarch64-pc-windows-msvc' }
foreach ($arch in ($env:ARCHES -split ' ' | Where-Object { $_ })) {
rustup target add $triples[$arch]
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
node config/scripts/build-windows-cli-launcher.mjs --arch $arch --output ".build/windows-cli-launcher/$arch/orca.exe"
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
}
@@ -0,0 +1,33 @@
name: Restore xterm build dependencies
description: Reuse installed dependencies for the pinned upstream checkout.
outputs:
cache-key:
description: Exact dependency cache key.
value: ${{ steps.restore.outputs.cache-primary-key }}
cache-hit:
description: Whether the exact installed dependency cache was restored.
value: ${{ steps.restore.outputs.cache-hit }}
runs:
using: composite
steps:
- id: runtime
shell: bash
run: |
. /etc/os-release
echo "image=$ID-$VERSION_ID" >> "$GITHUB_OUTPUT"
echo "node=$(node --version)" >> "$GITHUB_OUTPUT"
- id: restore
uses: actions/cache/restore@v5
with:
path: |
${{ runner.temp }}/xterm-patch-build/upstream/.git
${{ runner.temp }}/xterm-patch-build/upstream/node_modules
key: xterm-dependencies-v1-${{ runner.os }}-${{ steps.runtime.outputs.image }}-${{ runner.arch }}-${{ steps.runtime.outputs.node }}-${{ hashFiles('config/patches/xterm-upstream.json', 'config/scripts/regenerate-xterm-patches.mjs', 'config/scripts/xterm-patch-text.mjs', '.github/actions/prepare-xterm-dependencies/action.yml') }}
- name: Restore download cache on an installed-dependency miss
if: steps.restore.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
path: |
~/.npm
${{ runner.temp }}/xterm-patch-build/upstream/.git
key: xterm-upstream-${{ hashFiles('config/patches/xterm-upstream.json') }}
@@ -0,0 +1,58 @@
name: Restore pnpm verification
description: Restore the pnpm-owned lockfile verdict without installing dependencies.
inputs:
enabled:
description: Restore a verification record on supported runners.
default: 'true'
container-toolchain:
description: Use the manifest's pnpm version and default Linux cache path without installing host pnpm.
default: 'false'
outputs:
path:
description: The pnpm-owned verification record.
value: ${{ steps.verification-cache.outputs.path }}
key:
description: Exact OS, architecture, pnpm version and policy key.
value: ${{ steps.verification-cache.outputs.key }}
cache-hit:
description: Whether the exact record was restored.
value: ${{ steps.verification-cache-restore.outputs.cache-hit }}
runs:
using: composite
steps:
- name: Resolve pnpm verification cache
id: verification-cache
if: >-
inputs.enabled == 'true' &&
(runner.os == 'Linux' ||
(runner.os == 'Windows' && (runner.arch == 'X64' || runner.arch == 'ARM64')) ||
(runner.os == 'macOS' && runner.arch == 'X64'))
shell: bash
env:
POLICY_HASH: ${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc') }}
CONTAINER_TOOLCHAIN: ${{ inputs.container-toolchain }}
run: |
if [ "$CONTAINER_TOOLCHAIN" = true ]; then
test "$RUNNER_OS" = Linux
# Cache versions include paths, so match the native Linux producer.
cache_path="$(node -p "require('node:path').join(process.env.XDG_CACHE_HOME || require('node:path').join(require('node:os').homedir(), '.cache'), 'pnpm')")"
pnpm_version="$(node -p "require('./package.json').packageManager.match(/^pnpm@([^+]+)/)[1]")"
mkdir -p "$cache_path"
else
cache_path="$(pnpm cache path)"
pnpm_version="$(pnpm --version)"
fi
printf 'path=%s/lockfile-verified.jsonl\n' "$cache_path" >> "$GITHUB_OUTPUT"
printf 'key=pnpm-verification-v1-%s-%s-%s-%s\n' "$RUNNER_OS" "$RUNNER_ARCH" "$pnpm_version" "$POLICY_HASH" >> "$GITHUB_OUTPUT"
- name: Restore pnpm verification record
id: verification-cache-restore
if: steps.verification-cache.outputs.key != ''
continue-on-error: true
uses: actions/cache/restore@v5
with:
path: ${{ steps.verification-cache.outputs.path }}
key: ${{ steps.verification-cache.outputs.key }}
@@ -0,0 +1,8 @@
name: Set up WSL test runtime
description: Install a checksum-pinned Ubuntu WSL1 guest with executable Node and Git for real terminal tests.
runs:
using: composite
steps:
- name: Provision Ubuntu WSL1
shell: pwsh
run: '& "${{ github.action_path }}/setup.ps1"'
@@ -0,0 +1,32 @@
$ErrorActionPreference = 'Stop'
if (-not $IsWindows) { throw 'WSL test provisioning requires a Windows runner' }
$rootfs = Join-Path $env:RUNNER_TEMP 'noble-rootfs.tar.gz'
Invoke-WebRequest 'https://releases.ubuntu.com/24.04.4/ubuntu-24.04.4-wsl-amd64.wsl' -OutFile $rootfs
if ((Get-FileHash $rootfs -Algorithm SHA256).Hash.ToLowerInvariant() -ne '9b2f7730dc68227dd04a9f3e5eab86ad85caf556b8606ad94f1f29ff5c4fd3f5') { throw 'Ubuntu rootfs checksum mismatch' }
$distroDir = Join-Path $env:RUNNER_TEMP 'orca-wsl-ubuntu'
wsl.exe --import Ubuntu $distroDir $rootfs --version 1
if ($LASTEXITCODE -ne 0) { throw "WSL import failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/true
if ($LASTEXITCODE -ne 0) { throw "WSL guest did not start: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/apt-get update
if ($LASTEXITCODE -ne 0) { throw "WSL apt update failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/apt-get install --yes git curl xz-utils
if ($LASTEXITCODE -ne 0) { throw "WSL git install failed: $LASTEXITCODE" }
$kernelMsi = Join-Path $env:RUNNER_TEMP 'wsl_update_x64.msi'
Invoke-WebRequest 'https://wslstorestorage.blob.core.windows.net/wslblob/wsl_update_x64.msi' -OutFile $kernelMsi
if ((Get-FileHash $kernelMsi -Algorithm SHA256).Hash.ToLowerInvariant() -ne '4d09c776c8d45f70a202281d18e19be1118f53159b0c217a5274a31ce18525fe') { throw 'WSL kernel installer checksum mismatch' }
$installer = Start-Process msiexec.exe -ArgumentList @('/i', $kernelMsi, '/quiet', '/norestart') -Wait -PassThru
if ($installer.ExitCode -ne 0) { throw "WSL kernel installation failed: $($installer.ExitCode)" }
wsl.exe --status
if ($LASTEXITCODE -ne 0) { throw "WSL status failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/curl --fail --silent --show-error --location https://nodejs.org/dist/v22.14.0/node-v22.14.0-linux-x64.tar.xz --output /tmp/orca-node.tar.xz
if ($LASTEXITCODE -ne 0) { throw 'Node download failed' }
$nodeHash = wsl.exe --distribution Ubuntu --user root --exec /usr/bin/sha256sum /tmp/orca-node.tar.xz
if ($LASTEXITCODE -ne 0 -or -not ($nodeHash -match '^69b09dba5c8dcb05c4e4273a4340db1005abeafe3927efda2bc5b249e80437ec')) { throw 'Node checksum mismatch' }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/tar -xJf /tmp/orca-node.tar.xz -C /usr/local --strip-components=1
if ($LASTEXITCODE -ne 0) { throw 'Node extraction failed' }
wsl.exe --distribution Ubuntu --user root --exec /usr/local/bin/node --version
if ($LASTEXITCODE -ne 0) { throw 'Node cannot execute in WSL' }
wsl.exe --list --verbose
if ($LASTEXITCODE -ne 0) { throw "WSL enumeration failed: $LASTEXITCODE" }
+11
View File
@@ -0,0 +1,11 @@
version: 2
updates:
# Why only ripgrep: the bundled rg ships in every artifact and to SSH remotes, and a bump is a
# one-line pin change (the SSH cache keys on the binary's hash). Other dependencies stay manual.
- package-ecosystem: npm
directory: /
schedule:
interval: monthly
allow:
- dependency-name: '@vscode/ripgrep-universal'
open-pull-requests-limit: 1
+8 -6
View File
@@ -1,18 +1,20 @@
## ELI5
<!-- Simple high-level explanation -->
<!-- Simple high-level explanation, in plain language. No jargon. -->
## What Changed
<!-- Describe the change clearly and keep scope tight. -->
<!-- Describe the change clearly and keep scope tight. Cover the before and after as the user experiences it, and the mechanism you changed — not just the symptom. -->
## Why
<!-- What problem does this solve, and why is this approach right? -->
<!-- What problem does this solve, and why is this approach better than the alternatives you considered? -->
## Linked Issue
<!-- Link the issue this PR addresses, there should ALWAYS be one -->
_If you do not have one and are an outside contributors, your PR **wiil** be ignored. Refs is not sufficient. Link an actual issue_
<!-- Link the issue this PR addresses, there should ALWAYS be one (for outside contributors) -->
<!-- SPECIAL CASE: If you are a maintainer (member of stablyai org) AVOID opening needless issues. Only attach pre-existing ones -->
Fixes #
@@ -38,7 +40,7 @@ Fixes #
## Agent skill upstream boundary
- [ ] Not applicable, or this change follows `docs/reference/agent-skill-sharing-upstream-boundary.md` and copies or mechanically translates no upstream skill-installer source, tests, fixtures, registry entries, path tables, comments, or documentation.
- [ ] Not applicable, or this change copies or mechanically translates no upstream skill-installer source, tests, fixtures, registry entries, path tables, comments, or documentation.
## Notes
@@ -47,7 +49,7 @@ Ensure no issues in: Security, Cross-platoform support (Linux, Windows, Mac), Re
## Checklist
- [ ] This PR is small and focused
- [ ] I explained what changed and why (including ELI5)
- [ ] I explained what changed and why (ELI5, the user-facing before/after, the mechanism, and why over the alternatives)
- [ ] Before/after screenshots or videos attached for UI changes, or `N/A` with reason
- [ ] Self-reviewed for correctness, security, and performance
- [ ] Cross-platform, SSH/remote, and path/shortcut impact considered (or N/A)
@@ -0,0 +1,36 @@
import { readdirSync, readFileSync } from 'node:fs'
import { join } from 'node:path'
import { pathToFileURL } from 'node:url'
export function verifyRequiredPrContext(parseYaml) {
const directory = '.github/workflows'
const providers = readdirSync(directory)
.filter((file) => /\.ya?ml$/.test(file))
.flatMap((file) => {
const workflow = parseYaml(readFileSync(join(directory, file), 'utf8'))
return Object.entries(workflow.jobs)
.filter(([id, job]) => (job.name ?? id) === 'PR checks passed')
.map(([id, job]) => ({ owner: `${file}:${id}`, job }))
})
const owners = providers.map((provider) => provider.owner)
if (providers.length !== 1 || owners[0] !== 'pr.yml:verify') {
throw new Error(`Required PR context must belong only to pr.yml:verify; found ${owners}`)
}
const gate = providers[0].job
const condition = gate.if?.replace(/^\$\{\{\s*|\s*\}\}$/g, '').trim()
if (condition !== '!cancelled() && github.event.pull_request.draft != true') {
throw new Error('Required PR aggregate must run unless cancelled or the PR is a draft.')
}
if (gate['continue-on-error'] || gate.steps?.some((step) => step['continue-on-error'])) {
throw new Error('Required PR aggregate must not ignore failures.')
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const yaml = globalThis.Bun?.YAML
if (!yaml) {
throw new Error('Run workflow context validation with the pinned Bun runtime.')
}
verifyRequiredPrContext(yaml.parse)
console.log('Required PR check context verified.')
}
@@ -13,6 +13,9 @@ function readRootEntries(sha) {
return stdout.split('\0').filter(Boolean)
}
// These reviewed additions stay listed until the base tree carries them.
const REVIEWED_ROOT_ENTRIES = new Set(['cloud', 'opencode.json'])
function checkRootDirectoryEntries(argv) {
if (argv.length !== 2) {
console.error(`Usage: ${process.argv[1]} <base-sha> <head-sha>`)
@@ -21,7 +24,9 @@ function checkRootDirectoryEntries(argv) {
const [baseSha, headSha] = argv
const baseEntries = new Set(readRootEntries(baseSha))
const blockedEntries = readRootEntries(headSha).filter((entry) => !baseEntries.has(entry))
const blockedEntries = readRootEntries(headSha).filter(
(entry) => !baseEntries.has(entry) && !REVIEWED_ROOT_ENTRIES.has(entry)
)
if (blockedEntries.length === 0) {
console.log('Root directory guard passed: no new root-level files or folders.')
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
openbox --sm-disable > /tmp/orca-e2e-window-manager.log 2>&1 &
wm_pid=$!
cleanup() {
kill "$wm_pid" 2>/dev/null || true
wait "$wm_pid" 2>/dev/null || true
}
trap cleanup EXIT
ready=false
for attempt in {1..100}; do
if xprop -root _NET_SUPPORTING_WM_CHECK 2>/dev/null | rg -q 'window id # 0x[1-9a-fA-F]'; then
ready=true
break
fi
if ! kill -0 "$wm_pid" 2>/dev/null; then
cat /tmp/orca-e2e-window-manager.log
exit 1
fi
sleep 0.1
done
if [ "$ready" != true ]; then
echo 'Window manager did not acquire the Xvfb root window' >&2
exit 1
fi
"$@"
@@ -1,5 +1,7 @@
import { mkdir, writeFile } from 'node:fs/promises'
import { dirname } from 'node:path'
import { pathToFileURL } from 'node:url'
import { isAgentStateRulesTag } from '../../config/scripts/release-tag-patterns.mjs'
const repository = process.env.GITHUB_REPOSITORY ?? 'stablyai/orca'
const token = process.env.GITHUB_TOKEN
@@ -24,6 +26,22 @@ async function fetchJson(url) {
return response.json()
}
// Why rules releases are excluded: every running app fetches them every few hours, so they
// count fetches, not installs.
export function countReleaseDownloads(releases) {
let total = 0
for (const release of releases) {
if (release.draft || isAgentStateRulesTag(release.tag_name ?? '')) {
continue
}
for (const asset of release.assets ?? []) {
total += asset.download_count ?? 0
}
}
return total
}
async function getTotalReleaseDownloads() {
let page = 1
let total = 0
@@ -37,16 +55,7 @@ async function getTotalReleaseDownloads() {
return total
}
for (const release of releases) {
if (release.draft) {
continue
}
for (const asset of release.assets ?? []) {
total += asset.download_count ?? 0
}
}
total += countReleaseDownloads(releases)
page += 1
}
}
@@ -104,9 +113,11 @@ function renderBadge(value) {
`
}
const total = await getTotalReleaseDownloads()
const badge = renderBadge(formatDownloads(total))
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
const total = await getTotalReleaseDownloads()
const badge = renderBadge(formatDownloads(total))
await mkdir(dirname(outputPath), { recursive: true })
await writeFile(outputPath, badge)
console.log(`Rendered ${outputPath} from ${total} downloads.`)
await mkdir(dirname(outputPath), { recursive: true })
await writeFile(outputPath, badge)
console.log(`Rendered ${outputPath} from ${total} downloads.`)
}
+198 -29
View File
@@ -1,16 +1,17 @@
name: Adhoc macOS Dev Build
name: Adhoc macOS + Windows Dev Build
# Why: lets anyone cut a signed macOS build of an unlanded branch so the team can
# actually run an experimental feature for a few days, instead of reasoning about
# it from a diff. Hourly covers main; this covers everything that is not main yet.
# Why: lets anyone cut installable macOS and Windows builds of a trusted repo ref
# so the team can run an experimental feature instead of reasoning from a diff.
# Hourly covers current main; this covers experiments and on-demand validation.
#
# Deliberately narrow scope, same trade as hourly:
# - macOS only. Other platforms keep using RC/stable.
# - No tests, no lint, no e2e. PR CI and release-cut remain the gates.
# - Signed AND notarized, exactly like a release. macOS anchors a notarized
# app's TCC grants on identifier + team rather than on its cdhash, so those
# grants survive an update; an unnotarized build reads as a new client and
# silently loses file access under Documents/Desktop/Downloads.
# Deliberately narrow scope:
# - macOS and Windows desktop installers. Linux keeps using RC/stable.
# - No tests, no lint, no e2e. PR CI and release-cut remain the gates. The one
# exception is the orcad template: like release-cut, it is merged from the
# node-server lanes that build and qualify each SSH target's slot.
# - macOS is signed and notarized so TCC grants survive updates.
# - Windows is unsigned; the published release notes explain the one-time
# SmartScreen/manual-install requirement.
#
# Artifacts publish to stablyai/orca-adhoc — separate from both orca and
# orca-hourly. Separate from orca because the main repo's releases atom feed
@@ -82,8 +83,112 @@ env:
ADHOC_RETAIN_DAYS: 30
jobs:
build-adhoc-mac:
# Why: every job below checks out its own copy, and a branch name read per job builds whatever
# the branch points at when that job starts. A push mid-run then mixes commits, e.g. slot lanes
# from one commit merged by a template step from the next. Resolving once pins the whole run.
# The mac job still vets this commit's reachability before anything is signed.
resolve-ref:
if: github.repository == 'stablyai/orca'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
outputs:
sha: ${{ steps.resolve.outputs.sha }}
steps:
- name: Resolve the requested ref to one commit
id: resolve
shell: bash
env:
REQUESTED_REF: ${{ inputs.ref || github.ref_name }}
REPO_URL: https://github.com/${{ github.repository }}
run: |
set -euo pipefail
case "$REQUESTED_REF" in
refs/pull/*|pull/*)
echo "::error::Refusing to build PR ref '$REQUESTED_REF'; push the code to a branch of stablyai/orca instead."
exit 1
;;
esac
if [[ "$REQUESTED_REF" =~ ^[0-9a-f]{40}$ ]]; then
sha="$REQUESTED_REF"
else
# Branch first, as the mac job's vet does; a peeled annotated tag names its commit.
refs="$(git ls-remote "$REPO_URL" "refs/heads/$REQUESTED_REF" "refs/tags/$REQUESTED_REF" "refs/tags/$REQUESTED_REF^{}")"
sha=""
for name in "refs/heads/$REQUESTED_REF" "refs/tags/$REQUESTED_REF^{}" "refs/tags/$REQUESTED_REF"; do
sha="$(awk -v ref="$name" '$2 == ref { print $1; exit }' <<<"$refs")"
[[ -n "$sha" ]] && break
done
fi
if [[ -z "$sha" ]]; then
echo "::error::'$REQUESTED_REF' is not a branch, tag, or full commit SHA of stablyai/orca."
exit 1
fi
echo "Resolved $REQUESTED_REF -> $sha"
echo "sha=$sha" >>"$GITHUB_OUTPUT"
# Why: a branch cut before the orcad template landed has none to build or ship.
orcad-template-support:
needs: resolve-ref
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
outputs:
ships: ${{ steps.detect.outputs.ships }}
steps:
- name: Checkout the template packager only
uses: actions/checkout@v6
with:
ref: ${{ needs.resolve-ref.outputs.sha }}
sparse-checkout: |
/config/scripts/packaged-orcad-template.cjs
/config/tsconfig.relay.json
sparse-checkout-cone-mode: false
persist-credentials: false
# Why: a relay-era ref (before #27215) needs the Windows relay addon this build no longer compiles.
- name: Refuse a ref that still ships the SSH relay
shell: bash
run: |
if [[ -e config/tsconfig.relay.json ]]; then
echo "::error::This ref predates #27215 and still ships the SSH relay; build a commit at or after 2b4b60a87e."
exit 1
fi
- name: Detect whether the ref ships the orcad template
id: detect
shell: bash
run: |
if [[ -f config/scripts/packaged-orcad-template.cjs ]]; then
echo "ships=true" >>"$GITHUB_OUTPUT"
else
echo "ships=false" >>"$GITHUB_OUTPUT"
echo "::notice::This ref predates the orcad template; the build ships without it."
fi
# Design D2, as release-cut does: every desktop build ships the orcad template (server JS plus
# every target's addons), merged from the node-server lanes that qualify each slot at this ref.
# Without it an adhoc build cannot deploy managed orcad to an SSH host. No secrets, and the mac
# job vets the ref before anything is signed.
orcad-template:
needs: [resolve-ref, orcad-template-support]
if: needs.orcad-template-support.outputs.ships == 'true'
permissions:
contents: read
uses: ./.github/workflows/node-server-tests.yml
with:
ref: ${{ needs.resolve-ref.outputs.sha }}
build_template: true
build-adhoc-mac:
needs: [resolve-ref, orcad-template-support, orcad-template]
# Why not the implicit success(): a ref without the orcad template skips that job on purpose.
if: >-
!cancelled() && github.repository == 'stablyai/orca' &&
needs.orcad-template-support.result == 'success' &&
(needs.orcad-template.result == 'success' ||
(needs.orcad-template.result == 'skipped' &&
needs.orcad-template-support.outputs.ships == 'false'))
# Why an environment: it gives the signing/notary/App secrets somewhere to
# live that a stale copy of this workflow on an old branch cannot reach.
# Referencing it is a no-op until repo settings give it teeth; the intended
@@ -97,6 +202,7 @@ jobs:
version: ${{ steps.adhoc.outputs.version }}
head_sha: ${{ steps.adhoc.outputs.head_sha }}
published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }}
ships_orcad_template: ${{ needs.orcad-template-support.outputs.ships }}
runs-on: blacksmith-6vcpu-macos-15
# Why 150: it must exceed the worst case the retry budgets below can produce
# (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or
@@ -116,7 +222,8 @@ jobs:
id: vetted
shell: bash
env:
REQUESTED_REF: ${{ inputs.ref || github.ref_name }}
# The commit resolve-ref pinned for every job; vetting it keeps the reachability test.
REQUESTED_REF: ${{ needs.resolve-ref.outputs.sha }}
REPO_URL: https://github.com/${{ github.repository }}
run: |
set -euo pipefail
@@ -128,9 +235,12 @@ jobs:
esac
# Bare: a work-tree repo refuses to fetch over its own checked-out
# branch. tree:0 keeps the fetch to the commit graph — no trees, no
# blobs — so this stays cheap next to the build it fronts.
# blobs — so this stays cheap next to the build it fronts. reftable
# because this repo has branches that differ only in casing, and the
# files backend cannot store both on a case-insensitive runner disk —
# it fails the entire fetch, not just the one ref.
scratch="$RUNNER_TEMP/vet-requested-ref"
git init -q --bare "$scratch"
git init -q --bare --ref-format=reftable "$scratch"
git -C "$scratch" fetch -q --filter=tree:0 "$REPO_URL" '+refs/heads/*:refs/heads/*' '+refs/tags/*:refs/tags/*'
# Branch first to keep actions/checkout's old tie-break: bare
# rev-parse would prefer the tag when a branch shares its name.
@@ -164,7 +274,8 @@ jobs:
# always comes from the dispatch ref — naming the branch here instead
# applies main's current copy of this file to an arbitrary branch.
ref: ${{ steps.vetted.outputs.sha }}
fetch-depth: 0
# Version helpers only read HEAD; published versions come from the release API.
fetch-depth: 1
# This job only reads stablyai/orca and never pushes; every write goes
# to the adhoc repo through a minted App token passed by env. Not
# persisting the checkout credential shrinks the blast radius if a build
@@ -172,15 +283,30 @@ jobs:
persist-credentials: false
- name: Setup pnpm
uses: pnpm/action-setup@v6
uses: pnpm/setup@v2
with:
run_install: false
install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why: packaging bundles the pinned Node into the macOS terminal host. Keyed on the pin alone
# with no restore-keys, so a pin bump starts empty instead of carrying old archives forward.
- name: Locate the pinned Node runtime cache
shell: bash
run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV"
- name: Cache the pinned Node runtime
uses: actions/cache@v5
with:
path: ~/Library/Caches/orca-node-runtime
key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }}
- name: Cache electron-builder downloads
uses: actions/cache@v5
@@ -192,13 +318,19 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: signing is what makes an adhoc build installable over an existing
# Orca, so a missing cert must fail here rather than after a 20-minute build.
@@ -230,11 +362,14 @@ jobs:
fi
done
echo "head_sha=$(git rev-parse HEAD)" >>"$GITHUB_OUTPUT"
# Why the main repo's tags: package.json on a branch is as stale as the
# main it forked from, and stable patches never merge back into it.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh release list \
--repo "$GITHUB_REPOSITORY" --limit 100 --exclude-drafts \
--json tagName --jq '.[].tagName' || true)"
# Why git tags, not GitHub releases: unpublishing a buggy cut deletes the
# GitHub release and leaves the tag, which still owns that number.
# Releases-only let adhoc sit on a number already taken, so the updater
# would not install it. Empty on failure — the script then falls back
# to package.json.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \
"repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \
--jq '.[].ref | sub("^refs/tags/"; "")' || true)"
ORCA_PUBLISHED_VERSIONS="$published" ORCA_ADHOC_LABEL="${LABEL:-$REF}" \
node config/scripts/adhoc-build-version.mjs \
>"$RUNNER_TEMP/adhoc-identity.txt"
@@ -253,6 +388,14 @@ jobs:
# silent, which is correct for unvetted branch artifacts.
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
# After the app build so nothing that cleans out/ can drop it; electron-builder ships it.
- name: Download the orcad deployment template
if: needs.orcad-template-support.outputs.ships == 'true'
uses: actions/download-artifact@v8
with:
name: orcad-template
path: out/orcad-template
# Why the token is minted here and not at the top: installation tokens live
# one hour, everything before this point writes nothing, and the notary round
# trip inside the publish step can be tens of minutes. Minting after the build
@@ -311,7 +454,6 @@ jobs:
including back to Stable, works in-app from there."
echo "tag=$TAG" >>"$GITHUB_OUTPUT"
- name: Publish adhoc macOS artifacts
uses: nick-fields/retry@v4
with:
@@ -321,13 +463,15 @@ jobs:
timeout_minutes: 45
max_attempts: 2
retry_wait_seconds: 30
command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_ADHOC=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always
command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_ADHOC=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always
env:
# Why: electron-builder's github publisher targets the repo named in the
# config; the token must therefore carry write access to orca-adhoc.
GH_TOKEN: ${{ steps.app_token.outputs.token }}
ORCA_ADHOC_BUILD_VERSION: ${{ steps.adhoc.outputs.version }}
ORCA_BUILD_COMMIT: ${{ steps.adhoc.outputs.commit }}
# beforePack and afterPack fail the package when the template is absent.
ORCA_REQUIRE_ORCAD_TEMPLATE: ${{ needs.orcad-template-support.outputs.ships == 'true' && '1' || '' }}
CSC_LINK: ${{ secrets.MAC_CERTS }}
CSC_KEY_PASSWORD: ${{ secrets.MAC_CERTS_PASSWORD }}
# Why all three: electron-builder's notarize step authenticates to the
@@ -403,27 +547,51 @@ jobs:
echo "::warning::Could not discard draft $TAG; remove it manually."
- name: Prune expired adhoc releases
# Only after a live publish: $TAG is then a non-draft this step must not
# delete, and a run that failed before publishing has nothing to retire.
if: steps.publish_live.outcome == 'success'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
# Protect the tag this run just shipped, so no filter mistake can delete
# a build minutes after the person who cut it was told it exists.
TAG: ${{ steps.release.outputs.tag }}
run: |
set -euo pipefail
# Why compute the cutoff in bash rather than with jq's `now`: this runs
# once per dispatch, and a fixed epoch makes the threshold visible in the
# log when someone asks where their build went.
cutoff=$(( $(date -u +%s) - ADHOC_RETAIN_DAYS * 86400 ))
echo "Pruning adhoc releases created before $(date -u -r "$cutoff" '+%Y-%m-%dT%H:%M:%SZ')"
echo "Pruning adhoc releases published before $(date -u -r "$cutoff" '+%Y-%m-%dT%H:%M:%SZ')"
# --cleanup-tag so pruning does not leave orphan tags with no release or
# assets attached. Drafts are excluded: a stale draft is the failure
# path's business, not the retention window's.
stale="$(gh release list --repo "$ADHOC_REPO" --limit 200 --json tagName,createdAt,isDraft \
--jq "map(select(.isDraft | not)) | map(select((.createdAt | fromdateiso8601) < $cutoff)) | .[].tagName")"
#
# Age comes from publishedAt, never createdAt. GitHub reports createdAt
# as the date of the *commit* a release's tag points at, and every tag
# here is cut from this repo's one seed commit — so all of them carry
# that same createdAt, and the day the window rolled past it the entire
# channel expired at once and a single run deleted it. A release with no
# publishedAt is kept rather than aged by guesswork.
jq_filter='map(select(.isDraft | not))'
if [[ -n "${TAG:-}" ]]; then
jq_filter+=" | map(select(.tagName != \"${TAG//\"/\\\"}\"))"
fi
jq_filter+=" | map(select((.publishedAt // \"\") != \"\"))"
jq_filter+=" | map(select((.publishedAt | fromdateiso8601) < $cutoff)) | .[].tagName"
stale="$(gh release list --repo "$ADHOC_REPO" --limit 200 --json tagName,publishedAt,isDraft \
--jq "$jq_filter")"
if [[ -z "$stale" ]]; then
echo "Nothing to prune."
exit 0
fi
while read -r tag; do
[[ -n "$tag" ]] || continue
# Belt-and-suspenders: the filter above should already exclude $TAG.
if [[ -n "${TAG:-}" && "$tag" == "$TAG" ]]; then
echo "::warning::Prune list still included just-published $tag after protect; skipping delete."
continue
fi
echo "Pruning $tag"
gh release delete "$tag" --repo "$ADHOC_REPO" --yes --cleanup-tag || \
echo "::warning::Could not prune $tag"
@@ -452,3 +620,4 @@ jobs:
tag: ${{ needs.build-adhoc-mac.outputs.tag }}
ref: ${{ needs.build-adhoc-mac.outputs.head_sha }}
version: ${{ needs.build-adhoc-mac.outputs.version }}
orcad_template: ${{ needs.build-adhoc-mac.outputs.ships_orcad_template == 'true' }}
@@ -0,0 +1,94 @@
name: Publish agent state rules
# The only publisher of agent state rules releases. Merging never publishes: a maintainer runs
# this from main, first to `next` (RC and dev builds), then promotes the identical file to
# `stable` after a soak. Every job runs on the dispatched commit, so the publish job runs exactly
# the script and rules the gate tested.
#
# Why GITHUB_TOKEN: release-policy.yml deletes an agent-state-rules-* release any other author
# publishes. Its releases start no workflow, so release-policy.yml sees them only when someone
# edits one by hand, and then accepts them as bot-authored prereleases.
on:
workflow_dispatch:
inputs:
action:
description: 'publish-next builds from main and publishes to next; promote-stable copies the next file to stable'
required: true
type: choice
options:
- publish-next
- promote-stable
bundled_only:
description: 'publish-next: tell apps to fall back to the rules they shipped with; promote-stable then carries it to stable'
required: false
type: boolean
default: false
permissions:
contents: read
concurrency:
group: agent-state-rules-publish
cancel-in-progress: false
jobs:
gate:
if: github.ref == 'refs/heads/main' && inputs.action == 'publish-next'
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
- name: Run the rules gate (schema, regex safety, the bundle, every transcript replay)
shell: bash
run: |
set -euo pipefail
mapfile -t gate_files < <(git ls-files \
':(glob)src/main/runtime/agent-state-rules/**/*.test.ts' \
':(glob)src/main/runtime/readiness-census*.test.ts' \
':(glob)src/main/runtime/*transcript.test.ts' \
':(glob)src/main/runtime/*transcripts.test.ts' \
config/scripts/agent-state-rules-bundle.test.mjs)
pnpm exec vitest run --config config/vitest.config.ts "${gate_files[@]}"
publish-next:
needs: gate
runs-on: ubuntu-latest
timeout-minutes: 10
environment: agent-state-rules
permissions:
contents: write
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: Publish to next
env:
GH_TOKEN: ${{ github.token }}
BUNDLED_ONLY: ${{ inputs.bundled_only }}
shell: bash
run: |
set -euo pipefail
flags=()
[[ "$BUNDLED_ONLY" == "true" ]] && flags+=(--bundled-only)
node config/scripts/agent-state-rules-bundle.mjs publish-next "${flags[@]}"
promote-stable:
if: github.ref == 'refs/heads/main' && inputs.action == 'promote-stable'
runs-on: ubuntu-latest
timeout-minutes: 10
environment: agent-state-rules
permissions:
contents: write
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: Copy the next file to stable
env:
GH_TOKEN: ${{ github.token }}
run: node config/scripts/agent-state-rules-bundle.mjs promote-stable
@@ -0,0 +1,117 @@
name: Warm shared CI platform cache jobs
on:
workflow_call:
permissions:
contents: read
jobs:
warm-linux-runtime:
if: github.event_name != 'pull_request'
concurrency:
group: ci-cache-warmup-linux-runtime-${{ matrix.runner }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
strategy:
fail-fast: false
matrix:
runner: [ubuntu-24.04-arm, blacksmith-4vcpu-ubuntu-2404]
runs-on: ${{ matrix.runner }}
timeout-minutes: 10
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
node-version: '24'
cache-electron-package: 'true'
cache-pnpm-store-lookup-only: 'true'
- name: Populate shared Electron archive
run: node config/scripts/install-electron-package-binary.mjs
- name: Verify native cache is usable
run: node config/scripts/ensure-native-runtime.mjs --check-only
warm-windows:
concurrency:
group: ci-cache-warmup-windows-${{ matrix.os }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
strategy:
fail-fast: false
matrix:
os: [windows-2022, windows-11-arm]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
# Windows PR consumers download packages directly; retain only the build caches.
cache-pnpm-store: 'false'
- name: Verify native cache is usable
run: node config/scripts/ensure-native-runtime.mjs --check-only
- name: Cache Windows CLI launcher
if: runner.arch == 'X64'
uses: actions/cache@v5
with:
path: |
native/windows-cli-launcher/.build
native/windows-cli-launcher/target
native/windows-agent-hook/.build
native/windows-agent-hook/target
~/.cargo/registry
key: windows-cli-launcher-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('native/windows-cli-launcher/**', 'native/windows-agent-hook/**', 'config/scripts/build-windows-cli-launcher.mjs', 'config/scripts/build-windows-agent-hook.mjs', 'config/scripts/windows-rust-executable-build.mjs', 'resources/build/icon.ico', 'package.json') }}
- name: Build the Windows native executables
if: runner.arch == 'X64'
shell: pwsh
run: |
if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
rustup toolchain install stable --profile minimal
rustup default stable
}
node config/scripts/build-windows-agent-hook.mjs
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
node config/scripts/build-windows-cli-launcher.mjs
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
- name: Warm the Windows ARM server launcher
if: runner.arch == 'ARM64'
uses: ./.github/actions/prepare-windows-cli-launcher
with:
arches: arm64
warm-linux-package-fixtures:
if: github.event_name != 'pull_request'
concurrency:
group: ci-cache-warmup-linux-package-fixtures-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
runs-on: ubuntu-latest
timeout-minutes: 10
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/prepare-linux-package-fixture
with:
fixture: daemon-shutdown-descendants
save-cache: ${{ github.event_name != 'pull_request' }}
- uses: ./.github/actions/prepare-linux-package-fixture
with:
fixture: headless-serve-shutdown
save-cache: ${{ github.event_name != 'pull_request' }}
- uses: ./.github/actions/prepare-linux-package-fixture
with:
fixture: cli-launch-contract
save-cache: ${{ github.event_name != 'pull_request' }}
@@ -0,0 +1,63 @@
name: Warm shared CI platform caches
on:
push:
branches: [main]
paths:
- '.github/workflows/ci-cache-warmup.yml'
- '.github/workflows/ci-cache-warmup-platform-jobs.yml'
- '.github/workflows/ci-cache-warmup-platforms.yml'
- '.github/actions/install-node-dependencies/**'
- '.github/actions/restore-pnpm-verification/**'
- '.github/actions/prepare-native-runtime/**'
- '.github/actions/prepare-windows-cli-launcher/**'
- '.github/actions/prepare-git-compatibility/**'
- '.github/actions/prepare-linux-package-fixture/**'
- 'config/docker/headless-serve-shutdown/**'
- 'config/docker/cli-launch-contract/**'
- 'config/docker/daemon-shutdown-descendants/**'
- 'package.json'
- 'pnpm-lock.yaml'
- 'pnpm-workspace.yaml'
- '.npmrc'
- '.pnpmfile.cjs'
- 'config/scripts/ensure-native-runtime.mjs'
- 'config/scripts/initialize-windows-native-toolchain.mjs'
- 'config/scripts/capture-windows-native-environment.cmd'
- 'config/scripts/rebuild-native-deps.mjs'
- 'config/scripts/node-pty-job-ownership.cjs'
- 'config/scripts/windows-pe-machine.cjs'
- 'config/scripts/windows-process-tree-gyp-rebuild.mjs'
- 'config/scripts/windows-process-tree-creation-time.cjs'
- 'config/scripts/install-electron-package-binary.mjs'
- 'config/scripts/electron-platform-path.mjs'
- 'config/scripts/zip-extractor-command.mjs'
- 'src/shared/zip-extractor-command.ts'
- 'config/scripts/shared-electron-dist-cache.mjs'
- 'config/scripts/space-sharing-copy.mjs'
- 'config/patches/**'
- 'native/windows-registry/**'
- 'native/windows-security/**'
- 'native/windows-cli-launcher/**'
- 'native/windows-agent-hook/**'
- 'config/scripts/build-windows-cli-launcher.mjs'
- 'config/scripts/build-windows-agent-hook.mjs'
- 'config/scripts/windows-rust-executable-build.mjs'
- 'resources/build/icon.ico'
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- '.github/workflows/ci-cache-warmup-platforms.yml'
- '.github/workflows/ci-cache-warmup-platform-jobs.yml'
permissions:
contents: read
concurrency:
group: ci-cache-warmup-platforms-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
warm-platforms:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
uses: ./.github/workflows/ci-cache-warmup-platform-jobs.yml
+103
View File
@@ -0,0 +1,103 @@
name: Warm shared CI caches
on:
# Cache-input pushes seed immediately; this schedule repairs eviction and image changes.
schedule:
- cron: '41 */6 * * *'
workflow_dispatch:
push:
branches: [main]
paths:
- '.github/workflows/ci-cache-warmup.yml'
- '.github/workflows/ci-cache-warmup-platform-jobs.yml'
- '.github/workflows/ci-cache-warmup-platforms.yml'
- '.github/actions/install-node-dependencies/**'
- '.github/actions/restore-pnpm-verification/**'
- '.github/actions/prepare-native-runtime/**'
- '.github/actions/prepare-windows-cli-launcher/**'
- '.github/actions/prepare-git-compatibility/**'
- '.github/actions/prepare-headless-compiler/**'
- 'config/scripts/headless-detector-compiler-cache.mjs'
- '.github/actions/prepare-linux-package-fixture/**'
- '.github/actions/prepare-wayland-clipboard/**'
- 'native/wayland-clipboard/**'
- 'config/scripts/build-wayland-clipboard.mjs'
- 'config/docker/headless-serve-shutdown/**'
- 'config/docker/cli-launch-contract/**'
- 'config/docker/daemon-shutdown-descendants/**'
- 'package.json'
- 'pnpm-lock.yaml'
- 'pnpm-workspace.yaml'
- '.npmrc'
- '.pnpmfile.cjs'
- 'config/scripts/ensure-native-runtime.mjs'
- 'config/scripts/initialize-windows-native-toolchain.mjs'
- 'config/scripts/capture-windows-native-environment.cmd'
- 'config/scripts/rebuild-native-deps.mjs'
- 'config/scripts/node-pty-job-ownership.cjs'
- 'config/scripts/windows-pe-machine.cjs'
- 'config/scripts/windows-process-tree-gyp-rebuild.mjs'
- 'config/scripts/windows-process-tree-creation-time.cjs'
- 'config/scripts/install-electron-package-binary.mjs'
- 'config/scripts/electron-platform-path.mjs'
- 'config/scripts/zip-extractor-command.mjs'
- 'src/shared/zip-extractor-command.ts'
- 'config/scripts/shared-electron-dist-cache.mjs'
- 'config/scripts/space-sharing-copy.mjs'
- 'config/patches/**'
- 'native/windows-registry/**'
- 'native/windows-security/**'
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- '.github/workflows/ci-cache-warmup.yml'
- '.github/actions/prepare-headless-compiler/**'
- 'config/scripts/headless-detector-compiler-cache.mjs'
- '.github/actions/prepare-wayland-clipboard/**'
permissions:
contents: read
concurrency:
group: ci-cache-warmup-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
warm:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
concurrency:
group: ci-cache-warmup-linux-x64-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
runs-on: ubuntu-latest
timeout-minutes: 10
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
# Default-branch caches can be restored by every PR; PR caches cannot.
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
node-version: '24'
cache-electron-package: 'true'
cache-pnpm-store-lookup-only: 'true'
- uses: ./.github/actions/prepare-headless-compiler
with:
seed: 'true'
- name: Populate shared Electron archive
run: node config/scripts/install-electron-package-binary.mjs
- uses: ./.github/actions/prepare-git-compatibility
- uses: ./.github/actions/prepare-wayland-clipboard
with:
save-cache: 'true'
warm-platforms:
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
uses: ./.github/workflows/ci-cache-warmup-platform-jobs.yml
+130
View File
@@ -0,0 +1,130 @@
name: Clean obsolete PR work
on:
pull_request_target:
types: [closed, synchronize]
permissions:
actions: write
pull-requests: read
jobs:
clean:
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 5
steps:
- name: Cancel obsolete PR checks
if: github.event.action == 'synchronize' || github.event.pull_request.merged == false
uses: actions/github-script@v8
with:
script: |
const pr = context.payload.pull_request
const superseded = context.payload.action === 'synchronize'
const targets = [
['pr-checks', 'pr.yml'],
['node-server', 'node-server-tests.yml'],
['ssh-windows-hosts', 'ssh-windows-hosts.yml'],
['ssh-hostile-hosts', 'ssh-hostile-hosts.yml'],
['mobile', 'mobile.yml'],
['computer-e2e', 'computer-e2e.yml'],
['wsl-orcad-e2e', 'wsl-orcad-e2e.yml'],
['performance-contracts', 'performance-contracts.yml'],
['pr-test-loc', 'pr-test-loc.yml'],
['preflight-critical-path', 'preflight-critical-path-benchmark.yml'],
['preflight-cli', 'preflight-cli-platforms.yml'],
['macos-updater', 'macos-updater-tests.yml'],
['preflight-comparison', 'preflight-benchmark.yml'],
['windows-typecheck-parity', 'windows-typecheck-parity.yml'],
['zcode-transcript-contract', 'zcode-transcript-contract.yml'],
['release-ref-validation', 'release-ref-validation.yml'],
['docs-pr', 'docs.yml'],
['cloud-verify-refs/pull', 'cloud-verify.yml', '/merge'],
['pnpm-verification-pilot', 'ci-pnpm-verification-pilot.yml'],
['win-orcad-serve-switch-e2e', 'win-orcad-serve-switch-e2e.yml'],
['pi-provider-runtime', 'pi-provider-runtime.yml'],
['coding-plan-contract', 'zcode-credential-contract.yml', '-credentials'],
['release-javascript-comparison', 'release-javascript-benchmark.yml'],
['ci-cache-warmup-platforms', 'ci-cache-warmup-platforms.yml'],
['pi-owner-runtime', 'pi-owner-runtime.yml'],
['git-command-termination', 'git-command-termination-runtime.yml'],
['skill-roundtrip-pull_request', 'skill-update-roundtrip.yml'],
['ci-cache-warmup', 'ci-cache-warmup.yml']
]
const stillCurrent = async () => {
const { data: current } = await github.rest.pulls.get({
...context.repo, pull_number: pr.number
})
if (superseded) {
return current.state === 'open' && current.head.sha === pr.head.sha
}
return current.state === 'closed' && current.merged_at === null &&
current.closed_at === pr.closed_at
}
if (!Number.isSafeInteger(pr.number) || pr.number <= 0 ||
(superseded ? !/^[a-f0-9]{40}$/.test(pr.head?.sha ?? '') :
!Number.isFinite(Date.parse(pr.closed_at)))) {
throw new Error('Missing PR identity')
}
if (!await stillCurrent()) return
for (const [prefix, workflow, suffix = ''] of targets) {
const group = prefix === 'cloud-verify-refs/pull'
? `${prefix}/${pr.number}${suffix}` : `${prefix}-${pr.number}${suffix}`
let data
try {
const response = await github.request(
'GET /repos/{owner}/{repo}/actions/concurrency_groups/{concurrency_group_name}', {
...context.repo, concurrency_group_name: group,
headers: { 'X-GitHub-Api-Version': '2026-03-10' }
}
)
data = response.data
} catch (error) {
if (error.status === 404) continue
throw error
}
if (data.group_name !== group || !Array.isArray(data.group_members)) {
throw new Error(`Unexpected concurrency group: ${group}`)
}
for (const member of data.group_members) {
// Only whole PR runs; release/manual jobs never share this identity.
if (member.job_id !== undefined || !Number.isSafeInteger(member.run_id)) continue
const { data: run } = await github.rest.actions.getWorkflowRun({
...context.repo, run_id: member.run_id
})
if (run.event !== 'pull_request' || run.path !== `.github/workflows/${workflow}` ||
run.status === 'completed' ||
(superseded ? !/^[a-f0-9]{40}$/.test(run.head_sha ?? '') ||
run.head_sha === pr.head.sha :
!Number.isFinite(Date.parse(run.created_at)) ||
Date.parse(run.created_at) > Date.parse(pr.closed_at))) continue
if (!await stillCurrent()) return
try {
await github.rest.actions.cancelWorkflowRun({
...context.repo, run_id: member.run_id
})
core.info(`Requested cancellation of ${group}: ${member.run_id}`)
} catch (error) {
if (error.status !== 409) throw error
}
}
}
# No checkout: this runs trusted default-branch code, including for fork PRs.
- uses: actions/github-script@v8
if: ${{ !cancelled() && github.event.action == 'closed' }}
with:
script: |
const ref = `refs/pull/${context.payload.pull_request.number}/merge`
const caches = await github.paginate(github.rest.actions.getActionsCacheList, {
...context.repo, ref, per_page: 100
})
for (const cache of caches) {
if (cache.ref !== ref) throw new Error(`Unexpected cache ref: ${cache.ref}`)
try {
await github.rest.actions.deleteActionsCacheById({
...context.repo, cache_id: cache.id
})
} catch (error) {
if (error.status !== 404) throw error
}
}
core.info(`Removed ${caches.length} caches scoped to ${ref}`)
@@ -0,0 +1,67 @@
name: pnpm verification cache pilot
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths: ['.github/workflows/ci-pnpm-verification-pilot.yml']
workflow_dispatch:
permissions:
contents: read
concurrency:
group: pnpm-verification-pilot-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
ORCA_BACKGROUND_LAUNCH: '1'
jobs:
seed:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
strategy:
matrix:
runner: [ubuntu-latest, ubuntu-24.04-arm]
runs-on: ${{ matrix.runner }}
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
id: deps
# PR caches are scoped to the PR merge ref; main never restores them.
- uses: actions/cache/save@v5
if: steps.deps.outputs.verification-cache-hit != 'true'
with:
path: ${{ steps.deps.outputs.verification-cache-path }}
key: ${{ steps.deps.outputs.verification-cache-key }}
measure:
needs: seed
name: measure ${{ matrix.runner }} sample ${{ matrix.sample }} cache ${{ matrix.cache }}
strategy:
fail-fast: false
max-parallel: 4
matrix:
runner: [ubuntu-latest, ubuntu-24.04-arm]
sample: [1, 2, 3]
cache: ['false', 'true']
runs-on: ${{ matrix.runner }}
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
id: deps
with:
cache-pnpm-verification: ${{ matrix.cache }}
- name: Validate treatment and frozen install
env:
CACHE_ENABLED: ${{ matrix.cache }}
CACHE_HIT: ${{ steps.deps.outputs.verification-cache-hit }}
run: |
if [ "$CACHE_ENABLED" = true ]; then test "$CACHE_HIT" = true; fi
git diff --exit-code -- package.json pnpm-lock.yaml pnpm-workspace.yaml
node -e "require.resolve('vitest'); require.resolve('electron')"
+33
View File
@@ -0,0 +1,33 @@
name: CI runner demand
on:
schedule:
- cron: '23 4 * * *'
workflow_dispatch:
permissions:
contents: read
actions: read
concurrency:
group: ci-runner-demand
cancel-in-progress: false
jobs:
report:
runs-on: ubuntu-slim
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
sparse-checkout: config/scripts
persist-credentials: false
- name: Measure the previous complete 24 hours
env:
GH_TOKEN: ${{ github.token }}
run: node config/scripts/ci-runner-demand.mjs
- uses: actions/upload-artifact@v7
with:
name: ci-runner-demand-${{ github.run_id }}
path: ci-demand/
retention-days: 30
+55
View File
@@ -0,0 +1,55 @@
name: Shared xterm dependency cache
on:
push:
branches: [main]
paths:
- '.github/workflows/ci-xterm-cache.yml'
- '.github/actions/prepare-xterm-dependencies/**'
- 'config/patches/xterm-upstream.json'
- 'config/scripts/regenerate-xterm-patches.mjs'
- 'config/scripts/xterm-patch-text.mjs'
- 'package.json'
# Refresh after Node patch releases or cache eviction without rebuilding on hits.
schedule:
- cron: '17 5 * * *'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: xterm-cache-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
ORCA_BACKGROUND_LAUNCH: '1'
jobs:
seed:
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-24.04-arm
timeout-minutes: 10
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: actions/setup-node@v6
with:
node-version-file: package.json
package-manager-cache: false
- uses: ./.github/actions/prepare-xterm-dependencies
id: cache
- name: Verify and populate dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: node config/scripts/regenerate-xterm-patches.mjs --check --work-dir="$RUNNER_TEMP/xterm-patch-build"
- uses: actions/cache/save@v5
if: steps.cache.outputs.cache-hit != 'true'
with:
path: |
${{ runner.temp }}/xterm-patch-build/upstream/.git
${{ runner.temp }}/xterm-patch-build/upstream/node_modules
key: ${{ steps.cache.outputs.cache-key }}
- name: Preserve fallback downloads
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v5
with:
path: |
~/.npm
${{ runner.temp }}/xterm-patch-build/upstream/.git
key: xterm-upstream-${{ hashFiles('config/patches/xterm-upstream.json') }}
@@ -0,0 +1,677 @@
name: Bootstrap Relay Staging Capacity
on:
workflow_dispatch:
inputs:
confirmation:
description: Enter BOOTSTRAP_STAGING_CAPACITY
required: true
type: string
permissions:
contents: read
id-token: write
concurrency:
group: relay-staging-mutation
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
bootstrap:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: staging
env:
GCP_PROJECT_ID: onorca-cloud-staging
GCP_REGION: us-central1
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
LEGACY_C3_IMAGE_DIGEST: sha256:2d0f6e6db2b0eb9d6aba188698de8330f8c30b4e76badfcf0fac3f3eb9508a87
steps:
- uses: actions/checkout@v4
- id: deploy-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- id: capacity-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
token_format: access_token
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-staging-terraform-state
object: terraform/state/cloud-sql-rollout/staging.lock
- uses: hashicorp/setup-terraform@v3
with:
terraform_wrapper: false
- uses: actions/setup-node@v4
with:
node-version: 24
- name: Require explicit bootstrap confirmation
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: test "${CONFIRMATION}" = "BOOTSTRAP_STAGING_CAPACITY"
- name: Read and verify reviewed 600/60 topology
shell: bash
run: |
node dev/scripts/infra.mjs init --env staging
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
> "${RUNNER_TEMP}/relay-gce-state.json"
DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars \
<<< 'local.relay_director_cells_json' | jq -r '.')"
DESIRED_IMAGES_JSON="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars \
<<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.image })' \
| jq -r '.')"
DESIRED_ZONES_JSON="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars \
<<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.zone })' \
| jq -r '.')"
DESIRED_TARGET_SIZES_JSON="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars \
<<< 'jsonencode(local.relay_gce_cell_target_sizes)' | jq -r '.')"
jq -e \
--argjson images "${DESIRED_IMAGES_JSON}" \
--argjson target_sizes "${DESIRED_TARGET_SIZES_JSON}" \
'([.[] | select(.id == "staging-gce-c2")] | length == 1) and
([.[] | select(.id == "staging-gce-c3")] | length == 1) and
(any(.[]; .id == "staging-gce-c2" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and
(any(.[]; .id == "staging-gce-c3" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and
($images["staging-gce-c2"] == $images["staging-gce-c3"]) and
($target_sizes["staging-gce-c2"] == 1) and
($target_sizes["staging-gce-c3"] == 1)' \
<<< "${DESIRED_CELLS_JSON}" >/dev/null
jq \
--argjson desired "${DESIRED_CELLS_JSON}" \
--argjson images "${DESIRED_IMAGES_JSON}" \
--argjson zones "${DESIRED_ZONES_JSON}" \
'($desired | map({key: .id, value: .}) | from_entries) as $cells |
to_entries | map(. as $entry | {
key: $entry.key,
value: ($entry.value + {
origin: $cells[$entry.key].url,
image: $images[$entry.key],
zone: $zones[$entry.key],
connection_hard_cap: $cells[$entry.key].connectionHardCap,
connection_unobserved_bound: $cells[$entry.key].connectionUnobservedBound
})
}) | from_entries' \
"${RUNNER_TEMP}/relay-gce-state.json" \
> "${RUNNER_TEMP}/relay-gce-topology.json"
ACTIVE_REVISION="$(gcloud run services describe orca-cloud-relay-staging \
--project "${GCP_PROJECT_ID}" \
--region us-central1 \
--format=json \
| jq -r '
[.status.traffic[] | select((.percent // 0) > 0)] |
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test -n "${ACTIVE_REVISION}"
DESIRED_IMAGE="$(jq -r '.["staging-gce-c2"]' <<< "${DESIRED_IMAGES_JSON}")"
gcloud run revisions describe "${ACTIVE_REVISION}" \
--project "${GCP_PROJECT_ID}" \
--region us-central1 \
--format=json \
| jq -e \
--arg image "${DESIRED_IMAGE}" \
--arg capacity_service_account "${CAPACITY_SERVICE_ACCOUNT}" \
'(.spec.containers[0].image == $image) and
any(.spec.containers[0].env[]?;
.name == "ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT" and
.value == $capacity_service_account)' >/dev/null
CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--format=json \
| jq -cer '
[.spec.containers[0].env[]? |
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
if length == 1 then .[0] | fromjson else error("missing director topology") end')"
jq -e --argjson desired "${DESIRED_CELLS_JSON}" '
def without_bootstrap_capacity:
map(if .id == "staging-gce-c2" or .id == "staging-gce-c3"
then del(.connectionHardCap, .connectionUnobservedBound)
else . end);
without_bootstrap_capacity == ($desired | without_bootstrap_capacity)
' <<< "${CURRENT_CELLS_JSON}" >/dev/null
- name: Bootstrap C2 then C3
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
topology="${RUNNER_TEMP}/relay-gce-topology.json"
fixed_one_instance_name() {
local cell_id="$1"
local mig_name zone
mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")"
zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
gcloud compute instance-groups managed list-instances \
"${mig_name}" \
--project "${GCP_PROJECT_ID}" \
--zone "${zone}" \
--format=json \
| jq -er '
if length == 1 and .[0].instanceStatus == "RUNNING" and
.[0].currentAction == "NONE"
then .[0].instance | split("/") | last
else error("legacy cell does not have one stable running instance") end'
}
fixed_one_instance_id() {
local cell_id="$1"
local instance_name="$2"
local zone
zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
gcloud compute instances describe "${instance_name}" \
--project "${GCP_PROJECT_ID}" \
--zone "${zone}" \
--format='value(id)'
}
write_legacy_metrics() {
local cell_id="$1"
local instance_id="$2"
local after="$3"
local output="$4"
gcloud logging read \
"resource.type=\"gce_instance\" AND
resource.labels.instance_id=\"${instance_id}\" AND
jsonPayload.event=\"orca_relay_runtime_metrics\" AND
jsonPayload.cellId=\"${cell_id}\" AND
timestamp>=\"${after}\"" \
--project "${GCP_PROJECT_ID}" \
--limit 10 \
--order desc \
--format json \
| jq '[.[] | {
timestamp,
cellId: .jsonPayload.cellId,
metricVersion: .jsonPayload.metricVersion,
totalConnections: .jsonPayload.totalConnections,
preAuthConnections: .jsonPayload.preAuthConnections,
controls: .jsonPayload.controls,
splices: .jsonPayload.splices,
pendingSplices: .jsonPayload.pendingSplices,
queuedBytes: .jsonPayload.queuedBytes
}]' > "${output}"
}
verify_legacy_cell() {
local cell_id="$1"
local admission="$2"
local after="$3"
local expected_instance_id="$4"
local runtime_started_after="${5:-}"
local previous_incarnation_digest="${6:-}"
local hard_cap="${7:-}"
local unobserved_bound="${8:-}"
local capacity_state="${9:-}"
local current_instance current_instance_id metrics origin result
local runtime_start_args=() incarnation_args=() capacity_args=()
current_instance="$(fixed_one_instance_name "${cell_id}")"
current_instance_id="$(fixed_one_instance_id "${cell_id}" "${current_instance}")"
test "${current_instance_id}" = "${expected_instance_id}"
metrics="${RUNNER_TEMP}/${cell_id}-legacy-runtime-metrics.json"
origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
if test -n "${runtime_started_after}"; then
runtime_start_args=(--runtime-started-after "${runtime_started_after}")
fi
if test -n "${previous_incarnation_digest}"; then
incarnation_args=(--previous-incarnation-digest "${previous_incarnation_digest}")
fi
if test -n "${hard_cap}"; then
capacity_args=(--hard-cap "${hard_cap}" --unobserved-bound "${unobserved_bound}")
fi
if test -n "${capacity_state}"; then
capacity_args+=(--capacity-state "${capacity_state}")
fi
for _attempt in $(seq 1 18); do
write_legacy_metrics \
"${cell_id}" "${current_instance_id}" "${after}" "${metrics}"
if result="$(node dev/scripts/verify-relay-legacy-bootstrap.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${origin}" \
--cell-id "${cell_id}" \
--admission "${admission}" \
--expected-image-digest "${LEGACY_C3_IMAGE_DIGEST}" \
--metrics-after "${after}" \
--metrics-file "${metrics}" \
"${runtime_start_args[@]}" \
"${incarnation_args[@]}" \
"${capacity_args[@]}")"; then
echo "${result}"
return 0
fi
sleep 10
done
return 1
}
post_admin() {
local origin="$1"
local path="$2"
local body="$3"
curl --fail --silent --show-error \
--request POST \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' \
--data "${body}" \
"${origin}${path}"
}
runtime_kind() {
local cell_id="$1"
local origin desired_digest digest
origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
desired_digest="$(jq -r --arg cell "${cell_id}" \
'.[$cell].image | split("@") | last' "${topology}")"
digest="$(post_admin "${origin}" /v1/admin/runtime-status '{"v":1}' \
| jq -er '.imageDigest')"
if test "${digest}" = "${LEGACY_C3_IMAGE_DIGEST}"; then
echo legacy
elif test "${digest}" = "${desired_digest}"; then
echo modern
else
echo 'bootstrap cell image is neither legacy nor reviewed' >&2
return 1
fi
}
cell_admission() {
local cell_id="$1"
post_admin "${DIRECTOR_ORIGIN}" /v1/admin/cell-status \
"$(jq -cn --arg cell "${cell_id}" '{v:1, cellId:$cell}')" \
| jq -er '.status.admissionState |
if . == "general" or . == "migration-only" then .
else error("bootstrap admission is not recoverable") end'
}
verify_modern_cell() {
local cell_id="$1"
local admission="$2"
local origin
origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${origin}" \
--cell-id "${cell_id}" \
--hard-cap 600 \
--unobserved-bound 60 \
--heartbeat fresh \
--admission "${admission}" \
--draining forbidden \
--activity allowed
}
ensure_modern_general() {
local cell_id="$1"
local admission="$2"
verify_modern_cell "${cell_id}" "${admission}"
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-id "${cell_id}" \
--mode restore \
--general-cell-ids "${cell_id}"
verify_modern_cell "${cell_id}" general
}
prepare_legacy_c3_fallback() {
legacy_c3_metrics_boundary="$(node -e \
'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')"
legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
legacy_c3_instance_id="$(fixed_one_instance_id \
staging-gce-c3 "${legacy_c3_instance}")"
verify_legacy_cell \
staging-gce-c3 general \
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}"
node dev/scripts/probe-relay-legacy-admission.mjs \
--cell-origin https://c3.relay-staging.onorca.dev
legacy_c3_restart_started_after=
legacy_c3_old_incarnation=
}
normalize_legacy_c3() {
legacy_pre_boundary="$(node -e \
'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')"
legacy_c2_instance="$(fixed_one_instance_name staging-gce-c2)"
legacy_c2_instance_id="$(fixed_one_instance_id \
staging-gce-c2 "${legacy_c2_instance}")"
verify_legacy_cell \
staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}"
node dev/scripts/probe-relay-legacy-admission.mjs \
--cell-origin https://c2.relay-staging.onorca.dev
legacy_c3_isolated=false
restore_legacy_c3_fallback() {
if test "${legacy_c3_isolated}" = true; then
verify_legacy_cell \
staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}"
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-id staging-gce-c3 \
--mode restore-fallback \
--general-cell-ids staging-gce-c2
fi
}
trap restore_legacy_c3_fallback EXIT
legacy_c3_isolated=true
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin https://c3.relay-staging.onorca.dev \
--cell-id staging-gce-c3 \
--mode isolate
legacy_c3_drain_boundary="$(node -e \
'process.stdout.write(new Date().toISOString())')"
legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
legacy_c3_instance_id="$(fixed_one_instance_id \
staging-gce-c3 "${legacy_c3_instance}")"
legacy_c3_drained="$(verify_legacy_cell \
staging-gce-c3 migration-only \
"${legacy_c3_drain_boundary}" "${legacy_c3_instance_id}")"
echo "${legacy_c3_drained}"
legacy_c3_old_incarnation="$(jq -er '.incarnationDigest' \
<<< "${legacy_c3_drained}")"
legacy_c3_restart_started_after="$(node -e \
'process.stdout.write(new Date().toISOString())')"
gcloud compute instance-groups managed recreate-instances \
orca-cloud-staging-relay-gce-c3 \
--instances "${legacy_c3_instance}" \
--project "${GCP_PROJECT_ID}" \
--zone us-central1-a \
--quiet
gcloud compute instance-groups managed wait-until \
orca-cloud-staging-relay-gce-c3 \
--stable \
--project "${GCP_PROJECT_ID}" \
--zone us-central1-a \
--timeout 900
legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
legacy_c3_instance_id="$(fixed_one_instance_id \
staging-gce-c3 "${legacy_c3_instance}")"
legacy_c3_metrics_boundary="$(node -e \
'process.stdout.write(new Date().toISOString())')"
verify_legacy_cell \
staging-gce-c3 migration-only \
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
"${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-id staging-gce-c3 \
--mode restore \
--general-cell-ids staging-gce-c2,staging-gce-c3
verify_legacy_cell \
staging-gce-c3 general \
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
"${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
legacy_c3_isolated=false
trap - EXIT
}
roll_cell() (
local cell_id="$1"
local fallback_cell_id="$2"
local target_kind="$3"
local fallback_kind="$4"
local active_revision cell_origin current_cells_json desired_bound desired_cap
local desired_cells_json director_result image mig_name plan
local fallback_origin plan_changes plan_result restored target_drain_boundary
local target_instance target_instance_id zone
cell_origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
fallback_origin="$(jq -r --arg cell "${fallback_cell_id}" \
'.[$cell].origin' "${topology}")"
zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")"
image="$(jq -r --arg cell "${cell_id}" '.[$cell].image' "${topology}")"
desired_cap="$(jq -r --arg cell "${cell_id}" \
'.[$cell].connection_hard_cap' "${topology}")"
desired_bound="$(jq -r --arg cell "${cell_id}" \
'.[$cell].connection_unobserved_bound' "${topology}")"
plan="${RUNNER_TEMP}/${cell_id}-capacity-bootstrap.tfplan"
restored=false
restore_fallback() {
if test "${restored}" = false; then
verify_fallback
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-id "${cell_id}" \
--mode restore-fallback \
--general-cell-ids "${fallback_cell_id}"
fi
}
verify_fallback() {
if test "${fallback_kind}" = legacy; then
verify_legacy_cell \
"${fallback_cell_id}" general \
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
"${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
return
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${fallback_origin}" \
--cell-id "${fallback_cell_id}" \
--hard-cap 600 \
--unobserved-bound 60 \
--heartbeat fresh \
--admission general \
--draining forbidden \
--activity allowed
}
verify_fallback
trap restore_fallback EXIT
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${cell_origin}" \
--cell-id "${cell_id}" \
--mode isolate
target_drain_boundary="$(node -e \
'process.stdout.write(new Date().toISOString())')"
if test "${target_kind}" = legacy; then
target_instance="$(fixed_one_instance_name "${cell_id}")"
target_instance_id="$(fixed_one_instance_id "${cell_id}" "${target_instance}")"
verify_legacy_cell \
"${cell_id}" migration-only \
"${target_drain_boundary}" "${target_instance_id}" \
'' '' "${desired_cap}" "${desired_bound}" absent-or-stale
else
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${cell_origin}" \
--cell-id "${cell_id}" \
--heartbeat either \
--admission migration-only \
--draining required \
--activity quiescent
fi
active_revision="$(gcloud run services describe orca-cloud-relay-staging \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--format=json \
| jq -r '
[.status.traffic[] | select((.percent // 0) > 0)] |
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test -n "${active_revision}"
current_cells_json="$(gcloud run revisions describe "${active_revision}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--format=json \
| jq -cer '
[.spec.containers[0].env[]? |
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
if length == 1 then .[0] | fromjson else error("missing director topology") end')"
desired_cells_json="$(jq -ce \
--arg cell "${cell_id}" \
--argjson cap "${desired_cap}" \
--argjson bound "${desired_bound}" \
'map(if .id == $cell then . + {
connectionHardCap: $cap,
connectionUnobservedBound: $bound
} else . end)' <<< "${current_cells_json}")"
director_result="$(node dev/scripts/deploy-relay-blue-green.mjs \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--service orca-cloud-relay-staging \
--image "${image}" \
--role director \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--capacity-cell-id "${cell_id}" \
--director-cells-json "${desired_cells_json}" \
--prune-revisions true \
--release-id "bootstrap-${cell_id}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}")"
echo "${director_result}"
if test "${target_kind}" = legacy; then
verify_legacy_cell \
"${cell_id}" migration-only \
"${target_drain_boundary}" "${target_instance_id}" \
'' '' "${desired_cap}" "${desired_bound}"
else
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${cell_origin}" \
--cell-id "${cell_id}" \
--hard-cap "${desired_cap}" \
--unobserved-bound "${desired_bound}" \
--heartbeat either \
--admission migration-only \
--draining required \
--activity quiescent
fi
terraform -chdir=infra/terraform plan \
-var-file=environments/staging.tfvars \
"-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \
-out="${plan}"
plan_result="$(terraform -chdir=infra/terraform show -json "${plan}" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode bootstrap-cell \
--cell-id "${cell_id}" \
--hard-cap 600 \
--unobserved-bound 60 \
--image "${image}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
echo "${plan_result}"
plan_changes="$(jq -r '.changes' <<< "${plan_result}")"
[[ "${plan_changes}" =~ ^(0|2)$ ]]
if test "${plan_changes}" = 2; then
terraform -chdir=infra/terraform apply -auto-approve "${plan}"
else
target_instance="$(fixed_one_instance_name "${cell_id}")"
gcloud compute instance-groups managed recreate-instances "${mig_name}" \
--instances "${target_instance}" \
--project "${GCP_PROJECT_ID}" \
--zone "${zone}" \
--quiet
fi
gcloud compute instance-groups managed wait-until "${mig_name}" \
--stable \
--project "${GCP_PROJECT_ID}" \
--zone "${zone}" \
--timeout 900
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${cell_origin}" \
--cell-id "${cell_id}" \
--hard-cap 600 \
--unobserved-bound 60 \
--heartbeat fresh \
--admission migration-only \
--draining forbidden \
--activity allowed
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-id "${cell_id}" \
--mode restore \
--general-cell-ids staging-gce-c2,staging-gce-c3
restored=true
trap - EXIT
)
c2_kind="$(runtime_kind staging-gce-c2)"
c3_kind="$(runtime_kind staging-gce-c3)"
c2_admission="$(cell_admission staging-gce-c2)"
c3_admission="$(cell_admission staging-gce-c3)"
bootstrap_phase="$(node dev/scripts/classify-relay-staging-bootstrap.mjs \
--c2-kind "${c2_kind}" \
--c2-admission "${c2_admission}" \
--c3-kind "${c3_kind}" \
--c3-admission "${c3_admission}")"
jq -cn --arg phase "${bootstrap_phase}" \
'{event:"relay_staging_bootstrap_phase", phase:$phase}'
case "${bootstrap_phase}" in
normalize-and-roll-both)
normalize_legacy_c3
roll_cell staging-gce-c2 staging-gce-c3 legacy legacy
roll_cell staging-gce-c3 staging-gce-c2 legacy modern
;;
resume-c2-then-c3)
prepare_legacy_c3_fallback
roll_cell staging-gce-c2 staging-gce-c3 legacy legacy
roll_cell staging-gce-c3 staging-gce-c2 legacy modern
;;
roll-c2)
ensure_modern_general staging-gce-c3 "${c3_admission}"
roll_cell staging-gce-c2 staging-gce-c3 legacy modern
;;
roll-c3)
ensure_modern_general staging-gce-c2 "${c2_admission}"
roll_cell staging-gce-c3 staging-gce-c2 legacy modern
;;
complete)
ensure_modern_general staging-gce-c2 "${c2_admission}"
ensure_modern_general staging-gce-c3 "${c3_admission}"
;;
*)
echo 'unsupported staging bootstrap phase' >&2
exit 1
;;
esac
- name: Verify both bootstrapped cells
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
for number in 2 3; do
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "https://c${number}.relay-staging.onorca.dev" \
--cell-id "staging-gce-c${number}" \
--hard-cap 600 \
--unobserved-bound 60 \
--heartbeat fresh \
--admission general \
--draining forbidden \
--activity allowed
done
@@ -0,0 +1,288 @@
name: Deploy Relay Asia Topology
on:
workflow_dispatch:
inputs:
environment:
description: Target Relay environment
required: true
type: choice
options: [staging, production]
mode:
description: Validate a saved plan or apply that exact plan
required: true
default: plan
type: choice
options: [plan, apply]
cell-ids:
description: Exact reviewed comma-separated cell set (Asia, or the US cells at the Asia shape)
required: true
type: string
image:
description: Full environment Relay image pinned by sha256 digest
required: true
type: string
confirmation:
description: Enter APPLY_RELAY_ASIA_TOPOLOGY for apply mode
required: false
type: string
permissions:
contents: read
id-token: write
concurrency:
group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
topology:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 30
environment: ${{ inputs.environment }}
env:
DEPLOY_MODE: ${{ inputs.mode }}
TARGET_ENVIRONMENT: ${{ inputs.environment }}
TARGET_CELL_IDS: ${{ inputs.cell-ids }}
TARGET_IMAGE: ${{ inputs.image }}
GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }}
CLOUD_SQL_INSTANCE: ${{ inputs.environment == 'production' && 'orca-cloud-auth-db' || 'orca-cloud-staging-auth-db' }}
VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360
VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17
# SHOW max_connections on the live instance, 2026-09-16; no flag is set.
VERIFIED_DEFAULT_MAX_CONNECTIONS: '500'
TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }}
TF_VARS: ${{ inputs.environment == 'production' && 'environments/production.tfvars' || 'environments/staging.tfvars' }}
TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }}
TOPOLOGY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT }}
steps:
- uses: actions/checkout@v4
- name: Validate the reviewed request before authentication
shell: bash
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: |
set -euo pipefail
test -n "${TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER}"
test -n "${TOPOLOGY_SERVICE_ACCOUNT}"
target_region=asia-east2
case "${TARGET_ENVIRONMENT}:${TARGET_CELL_IDS}" in
staging:staging-gce-c4) ;;
production:production-gce-c27,production-gce-c28,production-gce-c29) ;;
production:production-gce-c30) ;;
production:production-gce-c31) ;;
production:production-gce-c32,production-gce-c33) target_region=us-central1 ;;
production:production-gce-c34) ;;
production:production-gce-c35) ;;
*) echo "cell-ids do not match the reviewed environment topology" >&2; exit 1 ;;
esac
echo "TARGET_REGION=${target_region}" >> "${GITHUB_ENV}"
[[ "${TARGET_IMAGE}" =~ ^us-central1-docker\.pkg\.dev/${GCP_PROJECT_ID}/orca-cloud/relay@sha256:[0-9a-f]{64}$ ]]
if test "${DEPLOY_MODE}" = apply; then
test "${CONFIRMATION}" = APPLY_RELAY_ASIA_TOPOLOGY
else
test "${DEPLOY_MODE}" = plan
test -z "${CONFIRMATION}"
fi
- uses: hashicorp/setup-terraform@v3
with:
terraform_version: 1.15.8
terraform_wrapper: false
- uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ env.TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ env.TOPOLOGY_SERVICE_ACCOUNT }}
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }}
object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }}
- name: Require the checked Cloud SQL connection budget
shell: bash
run: |
set -euo pipefail
budget="$(node dev/scripts/relay-cloud-sql-connection-budget.mjs)"
checked_max="$(jq -er '.maxConnections' <<< "${budget}")"
jq -e '.withinBudget == true' <<< "${budget}" >/dev/null
if test "${TARGET_ENVIRONMENT}" = production; then
instance="$(gcloud sql instances describe "${CLOUD_SQL_INSTANCE}" \
--project "${GCP_PROJECT_ID}" --format=json)"
live_flag="$(jq -er '[.settings.databaseFlags[]? |
select(.name == "max_connections") | .value] |
if length <= 1 then (.[0] // "") else error("duplicate max_connections flags") end' \
<<< "${instance}")"
if test -n "${live_flag}"; then
live_max="${live_flag}"
live_source=explicit-flag
else
# No flag: the ceiling is the tier default measured for this exact
# shape; fail closed if the shape changes.
test "$(jq -er '.settings.tier' <<< "${instance}")" = \
"${VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER}"
test "$(jq -er '.databaseVersion' <<< "${instance}")" = \
"${VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION}"
live_max="${VERIFIED_DEFAULT_MAX_CONNECTIONS}"
live_source=verified-shape-default
fi
test "${live_max}" = "${checked_max}"
else
live_max="not-read-for-staging"
live_source=not-read-for-staging
fi
{
echo "### Relay Cloud SQL connection budget"
echo "- Checked maximum: ${checked_max}"
echo "- Configured maximum: $(jq -er '.configuredMaximum' <<< "${budget}")"
echo "- Rollout operating maximum: $(jq -er '.operatingMaximum' <<< "${budget}")"
echo "- Explicit reserve: $(jq -er '.explicitReserve' <<< "${budget}")"
echo "- Production live max_connections: ${live_max}"
echo "- Production live maximum source: ${live_source}"
} >> "${GITHUB_STEP_SUMMARY}"
- name: Initialize the exact environment state
run: terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}"
- id: targets
name: Build the exact additive target set
shell: bash
run: |
set -euo pipefail
file="${RUNNER_TEMP}/relay-asia-targets"
: > "${file}"
# The root region's network predates every cell; only asia-east2's is additive.
if test "${TARGET_REGION}" = asia-east2; then
printf '%s\n' \
'-target=google_compute_subnetwork.relay_gce_additional["asia-east2"]' \
'-target=google_compute_router.relay_gce_additional["asia-east2"]' \
'-target=google_compute_router_nat.relay_gce_additional["asia-east2"]' >> "${file}"
fi
printf '%s\n' '-target=google_compute_url_map.relay_gce[0]' >> "${file}"
IFS=, read -ra cells <<< "${TARGET_CELL_IDS}"
for cell_id in "${cells[@]}"; do
printf '%s\n' \
"-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \
"-target=google_compute_backend_service.relay_gce_cell[\"${cell_id}\"]" >> "${file}"
done
echo "file=${file}" >> "${GITHUB_OUTPUT}"
- id: live-images
name: Plan every non-target cell at the image it serves
shell: bash
run: |
set -euo pipefail
# The URL map target pulls every cell's template into the plan, and same-cap rolls
# leave committed images behind the served ones; read only templates out of state.
cells="${RUNNER_TEMP}/relay-committed-cells.json"
live="${RUNNER_TEMP}/relay-live-cell-templates.json"
overlay="${RUNNER_TEMP}/relay-asia-live-images.tfvars.json"
committed_plan="${RUNNER_TEMP}/relay-committed-cells.tfplan"
mapfile -t targets < "${{ steps.targets.outputs.file }}"
# Not console: it evaluates every output against state, where a new cell has no MIG yet.
terraform -chdir=infra/terraform plan -input=false -refresh=false -lock=false \
-var-file="${TF_VARS}" "${targets[@]}" -out="${committed_plan}" > /dev/null
terraform -chdir=infra/terraform show -json "${committed_plan}" \
| jq -ce '.variables.relay_gce_cells.value | objects' > "${cells}"
terraform -chdir=infra/terraform show -json | jq -ce '[
.values.root_module.resources[]?
| select(.mode == "managed" and
.type == "google_compute_instance_template" and .name == "relay_gce_cell")
| { index, metadata_startup_script: .values.metadata_startup_script }
]' > "${live}"
summary="$(node dev/scripts/relay-live-cell-image-overlay.mjs \
--cells-json "${cells}" --live-templates-json "${live}" \
--cell-ids "${TARGET_CELL_IDS}" --output "${overlay}")"
echo "file=${overlay}" >> "${GITHUB_OUTPUT}"
{
echo "### Live images held for non-target cells"
echo "- Cells whose committed image differs from the served one: $(jq -r '.drifted | join(", ")' <<< "${summary}")"
} >> "${GITHUB_STEP_SUMMARY}"
- name: Create and validate the saved topology plan
id: plan
shell: bash
run: |
set -euo pipefail
plan="${RUNNER_TEMP}/relay-asia-topology.tfplan"
plan_json="${RUNNER_TEMP}/relay-asia-topology.json"
mapfile -t targets < "${{ steps.targets.outputs.file }}"
terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \
-var-file="${TF_VARS}" -var-file="${{ steps.live-images.outputs.file }}" \
"${targets[@]}" -out="${plan}"
terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}"
committed="${RUNNER_TEMP}/relay-committed-asia-topology.json"
jq -e '{
relay_gce_cells: .variables.relay_gce_cells.value,
relay_gce_additional_region_subnetwork_cidrs:
.variables.relay_gce_additional_region_subnetwork_cidrs.value
}' "${plan_json}" > "${committed}"
node dev/scripts/prepare-relay-asia-topology-input.mjs \
--existing-json "${committed}" \
--environment "${TARGET_ENVIRONMENT}" \
--cell-ids "${TARGET_CELL_IDS}" \
--image "${TARGET_IMAGE}"
result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \
--plan-json "${plan_json}" \
--environment "${TARGET_ENVIRONMENT}" \
--cell-ids "${TARGET_CELL_IDS}" \
--region "${TARGET_REGION}" \
--image "${TARGET_IMAGE}")"
changes="$(jq -er '.changes' <<< "${result}")"
digest="$(sha256sum "${plan}" | awk '{print $1}')"
echo "plan=${plan}" >> "${GITHUB_OUTPUT}"
echo "changes=${changes}" >> "${GITHUB_OUTPUT}"
{
echo "### Relay Asia topology saved plan"
echo "- Environment: ${TARGET_ENVIRONMENT}"
echo "- Cells: ${TARGET_CELL_IDS}"
echo "- Region: ${TARGET_REGION}"
echo "- Mutating resources: ${changes}"
echo "- Saved-plan SHA-256: ${digest}"
} >> "${GITHUB_STEP_SUMMARY}"
- name: Apply only the validated saved plan
if: ${{ inputs.mode == 'apply' }}
run: terraform -chdir=infra/terraform apply -input=false -auto-approve "${{ steps.plan.outputs.plan }}"
- name: Prove the exact topology targets converged
if: ${{ inputs.mode == 'apply' }}
shell: bash
run: |
set -euo pipefail
mapfile -t targets < "${{ steps.targets.outputs.file }}"
plan="${RUNNER_TEMP}/relay-asia-topology-readback.tfplan"
plan_json="${RUNNER_TEMP}/relay-asia-topology-readback.json"
terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \
-var-file="${TF_VARS}" -var-file="${{ steps.live-images.outputs.file }}" \
"${targets[@]}" -out="${plan}"
terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}"
result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \
--plan-json "${plan_json}" \
--environment "${TARGET_ENVIRONMENT}" \
--cell-ids "${TARGET_CELL_IDS}" \
--region "${TARGET_REGION}" \
--image "${TARGET_IMAGE}")"
test "$(jq -er '.changes' <<< "${result}")" = 0
- name: Record the required selector-safe next step
if: ${{ inputs.mode == 'apply' }}
run: |
{
echo "### Required next step"
echo "The VMs are not eligible for ordinary placement yet."
echo "Register the exact new cells atomically as migration-only before any director configuration lists them."
echo "Rollback is migration-only admission; do not destroy the Asia network on rollout day."
} >> "${GITHUB_STEP_SUMMARY}"
@@ -0,0 +1,92 @@
name: Deploy Relay Fence Broker
on:
workflow_dispatch:
inputs:
image-digest:
description: Immutable broker image digest built from this main commit
required: true
type: string
permissions:
contents: read
id-token: write
concurrency:
group: production-cloud-sql-rollout
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
deploy:
if: >-
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
github.ref == 'refs/heads/main' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: production
env:
GCP_PROJECT_ID: onorca-cloud
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
SERVICE_NAME: orca-cloud-relay-fence
IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay-fence-broker
IMAGE_DIGEST: ${{ inputs.image-digest }}
steps:
- uses: actions/checkout@v4
- uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
- name: Resolve exact-commit broker image
run: |
[[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \
--format='value(image_summary.digest)')"
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
TAGS="$(gcloud artifacts docker tags list "${IMAGE_REPOSITORY}" \
--filter="version:${IMAGE_DIGEST}" \
--format=json)"
jq -e --arg tag "/tags/sha-${GITHUB_SHA}" \
'any(.[]; .tag | endswith($tag))' <<< "${TAGS}"
echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
- name: Deploy broker image only
run: |
gcloud run services update "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--image "${IMAGE}" \
--quiet
- name: Verify ready singleton revision
run: |
SERVICE="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--format=json)"
jq -e '.status.conditions[] | select(.type == "Ready" and .status == "True")' \
<<< "${SERVICE}"
REVISION="$(jq -r \
'[.status.traffic[] | select((.percent // 0) == 100)] |
if length == 1 then .[0].revisionName // empty else empty end' \
<<< "${SERVICE}")"
test -n "${REVISION}"
SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--format='value(spec.containers[0].image)')"
test "${SERVED_IMAGE}" = "${IMAGE}"
@@ -0,0 +1,824 @@
name: Deploy Relay Production Capacity Job
on:
workflow_call:
inputs:
mode:
required: true
type: string
target-cell-id:
required: true
type: string
confirmation:
required: true
type: string
monitor-run-id:
required: true
type: string
monitor-run-attempt:
required: true
type: string
evidence-mode:
required: true
type: string
wave-cell-ids:
required: true
type: string
wave-index:
required: true
type: string
source-wave-run-id:
required: true
type: string
permissions:
actions: read
contents: read
id-token: write
defaults:
run:
working-directory: cloud
jobs:
capacity:
if: ${{ github.ref == 'refs/heads/main' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 75
environment: production
env:
# These drains are unpaced, but restart-safe still waits out the production pace window
# from the drain, the same one the same-cap drain uses.
RESTART_SAFE_PACE_WINDOW_MS: '300000'
GCP_PROJECT_ID: onorca-cloud
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
DIRECTOR_SERVICE_NAME: orca-cloud-relay
DIRECTOR_ORIGIN: https://relay.onorca.dev
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
CAPACITY_CELL_IDS: production-gce-c7,production-gce-c8,production-gce-c9,production-gce-c10,production-gce-c13,production-gce-c14,production-gce-c15,production-gce-c16,production-gce-c19,production-gce-c20,production-gce-c21,production-gce-c22,production-gce-c23,production-gce-c24,production-gce-c25,production-gce-c26
PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d
COMPATIBLE_DIRECTOR_IMAGE_DIGEST: sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73
COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
DEPLOY_MODE: ${{ inputs.mode }}
EVIDENCE_MODE: ${{ inputs.evidence-mode }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }}
WAVE_INDEX: ${{ inputs.wave-index }}
SOURCE_WAVE_RUN_ID: ${{ inputs.source-wave-run-id }}
steps:
- name: Require exact reusable-workflow invocation
working-directory: .
run: |
[[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]]
if test "${EVIDENCE_MODE}" = continuation; then
test "${DEPLOY_MODE}" = apply
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
test "${WAVE_CELL_IDS}" != none
test "${SOURCE_WAVE_RUN_ID}" = none
elif test "${EVIDENCE_MODE}" = resume; then
test "${DEPLOY_MODE}" = apply
test "${WAVE_INDEX}" = resume
test "${WAVE_CELL_IDS}" != none
[[ "${SOURCE_WAVE_RUN_ID}" =~ ^[0-9]+$ ]]
else
test "${EVIDENCE_MODE}" = single
test "${WAVE_CELL_IDS}" = none
test "${WAVE_INDEX}" = 0
test "${SOURCE_WAVE_RUN_ID}" = none
fi
- name: Require production workflow configuration
working-directory: .
env:
DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
CAPACITY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
run: |
test -n "${GCP_REGION}"
test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
test -n "${DEPLOY_SERVICE_ACCOUNT}"
test -n "${CAPACITY_WORKLOAD_IDENTITY_PROVIDER}"
test -n "${CAPACITY_SERVICE_ACCOUNT}"
- uses: actions/checkout@v4
- id: resume-provenance
if: ${{ inputs.evidence-mode == 'resume' }}
env:
GH_TOKEN: ${{ github.token }}
run: |
test "${MONITOR_RUN_ATTEMPT}" = 1
case "${MONITOR_RUN_ID}:${SOURCE_WAVE_RUN_ID}:${WAVE_CELL_IDS}:${TARGET_CELL_ID}" in
31554591366:31555510376:production-gce-c16,production-gce-c15,production-gce-c14,production-gce-c13:production-gce-c13)
EXPECTED_SHA=a917e8e1fc1a2654e8cb81ba39b57733ec56be9c
EXPECTED_SOURCE_ATTEMPT=1
;;
31562760783:31563664692:production-gce-c10,production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c10)
EXPECTED_SHA=6082e9ca89a918ca51f0c87db003f5e8805b64b7
EXPECTED_SOURCE_ATTEMPT=1
;;
31571019947:31572080665:production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c8)
EXPECTED_SHA=e59958130c9d9b7a6cd805df2678d08997842c7c
EXPECTED_SOURCE_ATTEMPT=2
;;
*) exit 1 ;;
esac
MONITOR_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${MONITOR_RUN_ID}" \
--jq 'select(.name == "Monitor Relay Production" and
.path == ".github/workflows/cloud-monitor-relay-production.yml" and
.head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and
.event == "workflow_dispatch" and .conclusion == "success" and .run_attempt == 1) |
.head_sha')"
SOURCE_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \
--jq 'select(.name == "Deploy Relay Production Capacity" and
.path == ".github/workflows/cloud-deploy-relay-production-capacity.yml" and
.head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and
.event == "workflow_dispatch" and .conclusion == "failure") |
.head_sha')"
SOURCE_ATTEMPT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \
--jq '.run_attempt')"
[[ "${MONITOR_SHA}" =~ ^[0-9a-f]{40}$ ]]
test "${MONITOR_SHA}" = "${EXPECTED_SHA}"
test "${SOURCE_SHA}" = "${MONITOR_SHA}"
test "${SOURCE_ATTEMPT}" = "${EXPECTED_SOURCE_ATTEMPT}"
echo "commit-sha=${MONITOR_SHA}" >> "${GITHUB_OUTPUT}"
- name: Require fresh dry-run evidence reference
if: ${{ inputs.mode == 'apply' }}
run: |
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
- name: Download private dry-run evidence
if: ${{ inputs.mode == 'apply' }}
uses: actions/download-artifact@v4
with:
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ runner.temp }}/relay-monitor-evidence
github-token: ${{ github.token }}
run-id: ${{ inputs.monitor-run-id }}
- uses: pnpm/action-setup@v4
with:
package_json_file: cloud/package.json
- uses: actions/setup-node@v4
with:
node-version: 24
- run: pnpm install --frozen-lockfile
- uses: hashicorp/setup-terraform@v3
with:
terraform_wrapper: false
- name: Verify dry-run artifact before cloud authentication
if: ${{ inputs.mode == 'apply' }}
run: |
EVIDENCE_COMMIT_SHA="${GITHUB_SHA}"
if test "${EVIDENCE_MODE:-single}" = resume; then
EVIDENCE_COMMIT_SHA="${{ steps.resume-provenance.outputs.commit-sha }}"
fi
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
--run-id "${MONITOR_RUN_ID}" \
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
--commit-sha "${EVIDENCE_COMMIT_SHA}" \
--mode dry-run
- name: Reject previously consumed dry-run evidence
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
env:
GH_TOKEN: ${{ github.token }}
run: |
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
COUNT="$(gh api \
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
--jq '.total_count')"
test "${COUNT}" = "0"
- name: Download this workflow's wave authority
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
uses: actions/download-artifact@v4
with:
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ runner.temp }}/relay-wave-authority
github-token: ${{ github.token }}
run-id: ${{ github.run_id }}
- name: Download the failed wave authority for resume
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
uses: actions/download-artifact@v4
with:
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ runner.temp }}/relay-wave-authority
github-token: ${{ github.token }}
run-id: ${{ inputs.source-wave-run-id }}
- name: Require wave evidence consumed by this workflow
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
run: |
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${GITHUB_RUN_ID}"
- name: Require wave evidence consumed by the failed source workflow
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
run: |
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${SOURCE_WAVE_RUN_ID}"
- id: deploy-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
release: 'false'
- name: Require exact mutation confirmation
if: ${{ inputs.mode != 'verify' }}
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: |
if test "${EVIDENCE_MODE:-single}" = resume; then
test "${CONFIRMATION}" = "RESUME_SELECTED_CELL_TO_1000 ${TARGET_CELL_ID}"
elif test "${DEPLOY_MODE}" = apply; then
test "${CONFIRMATION}" = "RAISE_SELECTED_CELL_TO_1000"
else
test "${CONFIRMATION}" = "ROLL_BACK_SELECTED_CELL_TO_600 ${TARGET_CELL_ID}"
fi
- name: Initialize the exact production backend
run: node dev/scripts/infra.mjs init --env production
- name: Build the exact selected-cell configuration
shell: bash
run: |
if test "${DEPLOY_MODE}" = rollback; then
TARGET_HARD_CAP=600
else
TARGET_HARD_CAP=1000
fi
TARGET_UNOBSERVED_BOUND=60
TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"
[[ "${TARGET_HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]]
CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev"
CELLS_JSON="$(terraform -chdir=infra/terraform console \
-var-file=environments/production.tfvars \
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
OVERRIDE_CELLS_JSON="$(jq -ce \
--arg cell "${TARGET_CELL_ID}" \
--argjson cap "${TARGET_HARD_CAP}" \
--argjson bound "${TARGET_UNOBSERVED_BOUND}" \
'.[$cell].connection_hard_cap = $cap |
.[$cell].connection_unobserved_bound = $bound' \
<<< "${CELLS_JSON}")"
jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-capacity.tfvars.json"
BASE_CELLS_JSON="$(terraform -chdir=infra/terraform console \
-var-file=environments/production.tfvars \
<<< 'local.relay_director_cells_json' | jq -er '.')"
IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image"
ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone"
DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \
-var-file=environments/production.tfvars \
-var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
<<< "${IMAGE_EXPRESSION}" | jq -r '.')"
TARGET_ZONE="$(terraform -chdir=infra/terraform console \
-var-file=environments/production.tfvars \
-var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
<<< "${ZONE_EXPRESSION}" | jq -r '.')"
MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
| jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
jq -e --arg cell "${TARGET_CELL_ID}" \
'any(.[]; .id == $cell and .connectionHardCap == 1000 and
.connectionUnobservedBound == 60)' \
<<< "${BASE_CELLS_JSON}" >/dev/null
[[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]]
DESIRED_IMAGE_DIGEST="${DESIRED_IMAGE##*@}"
[[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]]
test "${MIG_NAME}" = "orca-cloud-relay-gce-${TARGET_HOSTNAME}"
{
echo "CELL_ORIGIN=${CELL_ORIGIN}"
echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}"
echo "TARGET_HARD_CAP=${TARGET_HARD_CAP}"
echo "TARGET_UNOBSERVED_BOUND=${TARGET_UNOBSERVED_BOUND}"
echo "DESIRED_IMAGE=${DESIRED_IMAGE}"
echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}"
echo "TARGET_ZONE=${TARGET_ZONE}"
echo "MIG_NAME=${MIG_NAME}"
echo "BASE_CELLS_JSON=${BASE_CELLS_JSON}"
} >> "${GITHUB_ENV}"
- name: Require the exact compatible production image and topology
shell: bash
run: |
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
ACTIVE_REVISION="$(jq -r \
'[.status.traffic[] | select((.percent // 0) > 0)] |
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \
<<< "${SERVICE_JSON}")"
test -n "${ACTIVE_REVISION}"
ACTIVE_REVISION_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
ACTIVE_IMAGE="$(jq -er '.spec.containers[0].image' <<< "${ACTIVE_REVISION_JSON}")"
ACTIVE_IMAGE_DIGEST="${ACTIVE_IMAGE##*@}"
if test "${ACTIVE_IMAGE}" != "${DESIRED_IMAGE}"; then
test "${ACTIVE_IMAGE_DIGEST}" = "${COMPATIBLE_DIRECTOR_IMAGE_DIGEST}"
test "${DESIRED_IMAGE_DIGEST}" = "${COMPATIBLE_CELL_IMAGE_DIGEST}"
fi
CURRENT_CELLS_JSON="$(jq -cer '[.spec.containers[0].env[]? |
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
if length == 1 then .[0] | fromjson else error("missing director topology") end' \
<<< "${ACTIVE_REVISION_JSON}")"
CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON="$(
node dev/scripts/read-relay-production-capacity-identity.mjs \
<<< "${ACTIVE_REVISION_JSON}"
)"
CLASSIFICATION="$(jq -nc \
--argjson baseCells "${BASE_CELLS_JSON}" \
--argjson currentCells "${CURRENT_CELLS_JSON}" \
--arg capacityCellIds "${CAPACITY_CELL_IDS}" \
--arg targetCellId "${TARGET_CELL_ID}" \
--argjson targetHardCap "${TARGET_HARD_CAP}" \
--argjson currentCapacityServiceAccount \
"${CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON}" \
'{baseCells:$baseCells, currentCells:$currentCells,
capacityCellIds:($capacityCellIds | split(",")),
targetCellId:$targetCellId, targetHardCap:$targetHardCap,
currentCapacityServiceAccount:$currentCapacityServiceAccount}' \
| node dev/scripts/classify-relay-production-capacity-director.mjs \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
TOPOLOGY_PHASE="$(jq -er '.topologyPhase' <<< "${CLASSIFICATION}")"
DESIRED_CELLS_JSON="$(jq -cer '.desiredCells' <<< "${CLASSIFICATION}")"
DIRECTOR_READY="$(jq -er \
'if (.directorReady | type) == "boolean" then
(.directorReady | tostring)
else error("invalid directorReady classification") end' \
<<< "${CLASSIFICATION}")"
{
echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}"
echo "TOPOLOGY_PHASE=${TOPOLOGY_PHASE}"
echo "DIRECTOR_READY=${DIRECTOR_READY}"
echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}"
} >> "${GITHUB_ENV}"
- name: Require the exact wave predecessor topology
if: ${{ inputs.mode == 'apply' && (inputs.evidence-mode == 'continuation' || inputs.evidence-mode == 'resume') }}
run: test "${TOPOLOGY_PHASE}" = predecessor
- name: Verify fresh dry-run evidence against the live selector
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
--run-id "${MONITOR_RUN_ID}" \
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
--commit-sha "${GITHUB_SHA}" \
--mode dry-run \
--mutation-mode capacity-transition \
--source-cell-id "${TARGET_CELL_ID}" \
--director-origin "${DIRECTOR_ORIGIN}"
- name: Recheck every live safety signal
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
pnpm incident:relay-preflight -- \
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
- name: Recheck exact wave state and every live safety signal
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
node dev/scripts/relay-production-capacity-wave.mjs build-preflight \
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
--wave-cell-ids "${WAVE_CELL_IDS}" \
--wave-index "${WAVE_INDEX}" \
--target-cell-id "${TARGET_CELL_ID}" \
--output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json"
RETRY_ARGS=()
if test "${WAVE_INDEX}" != 0; then RETRY_ARGS=(--retry-freshness); fi
pnpm incident:relay-preflight -- \
--state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \
"${RETRY_ARGS[@]}"
- name: Recheck exact isolated resume state and every live safety signal
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
node dev/scripts/relay-production-capacity-wave.mjs build-resume-preflight \
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
--wave-cell-ids "${WAVE_CELL_IDS}" \
--target-cell-id "${TARGET_CELL_ID}" \
--output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json"
pnpm incident:relay-preflight -- \
--state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \
--retry-freshness
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap 600 \
--unobserved-bound 60 \
--heartbeat fresh \
--admission migration-only \
--draining required \
--activity allowed \
--runtime required \
--expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}"
- name: Consume the single-use dry-run evidence
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
run: |
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
printf '%s\n' "${GITHUB_RUN_ID}" \
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
- name: Publish the consumed-evidence marker
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
uses: actions/upload-artifact@v4
with:
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
retention-days: 90
if-no-files-found: error
- name: Verify current selected-cell capacity
if: ${{ inputs.mode == 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
CURRENT_CAP="${TARGET_HARD_CAP}"
CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
if test "${TOPOLOGY_PHASE}" = predecessor; then
CURRENT_CAP=600
CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${CURRENT_CAP}" \
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
--heartbeat fresh \
--admission general \
--draining forbidden \
--activity allowed \
--expected-image-digests "${CURRENT_IMAGE_DIGEST}"
- name: Arm fail-closed mutation cleanup
if: ${{ inputs.mode != 'verify' }}
run: echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
- name: Reversibly isolate only the selected cell
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
test "${MUTATION_STARTED:-false}" = true || exit 0
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--mode isolate
- name: Drain the selected cell or prove an offline rollback
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
if node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--mode drain; then
echo "OFFLINE_ROLLBACK=false" >> "${GITHUB_ENV}"
elif test "${DEPLOY_MODE}" = rollback; then
echo "OFFLINE_ROLLBACK=true" >> "${GITHUB_ENV}"
else
exit 1
fi
- id: restart-auth-one
if: ${{ inputs.mode != 'verify' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- id: restart-gate-one
name: Require restart-safe selected-cell activity
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-one.outputs.id_token }}
run: |
if test "${OFFLINE_ROLLBACK:-false}" = true; then
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--heartbeat stale \
--admission migration-only \
--draining either \
--activity restart-safe \
--runtime unavailable
echo "settled=true" >> "${GITHUB_OUTPUT}"
exit 0
fi
CURRENT_CAP=600
if test "${TOPOLOGY_PHASE}" = desired; then
CURRENT_CAP="${TARGET_HARD_CAP}"
elif test "${TARGET_HARD_CAP}" = 600; then
CURRENT_CAP=1000
fi
GATE_LOG="${RUNNER_TEMP}/relay-capacity-restart-gate-one.log"
set +e
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${CURRENT_CAP}" \
--unobserved-bound 60 \
--heartbeat either \
--admission migration-only \
--draining required \
--activity restart-safe \
--runtime required \
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
--timeout-ms 450000 \
--expected-image-digests \
"${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" \
2> "${GATE_LOG}"
GATE_EXIT=$?
set -e
cat "${GATE_LOG}" >&2
if test "${GATE_EXIT}" = 0; then
echo "settled=true" >> "${GITHUB_OUTPUT}"
exit 0
fi
if test "$(wc -l < "${GATE_LOG}" | tr -d ' ')" = 1 &&
grep -Eq '^capacity transition verification timed out: \{.*\}$' "${GATE_LOG}"; then
echo "settled=false" >> "${GITHUB_OUTPUT}"
exit 0
fi
exit "${GATE_EXIT}"
- id: restart-auth-two
if: ${{ steps.restart-gate-one.outputs.settled == 'false' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Require extended restart-safe selected-cell activity
if: ${{ steps.restart-gate-one.outputs.settled == 'false' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-two.outputs.id_token }}
run: |
CURRENT_CAP=600
if test "${TOPOLOGY_PHASE}" = desired; then
CURRENT_CAP="${TARGET_HARD_CAP}"
elif test "${TARGET_HARD_CAP}" = 600; then
CURRENT_CAP=1000
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${CURRENT_CAP}" \
--unobserved-bound 60 \
--heartbeat either \
--admission migration-only \
--draining required \
--activity restart-safe \
--runtime required \
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
--timeout-ms 450000 \
--expected-image-digests \
"${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
- name: Deploy only the reviewed director topology
if: ${{ inputs.mode != 'verify' }}
run: |
if test "${DIRECTOR_READY}" = true; then exit 0; fi
RELEASE_ID="capacity-${TARGET_HOSTNAME}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
node dev/scripts/deploy-relay-blue-green.mjs \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--service "${DIRECTOR_SERVICE_NAME}" \
--image "${ACTIVE_IMAGE}" \
--role director \
--max-instances 5 \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--capacity-cell-id "${TARGET_CELL_ID}" \
--director-cells-json "${DESIRED_CELLS_JSON}" \
--min-instances 5 \
--prune-revisions false \
--release-id "${RELEASE_ID}"
- id: director-transition-auth
if: ${{ inputs.mode != 'verify' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Require fail-closed director transition
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.director-transition-auth.outputs.id_token }}
run: |
if test "${OFFLINE_ROLLBACK:-false}" = true; then
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--heartbeat stale \
--admission migration-only \
--draining either \
--activity restart-safe \
--runtime unavailable
exit 0
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${TARGET_HARD_CAP}" \
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
--heartbeat either \
--admission migration-only \
--draining required \
--activity restart-safe \
--runtime required \
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
--timeout-ms 480000 \
--expected-image-digests \
"${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
- id: capacity-auth
if: ${{ inputs.mode != 'verify' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Plan and apply only the empty selected cell
if: ${{ inputs.mode != 'verify' }}
shell: bash
run: |
terraform -chdir=infra/terraform plan \
-var-file=environments/production.tfvars \
-var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
-out="${RUNNER_TEMP}/relay-capacity-cell.tfplan"
PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-capacity-cell.tfplan" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode bootstrap-cell \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${TARGET_HARD_CAP}" \
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
--image "${DESIRED_IMAGE}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
echo "${PLAN_RESULT}"
PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")"
[[ "${PLAN_CHANGES}" =~ ^(0|1|2)$ ]]
if test "${PLAN_CHANGES}" != 0; then
terraform -chdir=infra/terraform apply \
-auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan"
else
INSTANCE="$(gcloud compute instance-groups managed list-instances \
"${MIG_NAME}" --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
--format=json | jq -er 'if length == 1 and
.[0].instanceStatus == "RUNNING" and .[0].currentAction == "NONE"
then .[0].instance | split("/") | last
else error("selected cell is not one stable running instance") end')"
gcloud compute instance-groups managed recreate-instances \
"${MIG_NAME}" --instances "${INSTANCE}" \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --quiet
fi
gcloud compute instance-groups managed wait-until \
"${MIG_NAME}" --stable --project "${GCP_PROJECT_ID}" \
--zone "${TARGET_ZONE}" --timeout 900
- id: capacity-transition-auth
if: ${{ inputs.mode != 'verify' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Verify fresh exact selected-cell heartbeat before admission
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
run: |
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${TARGET_HARD_CAP}" \
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
--heartbeat fresh \
--admission migration-only \
--draining forbidden \
--activity allowed \
--expected-image-digests "${DESIRED_IMAGE_DIGEST}"
- name: Restore only the selected cell to general admission
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
run: |
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--mode activate
- name: Verify the live general selected cell
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
run: |
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${TARGET_HARD_CAP}" \
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
--heartbeat fresh \
--admission general \
--draining forbidden \
--activity allowed \
--expected-image-digests "${DESIRED_IMAGE_DIGEST}"
- id: cleanup-auth
if: ${{ failure() && inputs.mode != 'verify' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Keep the selected cell isolated after a failed mutation
if: ${{ failure() && inputs.mode != 'verify' }}
continue-on-error: true
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }}
run: |
test "${MUTATION_STARTED:-false}" = true || exit 0
CLEANUP_STATUS=0
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--mode isolate || CLEANUP_STATUS=$?
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--mode drain || CLEANUP_STATUS=$?
exit "${CLEANUP_STATUS}"
@@ -0,0 +1,355 @@
name: Deploy Relay Production Capacity
on:
workflow_dispatch:
inputs:
mode:
description: Verify, change one cell, or raise a sequential wave
required: true
default: verify
type: choice
options:
- verify
- apply
- rollback
- wave-apply
- wave-resume
target-cell-id:
description: Exact serving cell for verify, apply, or rollback
required: true
default: production-gce-c26
type: choice
options:
- production-gce-c7
- production-gce-c8
- production-gce-c9
- production-gce-c10
- production-gce-c13
- production-gce-c14
- production-gce-c15
- production-gce-c16
- production-gce-c19
- production-gce-c20
- production-gce-c21
- production-gce-c22
- production-gce-c23
- production-gce-c24
- production-gce-c25
- production-gce-c26
wave-cell-ids:
description: Ordered comma-separated wave of two to four serving cells
required: false
default: none
type: string
confirmation:
description: Enter the exact single-cell or wave confirmation
required: false
type: string
monitor-run-id:
description: Successful fresh dry-run monitor workflow run ID for apply
required: false
type: string
monitor-run-attempt:
description: Exact dry-run monitor workflow attempt for apply
required: false
type: string
source-wave-run-id:
description: Failed wave run that isolated the resume target
required: false
type: string
permissions:
actions: read
contents: read
id-token: write
concurrency:
group: production-cloud-sql-rollout
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
single_cell:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode != 'wave-apply' && inputs.mode != 'wave-resume') }}
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
with:
mode: ${{ inputs.mode }}
target-cell-id: ${{ inputs.target-cell-id }}
confirmation: ${{ inputs.confirmation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
evidence-mode: single
wave-cell-ids: none
wave-index: '0'
source-wave-run-id: none
secrets: inherit
resume_cell:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-resume' && github.ref == 'refs/heads/main') }}
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
with:
mode: apply
target-cell-id: ${{ inputs.target-cell-id }}
confirmation: ${{ inputs.confirmation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
evidence-mode: resume
wave-cell-ids: ${{ inputs.wave-cell-ids }}
wave-index: resume
source-wave-run-id: ${{ inputs.source-wave-run-id }}
secrets: inherit
wave_gate:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-apply' && github.ref == 'refs/heads/main') }}
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 30
environment: production
outputs:
cells: ${{ steps.wave.outputs.cells }}
env:
DIRECTOR_ORIGIN: https://relay.onorca.dev
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d
COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f
WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }}
steps:
- name: Require production workflow configuration
working-directory: .
env:
DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
run: |
test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
test -n "${DEPLOY_SERVICE_ACCOUNT}"
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
package_json_file: cloud/package.json
- uses: actions/setup-node@v4
with:
node-version: 24
- run: pnpm install --frozen-lockfile
- id: wave
name: Validate the exact wave request
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: |
CELLS="$(node dev/scripts/relay-production-capacity-wave.mjs validate \
--wave-cell-ids "${WAVE_CELL_IDS}" \
--confirmation "${CONFIRMATION}")"
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
- name: Require fresh dry-run evidence reference
run: |
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
- name: Download private dry-run evidence
uses: actions/download-artifact@v4
with:
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ github.workspace }}/relay-monitor-evidence
github-token: ${{ github.token }}
run-id: ${{ inputs.monitor-run-id }}
- name: Verify dry-run artifact before cloud authentication
run: |
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
--directory "${OUTPUT_DIRECTORY}" \
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
--run-id "${MONITOR_RUN_ID}" \
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
--commit-sha "${GITHUB_SHA}" \
--mode dry-run
- name: Reject previously consumed dry-run evidence
env:
GH_TOKEN: ${{ github.token }}
run: |
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
COUNT="$(gh api \
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
--jq '.total_count')"
test "${COUNT}" = "0"
- id: deploy-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
release: 'false'
- name: Verify wave evidence against the live selector
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
FIRST_CELL="$(jq -er '.[0]' <<< '${{ steps.wave.outputs.cells }}')"
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
--directory "${OUTPUT_DIRECTORY}" \
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
--run-id "${MONITOR_RUN_ID}" \
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
--commit-sha "${GITHUB_SHA}" \
--mode dry-run \
--mutation-mode capacity-transition \
--source-cell-id "${FIRST_CELL}" \
--director-origin "${DIRECTOR_ORIGIN}"
- name: Recheck every live safety signal
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
pnpm incident:relay-preflight -- \
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json"
- name: Require exact 600/60 predecessor wave cells
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
while read -r CELL_ID; do
HOSTNAME="${CELL_ID#production-gce-}"
[[ "${HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]]
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "https://${HOSTNAME}.relay.onorca.dev" \
--cell-id "${CELL_ID}" \
--hard-cap 600 \
--unobserved-bound 60 \
--heartbeat fresh \
--admission general \
--draining forbidden \
--activity allowed \
--expected-image-digests \
"${PREDECESSOR_IMAGE_DIGEST},${COMPATIBLE_CELL_IMAGE_DIGEST}"
done < <(jq -r '.[]' <<< '${{ steps.wave.outputs.cells }}')
- name: Consume the single-use dry-run evidence
run: |
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
printf '%s\n' "${GITHUB_RUN_ID}" \
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
- name: Publish the consumed-evidence marker
uses: actions/upload-artifact@v4
with:
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
retention-days: 90
if-no-files-found: error
wave_cell_1:
needs: wave_gate
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
with:
mode: apply
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[0] }}
confirmation: RAISE_SELECTED_CELL_TO_1000
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
evidence-mode: continuation
wave-cell-ids: ${{ inputs.wave-cell-ids }}
wave-index: '0'
source-wave-run-id: none
secrets: inherit
wave_cell_2:
needs: [wave_gate, wave_cell_1]
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
with:
mode: apply
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[1] }}
confirmation: RAISE_SELECTED_CELL_TO_1000
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
evidence-mode: continuation
wave-cell-ids: ${{ inputs.wave-cell-ids }}
wave-index: '1'
source-wave-run-id: none
secrets: inherit
wave_cell_3:
if: ${{ needs.wave_cell_2.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[2] != null }}
needs: [wave_gate, wave_cell_2]
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
with:
mode: apply
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[2] }}
confirmation: RAISE_SELECTED_CELL_TO_1000
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
evidence-mode: continuation
wave-cell-ids: ${{ inputs.wave-cell-ids }}
wave-index: '2'
source-wave-run-id: none
secrets: inherit
wave_cell_4:
if: ${{ needs.wave_cell_3.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[3] != null }}
needs: [wave_gate, wave_cell_3]
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
with:
mode: apply
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[3] }}
confirmation: RAISE_SELECTED_CELL_TO_1000
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
evidence-mode: continuation
wave-cell-ids: ${{ inputs.wave-cell-ids }}
wave-index: '3'
source-wave-run-id: none
secrets: inherit
# Every wave job re-enters the run's lease with release: 'false'; only this job frees it.
release_lease:
if: always()
needs:
- single_cell
- resume_cell
- wave_gate
- wave_cell_1
- wave_cell_2
- wave_cell_3
- wave_cell_4
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
environment: production
steps:
- uses: actions/checkout@v4
- uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
release: 'true'
@@ -0,0 +1,310 @@
name: Deploy Relay Production Director
on:
workflow_dispatch:
inputs:
image-digest:
description: 'Immutable relay image digest (sha256: plus 64 lowercase hex characters)'
required: true
type: string
regional-placement-mode:
description: Preserve the live switch, explicitly enable Asia preference, or force US-first
required: true
default: preserve
type: choice
options: [preserve, enable, disable]
region-correction-cohort-percent:
description: 'Preserve the measured-correction cohort, or set an integer 0–100; durable rehome stays disabled'
required: true
default: preserve
type: string
reserve-placement:
description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve
required: true
default: preserve
type: choice
options: [preserve, 'off', dry-run, 'on']
shadow-seat-feed-cells:
description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused
required: true
default: preserve
type: string
prune-incompatible-revisions:
description: Retain only the newly verified serving and rollback revisions
required: true
default: false
type: boolean
confirmation:
description: Enter the exact confirmation required by a destructive option
required: false
type: string
expected-rehome-generation:
description: Exact durable regional-rehome generation; it must remain disabled
required: true
type: string
bootstrap-runtime-identity:
description: One-time move from the stamped-cell identity to the director identity
required: true
default: false
type: boolean
predecessor-image-digest:
description: Exact immutable serving predecessor digest for the one-time identity bootstrap
required: true
type: string
shape-baseline-revision:
description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts
required: false
type: string
permissions:
contents: read
id-token: write
# Director updates and candidate operations both mutate production relay control state.
concurrency:
group: production-cloud-sql-rollout
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
deploy:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: production
env:
GCP_PROJECT_ID: onorca-cloud
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
DIRECTOR_SERVICE_NAME: orca-cloud-relay
IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
IMAGE_DIGEST: ${{ inputs.image-digest }}
REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }}
REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }}
RESERVE_PLACEMENT: ${{ inputs.reserve-placement }}
SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }}
PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }}
# Floor the served revision must keep, matching relay_min_instances in
# environments/production.tfvars. This gate only fails a bad deploy; Terraform
# still owns the value, and the candidate inherits it from the serving revision.
DIRECTOR_MIN_INSTANCES: 5
DIRECTOR_MAX_INSTANCES: 5
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
PREDECESSOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_RUNTIME_SERVICE_ACCOUNT }}
REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain
EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }}
PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }}
SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }}
steps:
- uses: actions/checkout@v4
- id: google-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
- uses: actions/setup-node@v4
with:
node-version: 24
- name: Resolve immutable production image
shell: bash
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: |
if [[ ! "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
echo "image-digest must be an immutable lowercase sha256 digest" >&2
exit 1
fi
if test "${REGION_CORRECTION_COHORT_PERCENT}" != preserve; then
[[ "${REGION_CORRECTION_COHORT_PERCENT}" =~ ^([0-9]|[1-9][0-9]|100)$ ]]
fi
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
[[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]]
[[ "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]]
[[ "${BOOTSTRAP_RUNTIME_IDENTITY}" =~ ^(true|false)$ ]]
if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then
[[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false
test "${REGIONAL_PLACEMENT_MODE}" = preserve
test -z "${SHAPE_BASELINE_REVISION}"
test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY
elif test -n "${SHAPE_BASELINE_REVISION}"; then
[[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-[0-9]{5}-[a-z0-9]{3}$ ]]
test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false
test "${REGIONAL_PLACEMENT_MODE}" = preserve
test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE
elif test "${PRUNE_INCOMPATIBLE_REVISIONS}" = true; then
test "${REGIONAL_PLACEMENT_MODE}" = preserve
test "${CONFIRMATION}" = PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS
elif test "${REGIONAL_PLACEMENT_MODE}" = disable; then
test "${CONFIRMATION}" = FORCE_RELAY_US_FIRST
else
test -z "${CONFIRMATION}"
fi
echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
# Why: the deploy INHERITS the serving revision's floor, so when that revision has
# already lost it the candidate inherits zero, the in-script gate compares zero against
# zero and passes, and the post-deploy check below only notices after traffic moved.
# The documented rollback target is created at minimum instances zero, so promoting it
# arms exactly that. Refuse to inherit a degraded floor rather than latch it.
- name: Require a healthy serving floor before deploying
shell: bash
run: |
SERVING="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test -n "${SERVING}"
FLOOR="$(gcloud run revisions describe "${SERVING}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
if [[ "${FLOOR:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then
echo "serving revision ${SERVING} holds ${FLOOR:-0} minimum instances," \
"below ${DIRECTOR_MIN_INSTANCES}; deploying would inherit and latch it." >&2
echo "Restore the floor first: gcloud run services update ${DIRECTOR_SERVICE_NAME}" \
"--min-instances=${DIRECTOR_MIN_INSTANCES}" >&2
exit 1
fi
CEILING="$(gcloud run revisions describe "${SERVING}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
test "${CEILING}" = "${DIRECTOR_MAX_INSTANCES}"
echo "serving revision ${SERVING} holds ${FLOOR} minimum instances"
echo "SERVING_REVISION=${SERVING}" >> "${GITHUB_ENV}"
# Why: no --min-instances here. The candidate inherits the Terraform-owned
# scaling, and this step ends with 100% traffic on it. Pinning 1 rebuilt the
# per-instance admission shortage that took placement failures to ~70%.
- name: Deploy director blue/green
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
served_version="$(gcloud run revisions describe "${SERVING_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '[.spec.containers[0].env[]? |
select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
(.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
(.version // .key // empty)] |
if length == 1 then .[0] else empty end')"
if [[ "${served_version}" =~ ^[1-9][0-9]*$ ]]; then
current_version="${served_version}"
else
test "${REGIONAL_PLACEMENT_MODE}" = preserve
current_version="$(gcloud secrets versions describe latest \
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \
--format='value(name)' | awk -F/ '{print $NF}')"
[[ "${current_version}" =~ ^[1-9][0-9]*$ ]]
fi
current="$(gcloud secrets versions access "${current_version}" \
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")"
[[ "${current}" =~ ^(true|false)$ ]]
case "${REGIONAL_PLACEMENT_MODE}" in
preserve) desired="${current}" ;;
enable) desired=true ;;
disable) desired=false ;;
*) echo "regional-placement-mode is invalid" >&2; exit 1 ;;
esac
if test "${current}" != "${desired}"; then
target_version="$(printf '%s' "${desired}" | gcloud secrets versions add \
"${REGIONAL_PLACEMENT_SECRET}" --project "${GCP_PROJECT_ID}" --data-file=- \
--format='value(name)' --quiet | awk -F/ '{print $NF}')"
else
target_version="${current_version}"
fi
[[ "${target_version}" =~ ^[1-9][0-9]*$ ]]
RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
shape_args=()
if test -n "${SHAPE_BASELINE_REVISION}"; then
shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}")
fi
node dev/scripts/deploy-relay-blue-green.mjs \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--service "${DIRECTOR_SERVICE_NAME}" \
--image "${IMAGE}" \
--role director \
"${shape_args[@]}" \
--runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" \
--bootstrap-runtime-identity "${BOOTSTRAP_RUNTIME_IDENTITY}" \
--predecessor-image-digest "${PREDECESSOR_IMAGE_DIGEST}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience "${REHOME_AUDIENCE}" \
--rehome-control-origin https://relay.onorca.dev \
--admin-audience https://relay.onorca.dev/v1/admin/drain \
--expected-rehome-generation "${EXPECTED_REHOME_GENERATION}" \
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
--prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \
--release-id "${RELEASE_ID}" \
--regional-placement-secret-version "${target_version}" \
--region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}" \
--reserve-placement "${RESERVE_PLACEMENT}" \
--shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}"
echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}"
echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}"
- name: Verify served revision and native health
shell: bash
run: |
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--format=json)"
REVISION="$(jq -r '[.status.traffic[] | select((.percent // 0) > 0)] | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' <<< "${SERVICE_JSON}")"
test -n "${REVISION}"
SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--format='value(spec.containers[0].image)')"
test "${SERVED_IMAGE}" = "${IMAGE}"
SERVED_REGIONAL_PLACEMENT_SECRET="$(gcloud run revisions describe "${REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -cer '[.spec.containers[0].env[] |
select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
(.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
{secret: (.secret // .name), version: (.version // .key)}] |
if length == 1 then .[0] else error("regional placement secret missing") end')"
test "$(jq -r '.secret' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \
"${REGIONAL_PLACEMENT_SECRET}"
test "$(jq -r '.version' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \
"${REGIONAL_PLACEMENT_VERSION}"
test "$(gcloud secrets versions access "${REGIONAL_PLACEMENT_VERSION}" --project "${GCP_PROJECT_ID}" \
--secret "${REGIONAL_PLACEMENT_SECRET}")" = "${REGIONAL_PLACEMENT_ENABLED}"
# Why: a served revision with no warm-instance floor still passes health and digest
# checks while quietly shrinking per-instance admission capacity.
SERVED_MIN_INSTANCES="$(gcloud run revisions describe "${REVISION}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
if [[ "${SERVED_MIN_INSTANCES:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then
echo "served revision ${REVISION} holds ${SERVED_MIN_INSTANCES:-0} minimum instances, expected at least ${DIRECTOR_MIN_INSTANCES}" >&2
exit 1
fi
SERVED_MAX_INSTANCES="$(gcloud run revisions describe "${REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
test "${SERVED_MAX_INSTANCES}" = "${DIRECTOR_MAX_INSTANCES}"
SERVICE_URL="$(jq -r '.status.url' <<< "${SERVICE_JSON}")"
node dev/scripts/smoke-relay.mjs "${SERVICE_URL}"
@@ -0,0 +1,499 @@
name: Deploy Relay Production Multi-Target
on:
workflow_dispatch:
inputs:
source-cell-id:
description: Existing Terraform source cell ID
required: true
type: string
target-cell-ids:
description: Comma-separated distinct Terraform target cell IDs
required: true
type: string
general-cell-ids:
description: Comma-separated proven cells that remain eligible for ordinary placement
required: false
type: string
unobserved-connection-bound:
description: Exact worst-case unobserved connection bound proven by the passing load gate
required: false
type: string
failed-target-cell-id:
description: Registered failed target to fence and supersede
required: false
type: string
replacement-target-cell-id:
description: Healthy replacement for registered failed target
required: false
type: string
mode:
description: Preflight/audit are read-only; other modes mutate production
required: true
default: preflight
type: choice
options:
- audit
- preflight
- cutover-admission
- add-migration-cells
- promote-general-cell
- retire-migration-cell
- execute
- recover-forward
- fence-source
- supersede-target
confirmation:
description: Enter CUTOVER_SELECTOR, ADD_MIGRATION_CELLS, PROMOTE_GENERAL_CELL, RETIRE_MIGRATION_CELL, EVACUATE_MULTI, RECOVER_FORWARD, or FENCE_SOURCE
required: false
type: string
selector-attempt-id:
description: Exact durable selector attempt ID for admission mutations
required: false
type: string
monitor-run-id:
description: Successful fresh dry-run monitor workflow run ID
required: false
type: string
monitor-run-attempt:
description: Exact dry-run monitor workflow attempt
required: false
type: string
broker-operation-id:
description: Stable durable broker operation ID for target supersession
required: false
type: string
completed-fence-attempt-id:
description: Exact older completed fence attempt to recover without replay
required: false
type: string
completed-fence-commit:
description: Exact older fence commit bound to the completed attempt
required: false
type: string
completed-fence-operation:
description: Exact DONE Compute resize operation to adopt
required: false
type: string
completed-fence-state-serial:
description: Exact Terraform serial before the completed fence
required: false
type: string
completed-fence-plan-generation:
description: Exact saved-plan object generation
required: false
type: string
completed-fence-state-generation:
description: Exact current Terraform state object generation
required: false
type: string
completed-fence-state-sha256:
description: Exact current Terraform state object SHA-256
required: false
type: string
expected-lease-generation:
description: Exact live lease generation authorized for conditional takeover
required: false
type: string
expected-lease-operation-id:
description: Exact live lease operation ID authorized for takeover
required: false
type: string
expected-lease-request-digest:
description: Exact live lease request digest authorized for takeover
required: false
type: string
permissions:
actions: read
contents: read
id-token: write
concurrency:
group: production-cloud-sql-rollout
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
deploy:
if: >-
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
github.ref == 'refs/heads/main' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: production
env:
GCP_PROJECT_ID: onorca-cloud
DIRECTOR_ORIGIN: https://relay.onorca.dev
ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain
SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
TARGET_CELL_IDS: ${{ inputs.target-cell-ids }}
GENERAL_CELL_IDS: ${{ inputs.general-cell-ids }}
UNOBSERVED_CONNECTION_BOUND: ${{ inputs.unobserved-connection-bound }}
FAILED_TARGET_CELL_ID: ${{ inputs.failed-target-cell-id }}
REPLACEMENT_TARGET_CELL_ID: ${{ inputs.replacement-target-cell-id }}
DEPLOY_MODE: ${{ inputs.mode }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }}
BROKER_OPERATION_ID: ${{ inputs.broker-operation-id }}
COMPLETED_FENCE_ATTEMPT_ID: ${{ inputs.completed-fence-attempt-id }}
COMPLETED_FENCE_COMMIT: ${{ inputs.completed-fence-commit }}
COMPLETED_FENCE_OPERATION: ${{ inputs.completed-fence-operation }}
COMPLETED_FENCE_STATE_SERIAL: ${{ inputs.completed-fence-state-serial }}
COMPLETED_FENCE_PLAN_GENERATION: ${{ inputs.completed-fence-plan-generation }}
COMPLETED_FENCE_STATE_GENERATION: ${{ inputs.completed-fence-state-generation }}
COMPLETED_FENCE_STATE_SHA256: ${{ inputs.completed-fence-state-sha256 }}
EXPECTED_LEASE_GENERATION: ${{ inputs.expected-lease-generation }}
EXPECTED_LEASE_OPERATION_ID: ${{ inputs.expected-lease-operation-id }}
EXPECTED_LEASE_REQUEST_DIGEST: ${{ inputs.expected-lease-request-digest }}
steps:
- uses: actions/checkout@v4
- name: Require private fence-broker environment
if: >-
${{ inputs.mode == 'fence-source' ||
inputs.mode == 'supersede-target' }}
env:
FENCE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }}
FENCE_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }}
FENCE_BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
run: |
test -n "${FENCE_WORKLOAD_IDENTITY_PROVIDER}"
test -n "${FENCE_SERVICE_ACCOUNT}"
test -n "${FENCE_BROKER_URI}"
- name: Reject direct-runner Terraform fence aborts
if: ${{ inputs.mode == 'abort-fence-source' }}
run: |
echo "Terraform fence aborts require a reviewed private-broker recovery path." >&2
exit 1
- name: Require fresh dry-run evidence reference
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
run: |
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
- name: Download private dry-run evidence
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
uses: actions/download-artifact@v4
with:
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ runner.temp }}/relay-monitor-evidence
github-token: ${{ github.token }}
run-id: ${{ inputs.monitor-run-id }}
- uses: pnpm/action-setup@v4
with:
package_json_file: cloud/package.json
- uses: actions/setup-node@v4
with:
node-version: 24
- run: pnpm install --frozen-lockfile
- uses: hashicorp/setup-terraform@v3
with:
terraform_wrapper: false
- name: Verify dry-run artifact before cloud authentication
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
run: |
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
--run-id "${MONITOR_RUN_ID}" \
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
--commit-sha "${GITHUB_SHA}" \
--mode dry-run
- name: Reject previously consumed dry-run evidence
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
env:
GH_TOKEN: ${{ github.token }}
run: |
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
COUNT="$(gh api \
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
--jq '.total_count')"
test "${COUNT}" = "0"
- id: google-auth
if: ${{ inputs.mode != 'supersede-target' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
- name: Require explicit mutation confirmation
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: |
if [[ "${DEPLOY_MODE}" = "cutover-admission" ]]; then
test "${CONFIRMATION}" = "CUTOVER_SELECTOR"
elif [[ "${DEPLOY_MODE}" = "add-migration-cells" ]]; then
test "${CONFIRMATION}" = "ADD_MIGRATION_CELLS"
elif [[ "${DEPLOY_MODE}" = "promote-general-cell" ]]; then
test "${CONFIRMATION}" = "PROMOTE_GENERAL_CELL"
elif [[ "${DEPLOY_MODE}" = "retire-migration-cell" ]]; then
test "${CONFIRMATION}" = "RETIRE_MIGRATION_CELL"
elif [[ "${DEPLOY_MODE}" = "execute" ]]; then
test "${CONFIRMATION}" = "EVACUATE_MULTI"
elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then
test "${CONFIRMATION}" = "RECOVER_FORWARD"
elif [[ "${DEPLOY_MODE}" = "fence-source" ]]; then
test "${CONFIRMATION}" = "FENCE_SOURCE"
elif [[ "${DEPLOY_MODE}" = "supersede-target" ]]; then
test "${CONFIRMATION}" = "SUPERSEDE_TARGET"
elif [[ "${DEPLOY_MODE}" = "abort-fence-source" ]]; then
test "${CONFIRMATION}" = "ABORT_FENCE"
else
test "${CONFIRMATION}" = "FENCE_SOURCE"
fi
- name: Require exact source-fence broker contract
if: ${{ inputs.mode == 'fence-source' }}
run: |
test "${SOURCE_CELL_ID}" = "production-gce-c3"
test "${TARGET_CELL_IDS}" = "production-gce-c7,production-gce-c8,production-gce-c10,production-gce-c13,production-gce-c17,production-gce-c18"
[[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
[[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}"
[[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]]
else
test -z "${EXPECTED_LEASE_OPERATION_ID}"
test -z "${EXPECTED_LEASE_REQUEST_DIGEST}"
fi
- name: Require exact broker cell contract
if: ${{ inputs.mode == 'supersede-target' }}
run: |
test "${SOURCE_CELL_ID}" = "production-gce-c3"
test "${FAILED_TARGET_CELL_ID}" = "production-gce-c12"
test "${REPLACEMENT_TARGET_CELL_ID}" = "production-gce-c13"
test "${TARGET_CELL_IDS}" = "production-gce-c12,production-gce-c13"
if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then
[[ "${COMPLETED_FENCE_ATTEMPT_ID}" =~ ^[0-9a-f-]{36}$ ]]
[[ "${COMPLETED_FENCE_COMMIT}" =~ ^[0-9a-f]{40}$ ]]
[[ "${COMPLETED_FENCE_OPERATION}" =~ ^[A-Za-z0-9._-]{1,256}$ ]]
[[ "${COMPLETED_FENCE_STATE_SERIAL}" =~ ^[0-9]+$ ]]
[[ "${COMPLETED_FENCE_PLAN_GENERATION}" =~ ^[1-9][0-9]*$ ]]
[[ "${COMPLETED_FENCE_STATE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
[[ "${COMPLETED_FENCE_STATE_SHA256}" =~ ^[0-9a-f]{64}$ ]]
test -n "${EXPECTED_LEASE_GENERATION}"
fi
if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
[[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}"
[[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]]
else
test -z "${EXPECTED_LEASE_OPERATION_ID}"
test -z "${EXPECTED_LEASE_REQUEST_DIGEST}"
fi
- name: Read reviewed Terraform topology
if: ${{ inputs.mode != 'supersede-target' }}
run: |
node dev/scripts/infra.mjs init --env production
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
DIRECTOR_MIN_INSTANCES="$(terraform -chdir=infra/terraform console \
-var-file=environments/production.tfvars <<< 'var.relay_min_instances')"
[[ "${DIRECTOR_MIN_INSTANCES}" =~ ^[1-9][0-9]*$ ]]
echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
echo "DIRECTOR_MIN_INSTANCES=${DIRECTOR_MIN_INSTANCES}" >> "${GITHUB_ENV}"
- name: Verify fresh dry-run evidence against live selector
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
SCOPED_RECOVERY_ARGS=()
if [[ ("${DEPLOY_MODE}" = "execute" ||
"${DEPLOY_MODE}" = "recover-forward") &&
"${SOURCE_CELL_ID}" = "production-gce-c12" ]]; then
SCOPED_RECOVERY_ARGS=(
--scoped-recovery-source-cell-id
production-gce-c3
)
fi
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
--run-id "${MONITOR_RUN_ID}" \
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
--commit-sha "${GITHUB_SHA}" \
--mode dry-run \
--mutation-mode "${DEPLOY_MODE}" \
--source-cell-id "${SOURCE_CELL_ID}" \
"${SCOPED_RECOVERY_ARGS[@]}" \
--director-origin "${DIRECTOR_ORIGIN}"
- name: Recheck all live safety signals
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
pnpm incident:relay-preflight -- \
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
- name: Create single-use dry-run marker
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
run: |
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
printf '%s\n' "${GITHUB_RUN_ID}" \
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
- name: Consume dry-run evidence
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
uses: actions/upload-artifact@v4
with:
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
retention-days: 90
if-no-files-found: error
- id: google-fence-broker-auth
if: >-
${{ inputs.mode == 'fence-source' ||
inputs.mode == 'supersede-target' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
id_token_include_email: true
- name: Invoke private target-supersession broker
if: ${{ inputs.mode == 'supersede-target' }}
env:
BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }}
BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
run: |
[[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then
REQUEST="$(jq -cn \
--arg operationId "${BROKER_OPERATION_ID}" \
--arg fenceCommit "${GITHUB_SHA}" \
--arg attemptId "${COMPLETED_FENCE_ATTEMPT_ID}" \
--arg completedCommit "${COMPLETED_FENCE_COMMIT}" \
--arg gceOperation "${COMPLETED_FENCE_OPERATION}" \
--arg stateSerial "${COMPLETED_FENCE_STATE_SERIAL}" \
--arg planGeneration "${COMPLETED_FENCE_PLAN_GENERATION}" \
--arg stateGeneration "${COMPLETED_FENCE_STATE_GENERATION}" \
--arg stateSha256 "${COMPLETED_FENCE_STATE_SHA256}" \
--arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
--arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
--arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
completedFenceRecovery:{attemptId:$attemptId,fenceCommit:$completedCommit,
gceOperation:$gceOperation,terraformStateSerial:($stateSerial|tonumber),
planObjectGeneration:$planGeneration,
terraformStateObjectGeneration:$stateGeneration,
terraformStateObjectSha256:$stateSha256},
expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')"
elif [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
REQUEST="$(jq -cn \
--arg operationId "${BROKER_OPERATION_ID}" \
--arg fenceCommit "${GITHUB_SHA}" \
--arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
--arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
--arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')"
else
REQUEST="$(jq -cn \
--arg operationId "${BROKER_OPERATION_ID}" \
--arg fenceCommit "${GITHUB_SHA}" \
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,confirmation:"SUPERSEDE_TARGET"}')"
fi
curl --fail-with-body --max-time 1790 \
--request POST "${BROKER_URI}/v1/supersede-target" \
--header "Authorization: Bearer ${BROKER_ID_TOKEN}" \
--header 'Content-Type: application/json' \
--data "${REQUEST}"
- name: Invoke private source-fence broker
if: ${{ inputs.mode == 'fence-source' }}
env:
BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }}
BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
run: |
if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
REQUEST="$(jq -cn \
--arg operationId "${BROKER_OPERATION_ID}" \
--arg fenceCommit "${GITHUB_SHA}" \
--arg targetCellIds "${TARGET_CELL_IDS}" \
--arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
--arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
--arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
targetCellIds:($targetCellIds|split(",")),
expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
requestDigest:$leaseRequestDigest},confirmation:"FENCE_SOURCE"}')"
else
REQUEST="$(jq -cn \
--arg operationId "${BROKER_OPERATION_ID}" \
--arg fenceCommit "${GITHUB_SHA}" \
--arg targetCellIds "${TARGET_CELL_IDS}" \
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
targetCellIds:($targetCellIds|split(",")),confirmation:"FENCE_SOURCE"}')"
fi
curl --fail-with-body --max-time 1790 \
--request POST "${BROKER_URI}/v1/fence-source" \
--header "Authorization: Bearer ${BROKER_ID_TOKEN}" \
--header 'Content-Type: application/json' \
--data "${REQUEST}"
- name: Preflight or run multi-target evacuation
if: >-
${{ inputs.mode != 'fence-source' &&
inputs.mode != 'supersede-target' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/deploy-relay-gce-multi-target.mjs \
--project "${GCP_PROJECT_ID}" \
--director-origin "${DIRECTOR_ORIGIN}" \
--admin-audience "${ADMIN_AUDIENCE}" \
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
--source-cell-id "${SOURCE_CELL_ID}" \
--target-cell-ids "${TARGET_CELL_IDS}" \
--general-cell-ids "${GENERAL_CELL_IDS}" \
--unobserved-connection-bound "${UNOBSERVED_CONNECTION_BOUND}" \
--director-region "${{ vars.PRODUCTION_GCP_REGION }}" \
--director-service "orca-cloud-relay" \
--director-min-instances "${DIRECTOR_MIN_INSTANCES}" \
--selector-attempt-id "${SELECTOR_ATTEMPT_ID}" \
--failed-target-cell-id "${FAILED_TARGET_CELL_ID}" \
--replacement-target-cell-id "${REPLACEMENT_TARGET_CELL_ID}" \
--runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
--environment production \
--fence-commit "${GITHUB_SHA}" \
--terraform-dir infra/terraform \
--terraform-var-file environments/production.tfvars \
--mode "${DEPLOY_MODE}" \
--connection-ceiling 1000 \
--minimum-lease-remaining-ms 600000
@@ -0,0 +1,923 @@
name: Deploy Relay Production Same-Cap Job
on:
workflow_call:
inputs:
mode: { required: true, type: string }
target-cell-id: { required: true, type: string }
target-image-digest: { required: true, type: string }
rollback-image-digest: { required: true, type: string }
target-rehome-protocol: { required: true, type: string }
rollback-rehome-protocol: { required: true, type: string }
expected-selector-generation: { required: true, type: string }
expected-existing-only-cells: { required: true, type: string }
expected-migration-only-cells: { required: true, type: string }
expected-general-cells: { required: true, type: string }
expected-rehome-generation: { required: true, type: string }
wave-index: { required: true, type: string }
drain-pace-window-ms: { required: true, type: string }
permissions:
contents: read
id-token: write
defaults:
run:
working-directory: cloud
jobs:
rollout:
if: ${{ github.ref == 'refs/heads/main' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
# The pre-drain sample adds 3-8 min (20 at most); a timeout cancels the job and skips the failsafe.
# A 1200000 drain step alone may take 50 min.
timeout-minutes: 120
environment: production
env:
GCP_PROJECT_ID: onorca-cloud
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
DIRECTOR_ORIGIN: https://relay.onorca.dev
IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
DEPLOY_MODE: ${{ inputs.mode }}
TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }}
ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }}
TARGET_REHOME_PROTOCOL: ${{ inputs.target-rehome-protocol }}
ROLLBACK_REHOME_PROTOCOL: ${{ inputs.rollback-rehome-protocol }}
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
WAVE_INDEX: ${{ inputs.wave-index }}
# The cell spreads its drain sends over this window, so each drained host re-dials
# /v1/assign at hosts / window and is re-placed through the director's drain-return
# lane. The wave validator admits only its closed set, and anything below the 300000
# default only on US general cells: Asia drains are bound by their targets' accept
# rate. The transition wait below is the 15-min migration lease plus this window. A
# cell on an older image drains at once, or refuses a window above 300000.
DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }}
steps:
- name: Require exact reusable-workflow configuration
working-directory: .
env:
DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
CAPACITY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
run: |
[[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]]
[[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
[[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}"
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
# cell_1..cell_10 in the calling wave; the chain is static, so this range is too.
[[ "${WAVE_INDEX}" =~ ^[0-9]$ ]]
if test "${DEPLOY_MODE}" != verify && test "${GITHUB_RUN_ATTEMPT}" != 1; then
echo "mutations are single-dispatch: recover each remaining cell" >&2
echo "with its own fresh dispatch instead of a re-run" >&2
exit 1
fi
test -n "${GCP_REGION}"
test -n "${DEPLOY_WIF}"
test -n "${DEPLOY_SERVICE_ACCOUNT}"
test -n "${CAPACITY_WIF}"
test -n "${CAPACITY_SERVICE_ACCOUNT}"
test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with: { package_json_file: cloud/package.json }
- uses: actions/setup-node@v4
with:
node-version: 24
- run: pnpm install --frozen-lockfile
- uses: hashicorp/setup-terraform@v3
with: { terraform_wrapper: false }
# One approved-cell table, in the wave validator the dispatch gate already uses, so
# a cell's class and its wave's selector delta cannot drift apart between the two.
# Also re-checks the drain pace against this cell, before anything reads production.
- name: Resolve this cell's admission class and wave selector delta
run: |
CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \
--cell-id "${TARGET_CELL_ID}" --drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}")"
ENTRY_ADMISSION="$(jq -er '.entryAdmission' <<< "${CELL_CLASS}")"
SELECTOR_WAVE_DELTA="$(jq -er '.selectorWaveDelta' <<< "${CELL_CLASS}")"
if test "${DEPLOY_MODE}" = verify; then
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
else
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION \
+ (SELECTOR_WAVE_DELTA * WAVE_INDEX)))"
fi
{
echo "ENTRY_ADMISSION=${ENTRY_ADMISSION}"
echo "SELECTOR_WAVE_DELTA=${SELECTOR_WAVE_DELTA}"
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}"
} >> "${GITHUB_ENV}"
- id: deploy-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
release: 'false'
- name: Recheck aggregate SQL, pool, reconnect, migration, and selector safety
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
# Freshness-only failures are publish lag, not health, so the CLI retries
# them a bounded number of times. The expected selector comes from the
# dispatch inputs, which the rehome inspect below verifies against the
# live director; the pre-drain sample later reads the same file.
jq -n \
--arg existingOnly "${EXPECTED_EXISTING_ONLY_CELLS/none/}" \
--arg migrationOnly "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
--arg general "${EXPECTED_GENERAL_CELLS/none/}" \
'{existingOnly:$existingOnly,migrationOnly:$migrationOnly,general:$general}
| map_values(split(",") | map(select(length > 0)))' \
> "${RUNNER_TEMP}/relay-same-cap-selector.json"
pnpm incident:relay-preflight -- \
--no-monitor-state \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \
--wave-index "${WAVE_INDEX}" \
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
- name: Require durable rehome disabled and exact selector
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode inspect \
--director-origin "${DIRECTOR_ORIGIN}" \
--expected-selector-generation "${EFFECTIVE_SELECTOR_GENERATION}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
| jq -e '.control.enabled == false' >/dev/null
- name: Initialize the exact production backend
run: node dev/scripts/infra.mjs init --env production
- name: Resolve immutable same-cap cell configuration
shell: bash
run: |
TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"
case "${TARGET_HOSTNAME}" in
c7|c8|c9|c10|c13|c14|c15|c16|c19|c20|c21|c22|c23|c24|c25|c26)
EXPECTED_HARD_CAP=1000
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
c17|c18)
EXPECTED_HARD_CAP=600
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
c27|c28|c29|c30|c31|c34|c35)
EXPECTED_HARD_CAP=3000
EXPECTED_REGION=asia-east2
EXPECTED_DATABASE_POOL_MAX=16
;;
c32|c33)
EXPECTED_HARD_CAP=3000
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
*) exit 1 ;;
esac
EXPECTED_UNOBSERVED_BOUND=60
CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev"
CELLS_JSON="$(terraform -chdir=infra/terraform console \
-var-file=environments/production.tfvars \
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
SOURCE_CELLS="$(terraform -chdir=infra/terraform console \
-var-file=environments/production.tfvars \
<<< 'jsonencode(var.relay_region_rehome_source_cell_ids)' | jq -er '.')"
CURRENT_SHAPE="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${CELLS_JSON}")"
test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}"
test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \
"${EXPECTED_UNOBSERVED_BOUND}"
# The startup script emits a pool line only off the root default, so an unpinned cell
# must still be on that default or its plan would carry a line nothing reviews.
test "$(jq -r '.database_pool_max' <<< "${CURRENT_SHAPE}")" = \
"${EXPECTED_DATABASE_POOL_MAX:-10}"
TARGET_ZONE="$(jq -r '.zone' <<< "${CURRENT_SHAPE}")"
MIG_NAME="orca-cloud-relay-gce-${TARGET_HOSTNAME}"
if test "${DEPLOY_MODE}" = rollback; then
DESIRED_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}"
CURRENT_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}"
DESIRED_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
CURRENT_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
else
DESIRED_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}"
CURRENT_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}"
DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
CURRENT_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
fi
# The startup template emits rehome trust lines only for a declared source cell, so
# require membership exactly when either side of this roll expects those lines.
if test "${EXPECTED_REGION}" = us-central1 && {
test "${DESIRED_REHOME_PROTOCOL}" != 0 || test "${CURRENT_REHOME_PROTOCOL}" != 0
}; then
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
<<< "${SOURCE_CELLS}" >/dev/null
fi
DESIRED_IMAGE="${IMAGE_REPOSITORY}@${DESIRED_IMAGE_DIGEST}"
OVERRIDE_CELLS_JSON="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
--arg image "${DESIRED_IMAGE}" '.[$cell].image = $image' <<< "${CELLS_JSON}")"
jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-same-cap.tfvars.json"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${DESIRED_IMAGE}" \
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
test "${SERVED_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"
{
echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}"
echo "CELL_ORIGIN=${CELL_ORIGIN}"
echo "TARGET_ZONE=${TARGET_ZONE}"
echo "MIG_NAME=${MIG_NAME}"
echo "EXPECTED_HARD_CAP=${EXPECTED_HARD_CAP}"
echo "EXPECTED_UNOBSERVED_BOUND=${EXPECTED_UNOBSERVED_BOUND}"
echo "EXPECTED_DATABASE_POOL_MAX=${EXPECTED_DATABASE_POOL_MAX}"
echo "EXPECTED_REGION=${EXPECTED_REGION}"
echo "DESIRED_IMAGE=${DESIRED_IMAGE}"
echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}"
echo "CURRENT_IMAGE_DIGEST=${CURRENT_IMAGE_DIGEST}"
echo "DESIRED_REHOME_PROTOCOL=${DESIRED_REHOME_PROTOCOL}"
echo "CURRENT_REHOME_PROTOCOL=${CURRENT_REHOME_PROTOCOL}"
} >> "${GITHUB_ENV}"
- name: Verify exact current generation, digest, cap, and rollback point
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
# A single transient 5xx (LB warm-up behind a fresh instance) must not
# fail a canary; 4xx (auth, generation mismatch) still fails fast.
admin_post() {
local out="${RUNNER_TEMP}/$1.json"
if ! curl --fail-with-body --max-time 30 \
--retry 3 --retry-delay 2 --retry-connrefused --output "${out}" \
--request POST "$2" \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' --data "$3"; then
cat "${out}" >&2
return 1
fi
cat "${out}"
}
CURRENT_RUNTIME="$(admin_post current-runtime \
"${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
# Two different failures leave the cell on the rollback image, and the image
# alone cannot tell them apart. A rollback that failed between its template
# apply and its admission restore restarted the cell, so that cell is not
# draining and resumes. A wave that stopped after its drain and before its
# template apply never restarted anything, so its cell is still draining and
# is stranded: the drain flag only clears on a restart, so it has to be rolled.
LIVE_IMAGE_DIGEST="$(jq -r '.imageDigest' <<< "${CURRENT_RUNTIME}")"
LIVE_DRAINING="$(jq -r '.draining' <<< "${CURRENT_RUNTIME}")"
if test "${DEPLOY_MODE}" = rollback \
&& test "${LIVE_IMAGE_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"; then
if test "${LIVE_DRAINING}" = true; then
ROLLBACK_STAGE=stranded
else
ROLLBACK_STAGE=resume
fi
PREDECESSOR_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
PREDECESSOR_REHOME_PROTOCOL="${DESIRED_REHOME_PROTOCOL}"
else
ROLLBACK_STAGE=roll
PREDECESSOR_IMAGE_DIGEST="${CURRENT_IMAGE_DIGEST}"
PREDECESSOR_REHOME_PROTOCOL="${CURRENT_REHOME_PROTOCOL}"
fi
if test "${ROLLBACK_STAGE}" = resume; then
ROLLBACK_RESUME=true
else
ROLLBACK_RESUME=false
fi
# A stranded cell's template still carries the image the cell is serving, so that
# is the predecessor its plan is reviewed against. A template already moved on to
# the target is refused here rather than rolled backwards under a stale review.
if test "${ROLLBACK_STAGE}" = stranded; then
PLAN_ROLLBACK_IMAGE="${DESIRED_IMAGE}"
else
PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"
fi
# Rollback is the documented recovery from a failed canary, which
# leaves the cell migration-only (and possibly still marked
# draining); apply and verify still require the cell pristine in the
# class it is declared to serve in.
if test "${DEPLOY_MODE}" = rollback; then
PRECHECK_ADMISSION=general-or-migration-only
else
PRECHECK_ADMISSION="${ENTRY_ADMISSION}"
fi
# Draining sheds connections, and a migration-only cell holds none, so the flag
# carries no precondition there. It also outlives a failed wave, because the drain
# that set it is followed by no restart, which is the state a failed canary leaves.
if test "${DEPLOY_MODE}" = rollback \
|| test "${ENTRY_ADMISSION}" = migration-only; then
PRECHECK_DRAINING=either
else
PRECHECK_DRAINING=forbidden
fi
# A resumed rollback already restarted, so its cell has to come back not draining;
# that is what separates it from a wave that stopped before its template apply.
if test "${PRECHECK_DRAINING}" = either && test "${ROLLBACK_RESUME}" != true; then
PREDECESSOR_DRAINING_OK=true
else
PREDECESSOR_DRAINING_OK=false
fi
RESTORED_MIGRATION_CELLS="$(jq -rn \
--arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
--arg target "${TARGET_CELL_ID}" \
'$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
RESTORED_GENERAL_CELLS="$(jq -rn \
--arg value "${EXPECTED_GENERAL_CELLS/none/}" \
--arg target "${TARGET_CELL_ID}" \
'$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')"
test -n "${RESTORED_MIGRATION_CELLS}" || RESTORED_MIGRATION_CELLS=none
test -n "${RESTORED_GENERAL_CELLS}" || RESTORED_GENERAL_CELLS=none
ISOLATED_MIGRATION_CELLS="$(jq -rn \
--arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
--arg target "${TARGET_CELL_ID}" \
'$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')"
ISOLATED_GENERAL_CELLS="$(jq -rn \
--arg value "${EXPECTED_GENERAL_CELLS/none/}" \
--arg target "${TARGET_CELL_ID}" \
'$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
test -n "${ISOLATED_MIGRATION_CELLS}" || ISOLATED_MIGRATION_CELLS=none
test -n "${ISOLATED_GENERAL_CELLS}" || ISOLATED_GENERAL_CELLS=none
# A migration-only cell is already isolated and is handed back isolated, so both
# halves of its wave see exactly the membership it entered with.
if test "${ENTRY_ADMISSION}" = migration-only; then
RESTORED_MIGRATION_CELLS="${ISOLATED_MIGRATION_CELLS}"
RESTORED_GENERAL_CELLS="${ISOLATED_GENERAL_CELLS}"
fi
{
echo "ROLLBACK_RESUME=${ROLLBACK_RESUME}"
echo "ROLLBACK_STAGE=${ROLLBACK_STAGE}"
echo "PLAN_ROLLBACK_IMAGE=${PLAN_ROLLBACK_IMAGE}"
# The drain wait and the plan review both read the image this cell actually
# serves, which is the rollback image on a stranded cell and not the current one.
echo "PREDECESSOR_IMAGE_DIGEST=${PREDECESSOR_IMAGE_DIGEST}"
# The failsafe consumes these; deriving them here keeps them
# defined for a failure in any later step.
echo "ISOLATED_MIGRATION_CELLS=${ISOLATED_MIGRATION_CELLS}"
echo "ISOLATED_GENERAL_CELLS=${ISOLATED_GENERAL_CELLS}"
# No restart happens on resume, so isolate below is skipped and
# cannot advance the selector generation.
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${EFFECTIVE_SELECTOR_GENERATION}"
# A failed-canary rollback enters with the target migration-only,
# so the restore inspect cannot reuse the entry membership inputs.
echo "RESTORED_MIGRATION_CELLS=${RESTORED_MIGRATION_CELLS}"
echo "RESTORED_GENERAL_CELLS=${RESTORED_GENERAL_CELLS}"
} >> "${GITHUB_ENV}"
if ! jq -e --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \
--arg digest "${PREDECESSOR_IMAGE_DIGEST}" \
--arg region "${EXPECTED_REGION}" \
--argjson hardCap "${EXPECTED_HARD_CAP}" \
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
'.role == "cell" and .cellId == $cell and .cellUrl == $origin and
(.region == $region or
($region == "us-central1" and $protocol == 0 and .region == null)) and
.imageDigest == $digest and
.connectionCapacity.hardCap == $hardCap and
.connectionCapacity.unobservedBound == $unobservedBound and
(.draining == false or $drainingOk) and
(.regionalRehomeProtocol // 0) == $protocol' <<< "${CURRENT_RUNTIME}" >/dev/null
then
jq -r --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \
--arg digest "${PREDECESSOR_IMAGE_DIGEST}" \
--arg region "${EXPECTED_REGION}" \
--argjson hardCap "${EXPECTED_HARD_CAP}" \
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
'[
if .role != "cell" then "role" else empty end,
if .cellId != $cell then "cellId" else empty end,
if .cellUrl != $origin then "cellUrl" else empty end,
if (.region != $region and
($region != "us-central1" or $protocol != 0 or .region != null))
then "region" else empty end,
if .imageDigest != $digest then "imageDigest" else empty end,
if .connectionCapacity.hardCap != $hardCap then "hardCap" else empty end,
if .connectionCapacity.unobservedBound != $unobservedBound then "unobservedBound" else empty end,
if (.draining != false and ($drainingOk | not)) then "draining" else empty end,
if (.regionalRehomeProtocol // 0) != $protocol then "regionalRehomeProtocol" else empty end
] | "runtime predecessor mismatch fields=" + join(",")' \
<<< "${CURRENT_RUNTIME}" >&2
exit 1
fi
# The exact legacy digest binds omitted pre-region fields to US and protocol 0.
jq -r '[
if .region == null then "region" else empty end,
if .regionalRehomeProtocol == null then "regionalRehomeProtocol" else empty end
] | if length > 0 then "runtime predecessor normalized legacy fields=" + join(",") else empty end' \
<<< "${CURRENT_RUNTIME}"
CURRENT_DIRECTOR_STATUS="$(admin_post current-cell-status \
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
"$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
SOURCE_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \
<<< "${CURRENT_DIRECTOR_STATUS}")"
if test "${ROLLBACK_RESUME}" = true && ! jq -e \
'.status.admissionState == "migration-only"' \
<<< "${CURRENT_DIRECTOR_STATUS}" >/dev/null; then
echo 'resume requires the isolated migration-only cell a failed rollback leaves' >&2
exit 1
fi
[[ "${SOURCE_INCARNATION}" =~ ^[0-9a-f-]{36}$ ]]
echo "SOURCE_INCARNATION=${SOURCE_INCARNATION}" >> "${GITHUB_ENV}"
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh --admission "${PRECHECK_ADMISSION}" \
--draining "${PRECHECK_DRAINING}" --activity allowed \
--expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}"
# Hosts drained with nowhere to go keep redialling and pin the cell empty (c28, 2026-10-01).
# Every run that drains (apply, rollback) checks first; verify reads the same answer; a resume drains nothing.
- name: Require free general-cell slots for the selected cell's hosts
if: ${{ env.ROLLBACK_RESUME != 'true' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
HEADROOM_STATUS=0
HEADROOM="$(node dev/scripts/check-relay-same-cap-headroom.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --general-cells "${EXPECTED_GENERAL_CELLS}")" \
|| HEADROOM_STATUS=$?
echo "${HEADROOM}"
test "${HEADROOM_STATUS}" = 0
# The pre-drain sample sizes its window from this count.
TARGET_HOSTS="$(jq -er '.targetHosts' <<< "${HEADROOM}")"
echo "TARGET_HOSTS=${TARGET_HOSTS}" >> "${GITHUB_ENV}"
# The last read before isolation, so a roll drains on health measured minutes ago, not on a
# separate monitor run's verdict. Its window grows with the hosts the drain will re-place;
# every later wave samples again. Any trip fails here, before anything is changed. Rollback
# skips it: getting off a crash-looping image must not wait for the crashes to stop.
- name: Sample fleet health for a window sized to this drain
if: ${{ inputs.mode == 'apply' }}
timeout-minutes: 20
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
pnpm incident:relay-pre-drain-sample -- \
--target-cell-id "${TARGET_CELL_ID}" \
--target-hosts "${TARGET_HOSTS}" \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \
--wave-index "${WAVE_INDEX}" \
--selector-wave-delta "${SELECTOR_WAVE_DELTA}"
- name: Finish read-only verification
if: ${{ inputs.mode == 'verify' }}
run: echo 'Exact same-cap rollback point verified.'
# The sample can age the first one-hour token by up to 20 min; the drain's restart-safe
# wait (20 min at the default pace, 35 at the slowest) gets a fresh one.
- id: drain-auth
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Reversibly isolate and drain only the selected cell
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
id: drain
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.drain-auth.outputs.id_token }}
run: |
# Before any mutation: a cell whose image caps the pace below this one stops untouched.
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode pace-check \
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}"
# Opens the window the report-only shadow health gate below judges this cell over.
echo "drain-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
# A cell isolated by a failed canary is already migration-only, so
# isolate is a no-op there that does not advance the selector; the
# result's generation is authoritative either way.
ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
echo "${ISOLATE_RESULT}"
# Isolating a migration-only cell must be a read-only no-op; a change here would
# mean the live class is not the one this wave planned around.
if test "${ENTRY_ADMISSION}" = migration-only; then
jq -e '.changed == false' <<< "${ISOLATE_RESULT}" >/dev/null
fi
# The director re-places hosts off a cell only when the isolate stamped it, so a
# script too old to ask for the stamp produces today's behaviour and the canary
# reads as "the fix did nothing" with nothing to tell that from a wrong premise.
jq -e '.rollIsolated == true' <<< "${ISOLATE_RESULT}" >/dev/null
ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}"
DRAIN_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain \
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}")"
echo "${DRAIN_RESULT}"
# What the cell accepted, which is 0 when an older image fell back to an unpaced drain.
echo "drain-applied-pace-window-ms=$(jq -er '.paceWindowMs' <<< "${DRAIN_RESULT}")" \
>> "${GITHUB_OUTPUT}"
# Restart-safe means an empty runtime that lasts past the drain's pace window; leases
# held by unplaceable hosts are reported, not waited on (c28, 2026-10-01).
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat either --admission migration-only --draining required \
--activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \
--timeout-ms "$((900000 + DRAIN_PACE_WINDOW_MS))"
echo "drain-settled-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
- id: capacity-auth
if: ${{ inputs.mode != 'verify' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
- name: Require converged Terraform state and a stable MIG on resume
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }}
shell: bash
env:
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
run: |
# A cell on the root pool default emits no pool line, so pin one only where it exists.
POOL_ARGUMENTS=()
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
fi
# Zero resource changes prove the prior run's apply completed and no
# restart will follow, keeping the incarnation check honest. Root
# outputs may lag a targeted apply, so judge resource_changes only.
# The cell backend service is deliberately not targeted: the capacity role has no
# compute.backendServices.update, so naming it fails the apply after the MIG has
# rolled. Its drain and log_config settings ship in a root apply by a principal
# that holds the permission.
terraform -chdir=infra/terraform plan \
-var-file=environments/production.tfvars \
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
-out="${RUNNER_TEMP}/relay-same-cap-resume.tfplan"
if ! terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
| jq -e '[.resource_changes[]?
| select(.change.actions | any(. != "no-op" and . != "read"))]
| length == 0' >/dev/null
then
# An apply that failed before its template apply also resumes here
# (the cell still serves the rollback image), and repo drift since
# the cell's last roll (for example newly added rehome trust
# config) then legitimately replaces the template. Nothing is
# applied on resume either way, so accept exactly the drift the
# reviewed validator would let a real apply ship for the image the
# cell already serves: the template leaves and re-enters the
# rollback image, as exactly the template-and-MIG change pair.
terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
| jq -r '"resume found unconverged resources: " +
([.resource_changes[]?
| select(.change.actions | any(. != "no-op" and . != "read"))
| .address] | join(","))'
echo 'requiring reviewed rollback-image drift'
RESUME_REVIEW="$(terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--image "${DESIRED_IMAGE}" \
--rollback-image "${DESIRED_IMAGE}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
"${POOL_ARGUMENTS[@]}")"
echo "${RESUME_REVIEW}"
jq -e '.changes == 2' <<< "${RESUME_REVIEW}" >/dev/null
fi
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
- name: Apply only the selected same-cap template and MIG
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
id: apply
shell: bash
env:
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
run: |
# A cell on the root pool default emits no pool line, so pin one only where it exists.
POOL_ARGUMENTS=()
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
fi
terraform -chdir=infra/terraform plan \
-var-file=environments/production.tfvars \
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
-out="${RUNNER_TEMP}/relay-same-cap.tfplan"
PLAN_REVIEW="$(terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap.tfplan" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
--rollback-image "${PLAN_ROLLBACK_IMAGE}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
"${POOL_ARGUMENTS[@]}")"
echo "${PLAN_REVIEW}"
# Stamped before the apply, not after it: the new container announces its listener while
# the MIG is still converging, so a bound taken at the end of this step is already past
# the announcement the shadow gate looks for.
echo "apply-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
terraform -chdir=infra/terraform apply -auto-approve \
"${RUNNER_TEMP}/relay-same-cap.tfplan"
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
# A stranded cell already runs the reviewed template, so the apply above replaces
# no instance and the drain flag, which only a restart clears, would survive the
# whole wave. Fewer than the template-and-MIG pair means the template stayed put,
# including a MIG-only reconciliation. Recreate its one instance from the MIG's
# current template; a rolling action would rewrite the MIG's version name outside
# Terraform, and every later targeted plan would carry that revert.
if test "${ROLLBACK_STAGE}" = stranded \
&& jq -e '.changes < 2' <<< "${PLAN_REVIEW}" >/dev/null; then
STRANDED_INSTANCE="$(gcloud compute instance-groups managed list-instances \
"${MIG_NAME}" --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
--format=json \
| jq -er 'if length == 1 then .[0].instance | split("/") | last
else error("stranded cell MIG does not have exactly one instance") end')"
gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \
--instances "${STRANDED_INSTANCE}" \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --quiet
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
fi
# Recorded for the operator comparing verdicts; the gate's boot search opens at the
# apply-started-at stamp above, not here.
echo "apply-completed-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
- id: post-auth
if: ${{ inputs.mode != 'verify' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Verify new incarnation, exact image, protocol, and durable safety
if: ${{ inputs.mode != 'verify' }}
id: verify-target
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
# Rides out one DB stall: 5-7 s, then readiness grace and two 10 s LB
# health checks (c18, 10-07 18:49Z: 503 "no healthy upstream", healthy
# 16 s later). A cell still down after 60 s fails; other 4xx fail fast.
# Retried in this loop: the runner's 7.81 client (Ubuntu 22.04) never
# retries under --fail-with-body, so its --retry flags were inert.
admin_post() {
local out="${RUNNER_TEMP}/$1.json" window=60 delay=5 status
local deadline=$((SECONDS + window))
while true; do
# A refused or timed-out attempt writes no body; never report the last one's.
rm -f "${out}"
status="$(curl --silent --show-error --max-time 30 --output "${out}" \
--write-out '%{http_code}' --request POST "$2" \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' --data "$3")" || status=000
case "${status}" in
2??) cat "${out}"; return 0 ;;
000|408|429|5??) ((SECONDS + delay < deadline)) || break ;;
*) break ;;
esac
sleep "${delay}"
done
echo "admin_post $1: HTTP ${status}" >&2
cat "${out}" >&2 2>/dev/null || true
return 1
}
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh --admission migration-only --draining forbidden \
--activity allowed --expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" --timeout-ms 900000
TARGET_RUNTIME="$(admin_post target-runtime \
"${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
jq -e --arg digest "${DESIRED_IMAGE_DIGEST}" \
--argjson protocol "${DESIRED_REHOME_PROTOCOL}" \
'.imageDigest == $digest and (.regionalRehomeProtocol // 0) == $protocol' \
<<< "${TARGET_RUNTIME}" >/dev/null
TARGET_DIRECTOR_STATUS="$(admin_post target-cell-status \
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
"$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
TARGET_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \
<<< "${TARGET_DIRECTOR_STATUS}")"
if test "${ROLLBACK_RESUME}" = true; then
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
# No restart happened; the incarnation legitimately stays put.
test "${TARGET_INCARNATION}" = "${SOURCE_INCARNATION}"
else
test "${TARGET_INCARNATION}" != "${SOURCE_INCARNATION}"
fi
echo "TARGET_INCARNATION=${TARGET_INCARNATION}" >> "${GITHUB_ENV}"
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_ISOLATE}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \
--expected-general-cells "${ISOLATED_GENERAL_CELLS}" \
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
| jq -e '.control.enabled == false' >/dev/null
echo "verify-ended-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
- name: Prove exact per-host trust and idempotent no-neighbor behavior
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol != '0') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol != '0')) }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
node dev/scripts/probe-relay-rehome-trust.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-id "${TARGET_CELL_ID}" \
--cell-incarnation "${TARGET_INCARNATION}"
- name: Restore only the verified selected cell to its entry admission
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
# Activating a migration-only cell would promote it to a serving cell for good, so
# restore it with the idempotent isolate that reports the authoritative generation.
if test "${ENTRY_ADMISSION}" = migration-only; then
RESTORE_MODE=isolate
else
RESTORE_MODE=activate
fi
RESTORE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode "${RESTORE_MODE}")"
echo "${RESTORE_RESULT}"
SELECTOR_GENERATION_AFTER_RESTORE="$(jq -er '.generation' \
<<< "${RESTORE_RESULT}")"
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh --admission "${ENTRY_ADMISSION}" \
--draining forbidden --activity allowed \
--expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_RESTORE}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${RESTORED_MIGRATION_CELLS}" \
--expected-general-cells "${RESTORED_GENERAL_CELLS}" \
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
| jq -e '.control.enabled == false' >/dev/null
# Report only: this evaluates the oracles an operator reads by hand after a cell and records
# what it would have called, so its verdicts can be compared with the operator's over a full
# roll before it is ever allowed to block. Two independent guarantees keep it inert: the
# script exits 0 on every verdict, and continue-on-error keeps even a crash off the job's
# outcome. The failsafe below therefore cannot fire on anything this step observes.
#
# It runs after the restore, not before it, for two reasons: the cell goes back into
# admission on exactly today's schedule rather than waiting out a minute of log reads, and
# the window it judges has closed by then, so Cloud Logging's ingestion lag is behind it.
# These are fleet oracles anyway; when this does gate, what it gates is the next cell.
- name: Shadow health gate (report only)
id: shadow-gate
if: ${{ inputs.mode != 'verify' }}
continue-on-error: true
# continue-on-error bounds this step's contribution to the job outcome, not its clock,
# and its reads are serialised. A timed-out step is a failed step, which continue-on-error
# absorbs; without this bound a Logging 429 storm or an expired credential makes every
# read cost its full retry budget and can push the job past timeout-minutes, and a
# cancelled job takes the whole wave with it. A healthy gate is already minutes of
# serial reads, so both bounds sit above that: the script settles at seven minutes and
# reaching this eight is the pathological case. Eight on top of a cell of ~14 minutes
# (~45 at the slowest pace) still leaves the job's 120 minutes intact.
timeout-minutes: 8
env:
DRAIN_STARTED_AT: ${{ steps.drain.outputs.drain-started-at }}
DRAIN_SETTLED_AT: ${{ steps.drain.outputs.drain-settled-at }}
DRAIN_APPLIED_PACE_WINDOW_MS: ${{ steps.drain.outputs.drain-applied-pace-window-ms }}
APPLY_STARTED_AT: ${{ steps.apply.outputs.apply-started-at }}
APPLY_COMPLETED_AT: ${{ steps.apply.outputs.apply-completed-at }}
VERIFY_ENDED_AT: ${{ steps.verify-target.outputs.verify-ended-at }}
SHADOW_GATE_DIRECTORY: ${{ runner.temp }}/relay-same-cap-shadow-gate
SHADOW_GATE_NAME: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json
run: |
mkdir -p "${SHADOW_GATE_DIRECTORY}"
node dev/scripts/relay-same-cap-shadow-gate.mjs \
--cell-id "${TARGET_CELL_ID}" \
--cell-host "${TARGET_HOSTNAME}.relay.onorca.dev" \
--project-id "${GCP_PROJECT_ID}" \
--director-service orca-cloud-relay \
--drain-started-at "${DRAIN_STARTED_AT}" \
--apply-started-at "${APPLY_STARTED_AT}" \
--apply-completed-at "${APPLY_COMPLETED_AT}" \
--verify-ended-at "${VERIFY_ENDED_AT}" \
--drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \
--drain-applied-pace-window-ms "${DRAIN_APPLIED_PACE_WINDOW_MS}" \
--drain-settled-at "${DRAIN_SETTLED_AT}" \
--target-hosts "${TARGET_HOSTS:-}" \
--summary-file "${GITHUB_STEP_SUMMARY}" \
--output-file "${SHADOW_GATE_DIRECTORY}/${SHADOW_GATE_NAME}"
- name: Publish the shadow health gate verdict
if: ${{ inputs.mode != 'verify' }}
continue-on-error: true
# One small JSON file; a retrying upload must not spend the wave's remaining minutes either.
timeout-minutes: 2
uses: actions/upload-artifact@v4
with:
name: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json
path: ${{ runner.temp }}/relay-same-cap-shadow-gate
if-no-files-found: warn
retention-days: 14
overwrite: true
- id: cleanup-auth
if: ${{ failure() && inputs.mode != 'verify' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Keep a failed cell isolated and rehome disabled
if: ${{ failure() && inputs.mode != 'verify' }}
continue-on-error: true
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }}
run: |
test "${MUTATION_STARTED:-false}" = true || exit 0
ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
echo "${ISOLATE_RESULT}"
# Same reason as the isolate step above: a failed wave leaves this cell isolated
# deliberately, and the stamp is what lets its hosts leave.
jq -e '.rollIsolated == true' <<< "${ISOLATE_RESULT}" >/dev/null
# The isolate result carries the authoritative post-isolate generation;
# fixed offsets are wrong whenever an earlier isolate was a no-op.
FAILSAFE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
--expected-selector-generation "${FAILSAFE_GENERATION}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \
--expected-general-cells "${ISOLATED_GENERAL_CELLS}" \
--expected-control-generation "${EXPECTED_REHOME_GENERATION}"
@@ -0,0 +1,433 @@
name: Deploy Relay Production Same-Cap
on:
workflow_dispatch:
inputs:
mode:
description: Verify, roll one canary, roll a bounded batch, or roll back
required: true
default: verify
type: choice
options: [verify, canary-apply, batch-apply, rollback]
cell-ids:
description: Ordered comma-separated serving cells; one canary or two to ten batch cells
required: true
type: string
target-image-digest:
description: Exact immutable compatibility image digest
required: true
type: string
rollback-image-digest:
description: Exact immutable currently serving rollback digest
required: true
type: string
target-rehome-protocol:
description: Exact target regional-rehome protocol
required: true
default: '1'
type: choice
options: ['0', '1', '3']
rollback-rehome-protocol:
description: Exact rollback regional-rehome protocol
required: true
default: '0'
type: choice
options: ['0', '1', '3']
expected-selector-generation:
description: Exact selector generation before the first cell
required: true
type: string
expected-existing-only-cells:
description: Exact existing-only membership, or none
required: true
type: string
expected-migration-only-cells:
description: Exact migration-only membership, or none
required: true
type: string
expected-general-cells:
description: Exact general membership, or none
required: true
type: string
expected-rehome-generation:
description: Exact durable regional-rehome control generation; it must be disabled
required: true
type: string
drain-pace-window-ms:
description: Window the cell spreads its drain over; below 300000 only for US general cells; any other value named in the confirmation
required: true
default: '300000'
type: choice
options: ['300000', '60000', '30000', '900000', '1200000']
canary-run-id:
description: Successful same-code canary in this rehome control generation; reusable across batches
required: false
type: string
confirmation:
description: Exact digest-and-cell-bound mutation confirmation
required: false
type: string
permissions:
actions: read
contents: read
id-token: write
concurrency:
group: production-cloud-sql-rollout
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
gate:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
runs-on: blacksmith-2vcpu-ubuntu-2204
# Headroom for the full-history checkout the canary provenance check needs.
timeout-minutes: 15
environment: production
outputs:
cells: ${{ steps.wave.outputs.cells }}
job-mode: ${{ steps.wave.outputs.job-mode }}
steps:
# Full history: the canary authority a batch verifies is sealed at an ancestor commit, and
# the provenance check fails closed on a commit a shallow clone left out.
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-node@v4
with: { node-version: 24 }
- id: wave
env:
MODE: ${{ inputs.mode }}
CELL_IDS: ${{ inputs.cell-ids }}
TARGET_DIGEST: ${{ inputs.target-image-digest }}
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
CONFIRMATION: ${{ inputs.confirmation }}
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }}
run: |
CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \
--mode "${MODE}" --cell-ids "${CELL_IDS}" \
--target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}" \
--drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}")"
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then
echo 'job-mode=apply' >> "${GITHUB_OUTPUT}"
else
echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}"
fi
- name: Download exact prior canary authority
if: ${{ inputs.mode == 'batch-apply' }}
uses: actions/download-artifact@v4
with:
name: relay-same-cap-canary-${{ inputs.canary-run-id }}
path: ${{ runner.temp }}/relay-same-cap-canary
github-token: ${{ github.token }}
run-id: ${{ inputs.canary-run-id }}
- name: Verify canary authority against this batch
if: ${{ inputs.mode == 'batch-apply' }}
env:
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
CELL_IDS: ${{ inputs.cell-ids }}
TARGET_DIGEST: ${{ inputs.target-image-digest }}
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }}
run: |
node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
--file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
--commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
--cell-ids "${CELL_IDS}" \
--target-digest "${TARGET_DIGEST}" \
--rollback-digest "${ROLLBACK_DIGEST}" \
--drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \
--selector-generation "${SELECTOR_GENERATION}" \
--rehome-generation "${REHOME_GENERATION}"
cell_1:
needs: gate
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[0] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
wave-index: '0'
secrets: inherit
cell_2:
if: ${{ needs.cell_1.result == 'success' && fromJSON(needs.gate.outputs.cells)[1] != null }}
needs: [gate, cell_1]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[1] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
wave-index: '1'
secrets: inherit
cell_3:
if: ${{ needs.cell_2.result == 'success' && fromJSON(needs.gate.outputs.cells)[2] != null }}
needs: [gate, cell_2]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[2] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
wave-index: '2'
secrets: inherit
cell_4:
if: ${{ needs.cell_3.result == 'success' && fromJSON(needs.gate.outputs.cells)[3] != null }}
needs: [gate, cell_3]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[3] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
wave-index: '3'
secrets: inherit
cell_5:
if: ${{ needs.cell_4.result == 'success' && fromJSON(needs.gate.outputs.cells)[4] != null }}
needs: [gate, cell_4]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[4] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
wave-index: '4'
secrets: inherit
cell_6:
if: ${{ needs.cell_5.result == 'success' && fromJSON(needs.gate.outputs.cells)[5] != null }}
needs: [gate, cell_5]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[5] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
wave-index: '5'
secrets: inherit
cell_7:
if: ${{ needs.cell_6.result == 'success' && fromJSON(needs.gate.outputs.cells)[6] != null }}
needs: [gate, cell_6]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[6] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
wave-index: '6'
secrets: inherit
cell_8:
if: ${{ needs.cell_7.result == 'success' && fromJSON(needs.gate.outputs.cells)[7] != null }}
needs: [gate, cell_7]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[7] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
wave-index: '7'
secrets: inherit
cell_9:
if: ${{ needs.cell_8.result == 'success' && fromJSON(needs.gate.outputs.cells)[8] != null }}
needs: [gate, cell_8]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[8] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
wave-index: '8'
secrets: inherit
cell_10:
if: ${{ needs.cell_9.result == 'success' && fromJSON(needs.gate.outputs.cells)[9] != null }}
needs: [gate, cell_9]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[9] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
wave-index: '9'
secrets: inherit
seal_canary:
if: ${{ inputs.mode == 'canary-apply' }}
needs: [gate, cell_1]
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with: { node-version: 24 }
# Its pace checks are what let a batch drain faster than the default; a missing artifact
# seals UNVERIFIED, which only the default accepts. Named from the gate's normalized cell,
# as cell_1 uploaded it, so a padded cell-ids input cannot miss it.
- name: Download the canary cell's shadow health verdict
continue-on-error: true
uses: actions/download-artifact@v4
with:
name: relay-same-cap-shadow-gate-${{ fromJSON(needs.gate.outputs.cells)[0] }}-${{ github.run_id }}.json
path: ${{ runner.temp }}/relay-same-cap-shadow-gate
- name: Seal exact successful canary authority
run: |
mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary"
node dev/scripts/relay-production-same-cap-wave.mjs create-canary \
--cell-id "${{ fromJSON(needs.gate.outputs.cells)[0] }}" \
--target-digest "${{ inputs.target-image-digest }}" \
--rollback-digest "${{ inputs.rollback-image-digest }}" \
--confirmation "${{ inputs.confirmation }}" \
--drain-pace-window-ms "${{ inputs.drain-pace-window-ms }}" \
--shadow-report "${RUNNER_TEMP}/relay-same-cap-shadow-gate/relay-same-cap-shadow-gate-${{ fromJSON(needs.gate.outputs.cells)[0] }}-${GITHUB_RUN_ID}.json" \
--commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \
--selector-generation "${{ inputs.expected-selector-generation }}" \
--rehome-generation "${{ inputs.expected-rehome-generation }}" \
> "${RUNNER_TEMP}/relay-same-cap-canary/authority.json"
- uses: actions/upload-artifact@v4
with:
name: relay-same-cap-canary-${{ github.run_id }}
path: ${{ runner.temp }}/relay-same-cap-canary/authority.json
retention-days: 30
if-no-files-found: error
# Every cell job re-enters the run's lease with release: 'false'; only this job frees it.
release_lease:
if: always()
needs:
- gate
- cell_1
- cell_2
- cell_3
- cell_4
- cell_5
- cell_6
- cell_7
- cell_8
- cell_9
- cell_10
- seal_canary
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
environment: production
steps:
- uses: actions/checkout@v4
- uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
release: 'true'
@@ -0,0 +1,219 @@
name: Deploy Relay Production Candidate
on:
workflow_dispatch:
inputs:
source-cell-id:
description: Existing Terraform cell ID to evacuate
required: true
type: string
target-cell-id:
description: Distinct Terraform candidate cell ID
required: true
type: string
mode:
description: Audit/preflight are read-only; recover/continue resume committed work; disable/enable/reset/execute mutate admission
required: true
default: preflight
type: choice
options:
- audit
- preflight
- recover-forward
- continue-evacuation
- disable-cell
- enable-empty-cell
- reset-empty-candidate
- execute
confirmation:
description: Enter RECOVER_FORWARD, CONTINUE_EVACUATION, DISABLE_CELL, ENABLE_CELL, RESET_CANDIDATE, or EVACUATE for the matching mutation
required: false
type: string
monitor-run-id:
description: Successful fresh dry-run monitor workflow run ID
required: false
type: string
monitor-run-attempt:
description: Exact dry-run monitor workflow attempt
required: false
type: string
permissions:
actions: read
contents: read
id-token: write
concurrency:
group: production-cloud-sql-rollout
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
candidate:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: production
env:
GCP_PROJECT_ID: onorca-cloud
DIRECTOR_ORIGIN: https://relay.onorca.dev
ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain
SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
DEPLOY_MODE: ${{ inputs.mode }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
steps:
- uses: actions/checkout@v4
- name: Require fresh dry-run evidence reference
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
run: |
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
- name: Download private dry-run evidence
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
uses: actions/download-artifact@v4
with:
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ runner.temp }}/relay-monitor-evidence
github-token: ${{ github.token }}
run-id: ${{ inputs.monitor-run-id }}
- uses: pnpm/action-setup@v4
with:
package_json_file: cloud/package.json
- uses: actions/setup-node@v4
with:
node-version: 24
- run: pnpm install --frozen-lockfile
- uses: hashicorp/setup-terraform@v3
with:
terraform_wrapper: false
- name: Verify dry-run artifact before cloud authentication
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
run: |
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
--run-id "${MONITOR_RUN_ID}" \
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
--commit-sha "${GITHUB_SHA}" \
--mode dry-run
- name: Reject previously consumed dry-run evidence
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
env:
GH_TOKEN: ${{ github.token }}
run: |
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
COUNT="$(gh api \
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
--jq '.total_count')"
test "${COUNT}" = "0"
- id: google-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
- name: Require explicit mutation confirmation
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: |
if [[ "${DEPLOY_MODE}" = "execute" ]]; then
test "${CONFIRMATION}" = "EVACUATE"
elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then
test "${CONFIRMATION}" = "RECOVER_FORWARD"
elif [[ "${DEPLOY_MODE}" = "continue-evacuation" ]]; then
test "${CONFIRMATION}" = "CONTINUE_EVACUATION"
elif [[ "${DEPLOY_MODE}" = "disable-cell" ]]; then
test "${CONFIRMATION}" = "DISABLE_CELL"
elif [[ "${DEPLOY_MODE}" = "enable-empty-cell" ]]; then
test "${CONFIRMATION}" = "ENABLE_CELL"
else
test "${CONFIRMATION}" = "RESET_CANDIDATE"
fi
- name: Read reviewed Terraform topology
run: |
node dev/scripts/infra.mjs init --env production
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
- name: Verify fresh dry-run evidence against live selector
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
--run-id "${MONITOR_RUN_ID}" \
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
--commit-sha "${GITHUB_SHA}" \
--mode dry-run \
--mutation-mode "${DEPLOY_MODE}" \
--source-cell-id "${SOURCE_CELL_ID}" \
--director-origin "${DIRECTOR_ORIGIN}"
- name: Recheck all live safety signals
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
pnpm incident:relay-preflight -- \
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
- name: Create single-use dry-run marker
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
run: |
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
printf '%s\n' "${GITHUB_RUN_ID}" \
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
- name: Consume dry-run evidence
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
uses: actions/upload-artifact@v4
with:
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
retention-days: 90
if-no-files-found: error
- name: Preflight or evacuate exact GCE candidate
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/deploy-relay-gce-candidate.mjs \
--project "${GCP_PROJECT_ID}" \
--director-origin "${DIRECTOR_ORIGIN}" \
--admin-audience "${ADMIN_AUDIENCE}" \
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
--source-cell-id "${SOURCE_CELL_ID}" \
--target-cell-id "${TARGET_CELL_ID}" \
--runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
--mode "${DEPLOY_MODE}"
@@ -0,0 +1,111 @@
name: Deploy Relay Staging GCE Candidate
on:
workflow_dispatch:
inputs:
source-cell-id:
description: Existing Terraform cell ID to evacuate
required: true
type: string
target-cell-id:
description: Distinct Terraform candidate cell ID
required: true
type: string
mode:
description: Preflight is read-only; reset repairs an empty candidate; execute evacuates
required: true
default: preflight
type: choice
options:
- preflight
- reset-empty-candidate
- execute
confirmation:
description: Enter RESET_CANDIDATE for reset or EVACUATE for execute
required: false
type: string
permissions:
contents: read
id-token: write
concurrency:
group: relay-staging-mutation
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
candidate:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: staging
env:
GCP_PROJECT_ID: onorca-cloud-staging
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
ADMIN_AUDIENCE: https://relay-staging.onorca.dev/v1/admin/drain
SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
DEPLOY_MODE: ${{ inputs.mode }}
steps:
- uses: actions/checkout@v4
- id: google-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-staging-terraform-state
object: terraform/state/cloud-sql-rollout/staging.lock
- uses: hashicorp/setup-terraform@v3
with:
terraform_wrapper: false
- uses: actions/setup-node@v4
with:
node-version: 24
- name: Require explicit mutation confirmation
if: ${{ inputs.mode != 'preflight' }}
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: |
if [[ "${DEPLOY_MODE}" = "execute" ]]; then
test "${CONFIRMATION}" = "EVACUATE"
else
test "${CONFIRMATION}" = "RESET_CANDIDATE"
fi
- name: Read reviewed Terraform topology
run: |
node dev/scripts/infra.mjs init --env staging
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
- name: Preflight or evacuate exact GCE candidate
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/deploy-relay-gce-candidate.mjs \
--project "${GCP_PROJECT_ID}" \
--director-origin "${DIRECTOR_ORIGIN}" \
--admin-audience "${ADMIN_AUDIENCE}" \
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
--source-cell-id "${SOURCE_CELL_ID}" \
--target-cell-id "${TARGET_CELL_ID}" \
--runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
--mode "${DEPLOY_MODE}"
@@ -0,0 +1,241 @@
name: Deploy Relay Staging
on:
workflow_dispatch:
inputs:
expected-image-digest:
description: Exact checked-in production Relay sha256 digest to deploy
required: true
type: string
reserve-placement:
description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve
required: true
default: preserve
type: choice
options: [preserve, 'off', dry-run, 'on']
expected-rehome-generation:
description: Exact durable regional-rehome generation; it must remain disabled
required: true
type: string
bootstrap-runtime-identity:
description: One-time move from the stamped-cell identity to the director identity
required: true
default: false
type: boolean
predecessor-image-digest:
description: Bootstrap only. Exact immutable serving predecessor digest
required: false
type: string
capacity-cell-id:
description: Register this cell's reviewed staging.tfvars cap and bound with the director
required: true
default: none
type: choice
options: [none, staging-gce-c1, staging-gce-c2, staging-gce-c3, staging-gce-c4]
min-instances:
description: Director floor; preserve keeps the serving revision's, so a reserve run's 2 survives
required: true
default: preserve
type: choice
options: [preserve, '0', '2']
confirmation:
description: BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY for a bootstrap, ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE with a shape baseline
required: false
type: string
shape-baseline-revision:
description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts
required: false
type: string
shadow-seat-feed-cells:
description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused
required: true
default: preserve
type: string
permissions:
contents: read
id-token: write
concurrency:
# Staging deploy, candidate, auth, and power operations must never overlap.
group: relay-staging-mutation
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
deploy:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: staging
env:
GCP_PROJECT_ID: onorca-cloud-staging
GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging
REPOSITORY_ID: orca-cloud
IMAGE_NAME: relay
EXPECTED_IMAGE_DIGEST: ${{ inputs.expected-image-digest }}
CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
# The director must trust the identity every staging Relay workflow authenticates as.
DEPLOY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
ASIA_PROOF_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: orca-cloud-staging-relay-dir@onorca-cloud-staging.iam.gserviceaccount.com
PREDECESSOR_RUNTIME_SERVICE_ACCOUNT: orca-cloud-staging-relay@onorca-cloud-staging.iam.gserviceaccount.com
REHOME_AUDIENCE: https://relay-staging.onorca.dev/v1/admin/host-drain
EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }}
PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }}
CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }}
MIN_INSTANCES: ${{ inputs.min-instances }}
SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }}
steps:
- uses: actions/checkout@v4
- name: Require the expected immutable image and identity request
shell: bash
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: |
set -euo pipefail
[[ "${EXPECTED_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${BOOTSTRAP_RUNTIME_IDENTITY}" =~ ^(true|false)$ ]]
[[ "${MIN_INSTANCES}" =~ ^(preserve|0|2)$ ]]
if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then
[[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
test "${CAPACITY_CELL_ID}" = none
test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY
else
test -z "${PREDECESSOR_IMAGE_DIGEST}"
fi
if test -n "${SHAPE_BASELINE_REVISION}"; then
[[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-staging-[0-9]{5}-[a-z0-9]{3}$ ]]
test "${BOOTSTRAP_RUNTIME_IDENTITY}" = false
test "${CAPACITY_CELL_ID}" = none
test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE
fi
- uses: hashicorp/setup-terraform@v3
with:
terraform_version: 1.15.8
terraform_wrapper: false
- id: google-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Bind the request to the checked-in staging C4 image
shell: bash
run: |
set -euo pipefail
terraform -chdir=infra/terraform init -reconfigure \
-backend-config=backend/staging.hcl -input=false
IMAGE="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars \
<<< 'var.relay_gce_cells["staging-gce-c4"].image' | jq -er '.')"
test "${IMAGE}" = \
"${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${EXPECTED_IMAGE_DIGEST}"
echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
if test "${CAPACITY_CELL_ID}" != none; then
# The deploy script refuses any topology change beyond this one cell's cap and bound.
DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars \
<<< 'local.relay_director_cells_json' | jq -er '.')"
jq -e --arg cell "${CAPACITY_CELL_ID}" \
'any(.[]; .id == $cell and .connectionHardCap != null)' \
<<< "${DESIRED_CELLS_JSON}" >/dev/null
echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" >> "${GITHUB_ENV}"
fi
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-staging-terraform-state
object: terraform/state/cloud-sql-rollout/staging.lock
- uses: actions/setup-node@v4
with:
node-version: 24
- name: Require the mirrored immutable image
run: |
DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
test "${DIGEST}" = "${EXPECTED_IMAGE_DIGEST}"
- name: Deploy director blue/green
env:
# An image change reads every cell's admit mode before the traffic move.
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
RESERVE_PLACEMENT: ${{ inputs.reserve-placement }}
SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }}
run: |
regional_version="$(gcloud secrets versions describe latest \
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \
--format='value(name)' | awk -F/ '{print $NF}')"
[[ "${regional_version}" =~ ^[1-9][0-9]*$ ]]
RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
identity_args=()
if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then
identity_args=(
--bootstrap-runtime-identity true
--predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}"
--predecessor-image-digest "${PREDECESSOR_IMAGE_DIGEST}"
)
fi
# Omitted, the deploy script inherits the serving revision's floor.
floor_args=()
if test "${MIN_INSTANCES}" != preserve; then
floor_args=(--min-instances "${MIN_INSTANCES}")
fi
shape_args=()
if test -n "${SHAPE_BASELINE_REVISION}"; then
shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}")
fi
topology_args=()
if test "${CAPACITY_CELL_ID}" != none; then
topology_args=(
--capacity-cell-id "${CAPACITY_CELL_ID}"
--director-cells-json "${DESIRED_CELLS_JSON}"
)
fi
node dev/scripts/deploy-relay-blue-green.mjs \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--service "${DIRECTOR_SERVICE_NAME}" \
--image "${IMAGE}" \
--role director \
--max-instances 2 \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--deploy-service-account "${DEPLOY_SERVICE_ACCOUNT}" \
--asia-proof-service-account "${ASIA_PROOF_SERVICE_ACCOUNT}" \
--regional-placement-secret-version "${regional_version}" \
"${floor_args[@]}" \
--runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
"${identity_args[@]}" \
"${shape_args[@]}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience "${REHOME_AUDIENCE}" \
--rehome-control-origin https://relay-staging.onorca.dev \
--expected-rehome-generation "${EXPECTED_REHOME_GENERATION}" \
"${topology_args[@]}" \
--admin-audience https://relay-staging.onorca.dev/v1/admin/drain \
--reserve-placement "${RESERVE_PLACEMENT}" \
--shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}" \
--release-id "${RELEASE_ID}"
- name: Smoke director health
run: |
URL="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(status.url)')"
node dev/scripts/smoke-relay.mjs "${URL}"
@@ -0,0 +1,90 @@
name: Monitor Relay Cell Clock Skew
# Why: the 2026-08-01 sustained HOST_OFFLINE incident traced to one cell's
# clock running ~100ms ahead, which deterministically failed every host
# challenge under a zero-tolerance freshness check. /health and /ready cannot
# see clock skew; this monitor alarms before drift reaches the (now 2s)
# client tolerance.
on:
workflow_dispatch:
schedule:
- cron: '17 * * * *'
permissions:
contents: read
defaults:
run:
working-directory: cloud
jobs:
skew:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
sparse-checkout: cloud/infra/terraform/environments/production.tfvars
sparse-checkout-cone-mode: false
persist-credentials: false
- name: Measure Date-header skew for every relay cell
run: |
set -u
# Date headers carry whole seconds; compare floored seconds on both
# sides so perfect sync reads 0/±1 and never flaps. An absolute
# floor-skew of >= 2 means real drift of at least ~1s — approaching
# the client's 2s challenge tolerance. Millisecond-precision deltas
# come from the desktop's named-check logging when activations fail.
ALARM_S=2
failures=0
reachable=0
unserved=0
# Every relay_gce_cells hostname (the block relayOpsCellsFromTerraform reads), fenced
# cells included, so a newly declared cell is probed from its first run.
cells="$(awk '
/^relay_gce_cells[ \t]*=[ \t]*\{/ { inside = 1; next }
inside && /^\}/ { exit }
inside && /^[ \t]*hostname[ \t]*=/ { split($0, quoted, "\""); print quoted[2] }
' infra/terraform/environments/production.tfvars || true)"
if [ -z "${cells}" ]; then
echo "::error::read no relay cells from production.tfvars; monitor blind"
exit 1
fi
for cell in ${cells}; do
url="https://${cell}.relay.onorca.dev/health"
# Why: *.relay.onorca.dev is a wildcard, so the load balancer answers with its own
# accurate Date for a fenced, dead, or never-provisioned cell. Timing that reads as
# perfect sync. Only an HTTP 200 is the relay process itself answering, so only a
# 200 carries a clock worth judging.
response="$(curl -sS -D - -o /dev/null --max-time 8 "${url}" 2>/dev/null || true)"
status="$(printf '%s' "${response}" | awk 'NR==1 {print $2}')"
header="$(printf '%s' "${response}" | tr -d '\r' | grep -i '^date:' || true)"
if [ "${status:-000}" != "200" ] || [ -z "${header}" ]; then
# Expected for the fenced cells; a dead unfenced cell is caught by the heartbeat
# and readiness alerts, not here. Named either way so it is never invisible.
echo "${cell}: not serving (status ${status:-none}); no relay clock to judge"
unserved=$((unserved + 1))
continue
fi
reachable=$((reachable + 1))
server_s="$(date -d "${header#*: }" +%s)"
local_s="$(date +%s)"
skew=$((server_s - local_s))
abs=${skew#-}
if [ "${abs}" -ge "${ALARM_S}" ]; then
echo "::error::${cell}: clock skew ${skew}s reaches ±${ALARM_S}s alarm"
failures=$((failures + 1))
else
echo "${cell}: skew ${skew}s"
fi
done
echo "cells serving: ${reachable}, not serving: ${unserved}, alarms: ${failures}"
if [ "${reachable}" -eq 0 ]; then
# Now meaningful: previously the load balancer answered for every name, so this
# could never fire and a total fleet outage reported "all healthy".
echo "::error::no relay cell is serving; monitor blind"
exit 1
fi
exit "$((failures > 0 ? 1 : 0))"
@@ -0,0 +1,215 @@
name: Monitor Relay Production Job
on:
workflow_call:
inputs:
mode:
required: true
type: string
expected-selector-generation:
required: true
type: string
expected-existing-only-cells:
required: true
type: string
expected-migration-only-cells:
required: true
type: string
expected-general-cells:
required: true
type: string
migration-policy:
required: true
type: string
recovery-source-cell-id:
required: true
type: string
capacity-cell-id:
required: true
type: string
permissions:
actions: read
contents: read
id-token: write
defaults:
run:
working-directory: cloud
jobs:
monitor:
if: ${{ github.ref == 'refs/heads/main' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 100
environment: production
env:
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
MIGRATION_POLICY: ${{ inputs.migration-policy }}
RECOVERY_SOURCE_CELL_ID: ${{ inputs.recovery-source-cell-id }}
CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }}
INCIDENT_ID: relay-${{ github.run_id }}-${{ inputs.mode }}
MONITOR_MODE: ${{ inputs.mode }}
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-incident
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
package_json_file: cloud/package.json
- uses: actions/setup-node@v4
with:
node-version: 24
- run: pnpm install --frozen-lockfile
- id: prior-attempt
if: ${{ github.run_attempt > 1 }}
run: echo "value=$((GITHUB_RUN_ATTEMPT - 1))" >> "${GITHUB_OUTPUT}"
- name: Restore prior private monitor state
if: ${{ github.run_attempt > 1 }}
uses: actions/download-artifact@v4
with:
name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ steps.prior-attempt.outputs.value }}
path: ${{ github.workspace }}/relay-incident
github-token: ${{ github.token }}
run-id: ${{ github.run_id }}
- name: Verify restored state provenance
if: ${{ github.run_attempt > 1 }}
run: |
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
--directory "${OUTPUT_DIRECTORY}" \
--incident-id "${INCIDENT_ID}" \
--run-id "${GITHUB_RUN_ID}" \
--run-attempt "${{ steps.prior-attempt.outputs.value }}" \
--commit-sha "${GITHUB_SHA}" \
--mode "${MONITOR_MODE}"
echo "RESTART_FLAG=--restart" >> "${GITHUB_ENV}"
- id: google-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- name: Verify exact-audience admin identity
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
- name: Run read-only relay dry-run
if: ${{ inputs.mode == 'dry-run' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
pnpm --filter @orca-cloud/relay-ops incident:monitor \
--environment production \
--incident-id "${INCIDENT_ID}" \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
--migration-policy "${MIGRATION_POLICY}" \
--recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
--capacity-cell-id "${CAPACITY_CELL_ID}" \
--interval-seconds 60 \
--output-directory "${OUTPUT_DIRECTORY}" \
--duration-minutes 15 \
--pre-drain-dry-run \
${RESTART_FLAG:-}
- name: Run first relay monitor segment
if: ${{ inputs.mode == 'monitor' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
pnpm --filter @orca-cloud/relay-ops incident:monitor \
--environment production \
--incident-id "${INCIDENT_ID}" \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
--migration-policy "${MIGRATION_POLICY}" \
--recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
--capacity-cell-id "${CAPACITY_CELL_ID}" \
--interval-seconds 60 \
--output-directory "${OUTPUT_DIRECTORY}" \
--duration-minutes 90 \
--max-samples-this-run 45 \
${RESTART_FLAG:-}
- id: google-auth-refresh
if: ${{ inputs.mode == 'monitor' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Run remaining relay monitor window
if: ${{ inputs.mode == 'monitor' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-refresh.outputs.id_token }}
run: |
node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
pnpm --filter @orca-cloud/relay-ops incident:monitor \
--environment production \
--incident-id "${INCIDENT_ID}" \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
--migration-policy "${MIGRATION_POLICY}" \
--recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
--capacity-cell-id "${CAPACITY_CELL_ID}" \
--interval-seconds 60 \
--output-directory "${OUTPUT_DIRECTORY}" \
--duration-minutes 90 \
--restart
- name: Seal private evidence provenance
if: ${{ always() }}
run: |
node dev/scripts/relay-monitor-evidence.mjs create \
--directory "${OUTPUT_DIRECTORY}" \
--incident-id "${INCIDENT_ID}" \
--run-id "${GITHUB_RUN_ID}" \
--run-attempt "${GITHUB_RUN_ATTEMPT}" \
--commit-sha "${GITHUB_SHA}" \
--mode "${MONITOR_MODE}"
- name: Publish aggregate job summary
if: ${{ always() }}
run: |
if [[ -f "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" ]]; then
cat "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" >> "${GITHUB_STEP_SUMMARY}"
else
echo "Relay monitor failed before its first aggregate checkpoint." \
>> "${GITHUB_STEP_SUMMARY}"
fi
- name: Upload private aggregate evidence
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ github.workspace }}/relay-incident
if-no-files-found: error
retention-days: 14
@@ -0,0 +1,75 @@
name: Monitor Relay Production
on:
workflow_dispatch:
inputs:
mode:
description: Run the required 15-minute pre-drain gate or a 90-minute incident watch
required: true
default: dry-run
type: choice
options:
- dry-run
- monitor
expected-selector-generation:
description: Exact durable admission-selector generation
required: true
type: string
expected-existing-only-cells:
description: Exact comma-separated existing-only cells, or none
required: true
type: string
expected-migration-only-cells:
description: Exact comma-separated migration-only cells, or none
required: true
type: string
expected-general-cells:
description: Exact comma-separated general cells, or none
required: true
type: string
migration-policy:
description: Migration checks matched to the intended mutation
required: true
default: strict
type: choice
options:
- strict
- recover-forward
- capacity-transition
recovery-source-cell-id:
description: Existing-only recovery source cell, or none for strict monitoring
required: true
default: none
type: string
capacity-cell-id:
description: General cell for a capacity-transition monitor, or none
required: true
default: none
type: string
permissions:
actions: read
contents: read
id-token: write
concurrency:
group: production-cloud-sql-rollout
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
monitor:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
uses: ./.github/workflows/cloud-monitor-relay-production-job.yml
with:
mode: ${{ inputs.mode }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
migration-policy: ${{ inputs.migration-policy }}
recovery-source-cell-id: ${{ inputs.recovery-source-cell-id }}
capacity-cell-id: ${{ inputs.capacity-cell-id }}
@@ -0,0 +1,557 @@
name: Operate Relay Asia Admission
on:
workflow_dispatch:
inputs:
environment:
description: Target Relay environment
required: true
type: choice
options: [staging, production]
mode:
description: Inspect, initialize, verify, atomically register, promote, or roll back admission
required: true
default: verify
type: choice
options: [inspect, initialize, verify, register, configure, promote, rollback]
cell-ids:
description: Exact reviewed comma-separated cell wave (Asia, or a US cell at the Asia shape)
required: true
type: string
selector-generation:
description: Exact live selector generation; leave empty only for inspect
required: false
type: string
selector-membership-sha256:
description: Exact fingerprint printed by inspect; required only for initialize
required: false
type: string
selector-attempt-id:
description: Durable unique attempt ID; empty for inspect, verify, and configure
required: false
type: string
image-digest:
description: Expected compatible Relay sha256 digest
required: true
type: string
director-image-digest:
description: Director sha256 digest; required only for configure
required: false
type: string
evidence-run-id:
description: Staging evidence run ID for C27, C27 canary run ID for C28/C29; C30-C35 take none and each proves itself by its own canary
required: false
type: string
evidence-run-attempt:
description: Exact evidence workflow run attempt; required with an evidence run ID
required: false
type: string
confirmation:
description: Exact typed confirmation for a mutation
required: false
type: string
permissions:
actions: read
contents: read
id-token: write
concurrency:
group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
admission:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 30
environment: ${{ inputs.environment }}
env:
DIRECTOR_ORIGIN: ${{ inputs.environment == 'production' && 'https://relay.onorca.dev' || 'https://relay-staging.onorca.dev' }}
AUTH_ORIGIN: ${{ inputs.environment == 'production' && 'https://login.onorca.dev' || 'https://auth-staging.onorca.dev' }}
DIRECTOR_SERVICE: ${{ inputs.environment == 'production' && 'orca-cloud-relay' || 'orca-cloud-relay-staging' }}
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }}
GCP_REGION: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_REGION || vars.STAGING_GCP_REGION }}
DIRECTOR_MAX_INSTANCES: ${{ inputs.environment == 'production' && '5' || '2' }}
TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }}
TARGET_ENVIRONMENT: ${{ inputs.environment }}
OPERATION_MODE: ${{ inputs.mode }}
TARGET_CELL_IDS: ${{ inputs.cell-ids }}
EXPECTED_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
EXPECTED_SELECTOR_MEMBERSHIP_SHA256: ${{ inputs.selector-membership-sha256 }}
SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }}
IMAGE_DIGEST: ${{ inputs.image-digest }}
DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }}
EVIDENCE_RUN_ID: ${{ inputs.evidence-run-id }}
EVIDENCE_RUN_ATTEMPT: ${{ inputs.evidence-run-attempt }}
OPERATION_CONFIRMATION: ${{ inputs.confirmation }}
DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
DEPLOY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
steps:
- uses: actions/checkout@v4
- name: Validate exact operation inputs before authentication
id: inputs
shell: bash
run: |
set -euo pipefail
test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
test -n "${DEPLOY_SERVICE_ACCOUNT}"
[[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
evidence_kind=none
canary_cell=none
canary_region=asia-east2
artifact_name=none
if test "${OPERATION_MODE}" = inspect; then
test -z "${EXPECTED_SELECTOR_GENERATION}"
test -z "${SELECTOR_ATTEMPT_ID}"
test -z "${OPERATION_CONFIRMATION}"
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
test -z "${DIRECTOR_IMAGE_DIGEST}"
else
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
fi
if test "${OPERATION_MODE}" = initialize; then
test "${EXPECTED_SELECTOR_GENERATION}" = 0
[[ "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" =~ ^[a-f0-9]{64}$ ]]
test -z "${DIRECTOR_IMAGE_DIGEST}"
[[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
test "${OPERATION_CONFIRMATION}" = INITIALIZE_ADMISSION_SELECTOR
elif test "${OPERATION_MODE}" = verify; then
test -z "${SELECTOR_ATTEMPT_ID}"
test -z "${OPERATION_CONFIRMATION}"
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
test -z "${DIRECTOR_IMAGE_DIGEST}"
elif test "${OPERATION_MODE}" = inspect; then
:
elif test "${OPERATION_MODE}" = configure; then
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
test -z "${SELECTOR_ATTEMPT_ID}"
[[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
test "${OPERATION_CONFIRMATION}" = CONFIGURE_ASIA_DIRECTOR
else
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
test -z "${DIRECTOR_IMAGE_DIGEST}"
[[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
case "${OPERATION_MODE}:${OPERATION_CONFIRMATION}" in
register:REGISTER_ASIA_MIGRATION_ONLY) ;;
promote:PROMOTE_ASIA_GENERAL) ;;
rollback:ROLLBACK_ASIA_MIGRATION_ONLY) ;;
*) echo "typed confirmation does not match the requested mutation" >&2; exit 1 ;;
esac
fi
if test "${TARGET_ENVIRONMENT}:${OPERATION_MODE}" = production:promote; then
case "${TARGET_CELL_IDS}" in
production-gce-c27)
evidence_kind=staging
artifact_name="relay-asia-staging-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}"
canary_cell=production-gce-c27
;;
production-gce-c28,production-gce-c29)
evidence_kind=c27
artifact_name="relay-asia-c27-canary-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}"
;;
production-gce-c30|production-gce-c31|production-gce-c34|production-gce-c35)
# No earlier proof binds a later cell's generation; its own canary below rolls it back on failure.
canary_cell="${TARGET_CELL_IDS}"
;;
production-gce-c32|production-gce-c33)
# A US cell at the Asia shape proves itself the same way, with load aimed at its own region.
canary_cell="${TARGET_CELL_IDS}"
canary_region=us-central1
;;
*) echo "production promotion wave is not reviewed" >&2; exit 1 ;;
esac
fi
if test "${evidence_kind}" = none; then
test -z "${EVIDENCE_RUN_ID}"
test -z "${EVIDENCE_RUN_ATTEMPT}"
else
[[ "${EVIDENCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
[[ "${EVIDENCE_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
fi
canary=false
if test "${canary_cell}" != none; then
canary=true
# Leaves room for the rollback attempt's -rollback suffix within 128 characters.
test "${#SELECTOR_ATTEMPT_ID}" -le 119
fi
{
echo "evidence_kind=${evidence_kind}"
echo "artifact_name=${artifact_name}"
echo "canary=${canary}"
echo "canary_cell=${canary_cell}"
echo "canary_region=${canary_region}"
echo "canary_hostname=${canary_cell##*-}"
} >> "${GITHUB_OUTPUT}"
- uses: actions/setup-node@v4
with:
node-version: 24
- uses: pnpm/action-setup@v4
with:
package_json_file: cloud/package.json
if: ${{ steps.inputs.outputs.canary == 'true' }}
- name: Install exact canary dependencies
if: ${{ steps.inputs.outputs.canary == 'true' }}
run: pnpm install --frozen-lockfile
- name: Build the canary Relay contract
if: ${{ steps.inputs.outputs.canary == 'true' }}
run: pnpm --filter @orca-cloud/relay-contract build
- name: Download immutable rollout evidence
if: ${{ steps.inputs.outputs.evidence_kind != 'none' }}
uses: actions/download-artifact@v4
with:
name: ${{ steps.inputs.outputs.artifact_name }}
path: ${{ runner.temp }}/relay-asia-input-evidence
github-token: ${{ github.token }}
run-id: ${{ inputs.evidence-run-id }}
- name: Verify evidence provenance and rollout binding before authentication
if: ${{ steps.inputs.outputs.evidence_kind != 'none' }}
env:
GH_TOKEN: ${{ github.token }}
EVIDENCE_KIND: ${{ steps.inputs.outputs.evidence_kind }}
shell: bash
run: |
set -euo pipefail
run_json="${RUNNER_TEMP}/relay-asia-evidence-run.json"
verified="${RUNNER_TEMP}/relay-asia-evidence-verified"
gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${EVIDENCE_RUN_ID}/attempts/${EVIDENCE_RUN_ATTEMPT}" > "${run_json}"
evidence_commit_sha="$(
jq -er '.head_sha | select(type == "string" and test("^[a-f0-9]{40}$"))' "${run_json}"
)"
command=(node dev/scripts/relay-asia-rollout-evidence.mjs "verify-${EVIDENCE_KIND}"
--evidence "${RUNNER_TEMP}/relay-asia-input-evidence/evidence.json"
--run-json "${run_json}"
--commit-sha "${evidence_commit_sha}"
--image-digest "${IMAGE_DIGEST}"
--now "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
--output "${verified}")
if test "${EVIDENCE_KIND}" = c27; then
command+=(--selector-generation "${EXPECTED_SELECTOR_GENERATION}")
fi
"${command[@]}"
test "$(< "${verified}")" = verified
- id: auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ env.DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ env.DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: ${{ env.DIRECTOR_ORIGIN }}/v1/admin/drain
id_token_include_email: true
- name: Require the exact director image before promotion
if: ${{ inputs.mode == 'promote' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
runtime="$(curl --fail-with-body --max-time 30 --request POST \
"${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' --data '{"v":1}')"
test "$(jq -r '.role' <<< "${runtime}")" = director
test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}"
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }}
object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }}
if: ${{ inputs.mode == 'configure' || steps.inputs.outputs.canary == 'true' }}
- uses: hashicorp/setup-terraform@v3
if: ${{ inputs.mode == 'configure' }}
with:
terraform_version: 1.15.8
terraform_wrapper: false
- name: Run the exact generation-bound admission operation
id: admission-operation
if: ${{ inputs.mode != 'configure' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
run: |
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment "${TARGET_ENVIRONMENT}" \
--mode "${OPERATION_MODE}" \
--cell-ids "${TARGET_CELL_IDS}" \
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
--expected-membership-sha256 "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" \
--attempt-id "${SELECTOR_ATTEMPT_ID}" \
--image-digest "${IMAGE_DIGEST}")"
generation="$(jq -er '.generation' <<< "${result}")"
states="$(jq -cS '.states // {}' <<< "${result}")"
membership="$(jq -cS '.membership // empty' <<< "${result}")"
membership_sha256="$(jq -r '.membershipSha256 // empty' <<< "${result}")"
echo "generation=${generation}" >> "${GITHUB_OUTPUT}"
result_dir="${RUNNER_TEMP}/relay-asia-admission-result"
mkdir -p "${result_dir}"
node dev/scripts/sanitize-relay-asia-admission-result.mjs \
<<< "${result}" > "${result_dir}/result.json"
{
echo "### Relay Asia admission"
echo "- Mode: ${OPERATION_MODE}"
echo "- Cells: ${TARGET_CELL_IDS}"
echo "- Result generation: ${generation}"
echo "- States: \`${states}\`"
if test -n "${membership}"; then echo "- Membership: \`${membership}\`"; fi
if test -n "${membership_sha256}"; then
echo "- Membership SHA-256: \`${membership_sha256}\`"
fi
} >> "${GITHUB_STEP_SUMMARY}"
- name: Verify the canary cell state and start the timed canary
id: canary-start
if: ${{ steps.inputs.outputs.canary == 'true' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }}
shell: bash
run: |
set -euo pipefail
# A later cell's launch cells were checked general by the promotion; verify reads only its digest.
case "${CANARY_CELL}" in
production-gce-c27)
verify_cells=production-gce-c27,production-gce-c28,production-gce-c29
expected_states='{"production-gce-c27":"general","production-gce-c28":"migration-only","production-gce-c29":"migration-only"}'
;;
production-gce-c30|production-gce-c31|production-gce-c32|production-gce-c33|production-gce-c34|production-gce-c35)
verify_cells="${CANARY_CELL}"
expected_states="{\"${CANARY_CELL}\":\"general\"}"
;;
*) exit 1 ;;
esac
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment production \
--mode verify \
--cell-ids "${verify_cells}" \
--expected-generation "${{ steps.admission-operation.outputs.generation }}" \
--image-digest "${IMAGE_DIGEST}")"
test "$(jq -cS '.states' <<< "${result}")" = "$(jq -cS '.' <<< "${expected_states}")"
echo "verify_cells=${verify_cells}" >> "${GITHUB_OUTPUT}"
echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
- name: Run a real five-minute canary control and splice
id: canary-load
if: ${{ steps.inputs.outputs.canary == 'true' }}
env:
CANARY_REGION: ${{ steps.inputs.outputs.canary_region }}
shell: bash
run: |
set -euo pipefail
log="${RUNNER_TEMP}/relay-asia-canary-load.jsonl"
report="${RUNNER_TEMP}/relay-asia-canary-load.json"
node dev/scripts/load-relay-controls.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--auth-origin "${AUTH_ORIGIN}" \
--preferred-region "${CANARY_REGION}" \
--relay-asia-load-principals 1 \
--controls 1 \
--splices 1 \
--capacity-hard-cap 3000 \
--ramp-seconds 0 \
--duration-seconds 300 \
--splice-hold-seconds 60 \
--required-lease-horizons 2 > "${log}"
jq -cer 'select(.event == "relay_load_complete")' "${log}" | tail -n 1 > "${report}"
echo "ended_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
- name: Collect regional, Relay SQL, and Cloud SQL canary evidence
if: ${{ steps.inputs.outputs.canary == 'true' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }}
CANARY_STARTED_AT: ${{ steps.canary-start.outputs.started_at }}
CANARY_VERIFY_CELLS: ${{ steps.canary-start.outputs.verify_cells }}
shell: bash
run: |
set -euo pipefail
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment production \
--mode verify \
--cell-ids "${CANARY_VERIFY_CELLS}" \
--expected-generation "${{ steps.admission-operation.outputs.generation }}" \
--image-digest "${IMAGE_DIGEST}")"
test "$(jq -r --arg cell "${CANARY_CELL}" '.states[$cell]' <<< "${result}")" = general
ended_at="${{ steps.canary-load.outputs.ended_at }}"
sleep 60
output="${RUNNER_TEMP}/relay-asia-output-evidence"
logs="${RUNNER_TEMP}/relay-asia-canary-runtime-metrics.json"
mkdir -p "${output}"
gcloud logging read \
"timestamp>=\"${CANARY_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \
--project "${GCP_PROJECT_ID}" \
--limit 20000 \
--format json > "${logs}"
node dev/scripts/relay-asia-rollout-evidence.mjs create-canary \
--cell-id "${CANARY_CELL}" \
--repository "${GITHUB_REPOSITORY}" \
--run-id "${GITHUB_RUN_ID}" \
--run-attempt "${GITHUB_RUN_ATTEMPT}" \
--commit-sha "${GITHUB_SHA}" \
--image-digest "${IMAGE_DIGEST}" \
--selector-generation "${{ steps.admission-operation.outputs.generation }}" \
--started-at "${CANARY_STARTED_AT}" \
--ended-at "${ended_at}" \
--load-report "${RUNNER_TEMP}/relay-asia-canary-load.json" \
--logs-json "${logs}" \
--output "${output}/evidence.json"
jq -r '.metrics | to_entries[] | "- \(.key): \(.value)"' \
"${output}/evidence.json" >> "${GITHUB_STEP_SUMMARY}"
- name: Upload immutable canary evidence
id: canary-evidence-upload
if: ${{ steps.inputs.outputs.canary == 'true' }}
uses: actions/upload-artifact@v4
with:
name: relay-asia-${{ steps.inputs.outputs.canary_hostname }}-canary-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/relay-asia-output-evidence/evidence.json
if-no-files-found: error
retention-days: 7
- name: Upload sanitized admission result
if: ${{ inputs.mode != 'configure' && steps.admission-operation.outcome == 'success' }}
uses: actions/upload-artifact@v4
with:
name: relay-asia-admission-result-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/relay-asia-admission-result/result.json
if-no-files-found: error
retention-days: 7
- name: Return an unproven canary cell to migration-only
if: ${{ always() && steps.inputs.outputs.canary == 'true' && steps.admission-operation.outcome != 'skipped' && steps.canary-evidence-upload.outcome != 'success' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }}
shell: bash
run: |
set -euo pipefail
promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment production \
--mode recover-promotion \
--cell-ids "${CANARY_CELL}" \
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
--attempt-id "${SELECTOR_ATTEMPT_ID}" \
--image-digest "${IMAGE_DIGEST}")"
if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment production \
--mode rollback \
--cell-ids "${CANARY_CELL}" \
--expected-generation "${promoted_generation}" \
--attempt-id "${SELECTOR_ATTEMPT_ID}-rollback" \
--image-digest "${IMAGE_DIGEST}")"
test "$(jq -r --arg cell "${CANARY_CELL}" '.states[$cell]' <<< "${result}")" = migration-only
- name: Require registered migration-only cells before director configuration
if: ${{ inputs.mode == 'configure' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
run: |
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment "${TARGET_ENVIRONMENT}" \
--mode registered \
--cell-ids "${TARGET_CELL_IDS}" \
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
--image-digest "${IMAGE_DIGEST}")"
test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true
- name: Build the additive director cell configuration
if: ${{ inputs.mode == 'configure' }}
id: director-config
shell: bash
run: |
set -euo pipefail
terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}"
topology="${RUNNER_TEMP}/relay-asia-state-topology.json"
current="${RUNNER_TEMP}/relay-current-director-cells.json"
desired="${RUNNER_TEMP}/relay-asia-director-cells.json"
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${topology}"
service="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
revision="$(jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end' \
<<< "${service}")"
gcloud run revisions describe "${revision}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -er '.spec.containers[0].env[] | select(.name == "ORCA_RELAY_CELLS_JSON") | .value | fromjson' \
> "${current}"
node dev/scripts/prepare-relay-asia-director-cells.mjs \
--current-json "${current}" \
--topology-json "${topology}" \
--output "${desired}" \
--cell-ids "${TARGET_CELL_IDS}" \
--image-digest "${IMAGE_DIGEST}"
echo "file=${desired}" >> "${GITHUB_OUTPUT}"
- name: Deploy the registered additive director topology
if: ${{ inputs.mode == 'configure' }}
env:
DIRECTOR_CELLS_FILE: ${{ steps.director-config.outputs.file }}
run: |
current="$(gcloud secrets versions access latest \
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")"
[[ "${current}" =~ ^(true|false)$ ]]
image="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}"
release_id="asia-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
node dev/scripts/deploy-relay-blue-green.mjs \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--service "${DIRECTOR_SERVICE}" \
--image "${image}" \
--role director \
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
--release-id "${release_id}" \
--director-cells-json "$(< "${DIRECTOR_CELLS_FILE}")" \
--prune-revisions false
- name: Verify selector and heartbeats after director configuration
if: ${{ inputs.mode == 'configure' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
run: |
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment "${TARGET_ENVIRONMENT}" \
--mode verify \
--cell-ids "${TARGET_CELL_IDS}" \
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
--image-digest "${IMAGE_DIGEST}")"
test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true
revision="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end')"
revision_json="$(gcloud run revisions describe "${revision}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
jq -e --arg image "us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}" \
'.spec.containers[0].image == $image' <<< "${revision_json}" > /dev/null
jq -er --arg secret "${REGIONAL_PLACEMENT_SECRET}" \
'[.spec.containers[0].env[] |
select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
(.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
{secret: (.secret // .name), version: (.version // .key)} |
select(.secret == $secret and (.version | test("^[1-9][0-9]*$")))] |
if length == 1 then .[0].version else error("regional switch version missing") end' \
<<< "${revision_json}" > "${RUNNER_TEMP}/relay-regional-placement-version"
regional_version="$(< "${RUNNER_TEMP}/relay-regional-placement-version")"
[[ "$(gcloud secrets versions access "${regional_version}" --project "${GCP_PROJECT_ID}" \
--secret "${REGIONAL_PLACEMENT_SECRET}")" =~ ^(true|false)$ ]]
echo "Director configuration now lists the registered migration-only Asia cells." >> "${GITHUB_STEP_SUMMARY}"
@@ -0,0 +1,125 @@
name: Operate Relay Production Cell Flags
# Writes one cell's runtime switch object and waits for the cell to report it applied.
# One cell per run; never fleet-wide. dry-run writes nothing.
on:
workflow_dispatch:
inputs:
mode:
description: dry-run prints the change; write applies it
required: true
default: dry-run
type: choice
options: [dry-run, write]
cell-id:
description: Exact cell id, e.g. production-gce-c26
required: true
type: string
break-glass:
description: >-
Only for a dead or unreachable reserve cell, with set admitMode=default. Typed as
BREAK GLASS <cell-id>; records db without the cell's read-back
required: false
default: ''
type: string
confirm-reserve:
description: Required for admitMode=reserve, typed as RESERVE <cell-id>
required: false
default: ''
type: string
set:
description: >-
Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a
key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck,
admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs,
reregisterInFlight
required: true
type: string
expected-generation:
description: Exact current object generation from a dry run, or 0 if none exists
required: true
type: string
confirmation:
description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
required: false
type: string
permissions:
contents: read
id-token: write
# One write at a time across the fleet keeps the audit trail in order.
concurrency:
group: relay-production-cell-flags
cancel-in-progress: false
defaults:
run:
# Explicit bash runs with pipefail, so the script's exit code survives the tee.
shell: bash
working-directory: cloud
jobs:
operate:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
runs-on: ubuntu-latest
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
timeout-minutes: 60
environment: production
env:
MODE: ${{ inputs.mode }}
CELL_ID: ${{ inputs.cell-id }}
FLAG_CHANGES: ${{ inputs.set }}
EXPECTED_GENERATION: ${{ inputs.expected-generation }}
CONFIRMATION: ${{ inputs.confirmation }}
CONFIRM_RESERVE: ${{ inputs.confirm-reserve }}
BREAK_GLASS: ${{ inputs.break-glass }}
GCP_PROJECT_ID: onorca-cloud
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 24
- id: google-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- name: Write or preview the cell's switch object
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
node dev/scripts/operate-relay-cell-flags.mjs \
--mode "${MODE}" --cell-id "${CELL_ID}" \
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
--project-id "${GCP_PROJECT_ID}" \
--set "${FLAG_CHANGES}" \
--director-origin "https://relay.onorca.dev" --confirm-reserve "${CONFIRM_RESERVE}" \
--break-glass "${BREAK_GLASS}" \
--expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-cell-flags.jsonl"
- name: Audit summary
if: ${{ always() }}
run: |
{
echo "### Relay cell flags: ${MODE} ${CELL_ID}"
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
echo '```json'
cat "${RUNNER_TEMP}/relay-cell-flags.jsonl" 2>/dev/null || echo '(no output)'
echo '```'
} >> "${GITHUB_STEP_SUMMARY}"
@@ -0,0 +1,341 @@
name: Operate Relay Production Rehome Job
on:
workflow_call:
inputs:
mode: { required: true, type: string }
director-image-digest: { required: true, type: string }
rollback-image-digest: { required: true, type: string }
expected-selector-generation: { required: true, type: string }
expected-existing-only-cells: { required: true, type: string }
expected-migration-only-cells: { required: true, type: string }
expected-general-cells: { required: true, type: string }
expected-control-generation: { required: true, type: string }
not-before: { required: true, type: string }
rate-per-minute: { required: true, type: string }
preference-max-age-ms: { required: true, type: string }
host-cooldown-ms: { required: true, type: string }
drain-grace-ms: { required: true, type: string }
confirmation: { required: true, type: string }
monitor-run-id: { required: true, type: string }
monitor-run-attempt: { required: true, type: string }
permissions:
actions: read
contents: read
id-token: write
defaults:
run:
# `shell: bash` adds pipefail; without it `node ... | tee` reports tee's exit code and a
# thrown inspect/apply passed green (Aug 28-29 and Sep 3 2026 runs).
shell: bash
working-directory: cloud
jobs:
control:
if: ${{ github.ref == 'refs/heads/main' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 30
environment: production
env:
GCP_PROJECT_ID: onorca-cloud
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
DIRECTOR_SERVICE: orca-cloud-relay
DIRECTOR_ORIGIN: https://relay.onorca.dev
REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain
MODE: ${{ inputs.mode }}
DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }}
ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }}
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
EXPECTED_CONTROL_GENERATION: ${{ inputs.expected-control-generation }}
NOT_BEFORE: ${{ inputs.not-before }}
RATE_PER_MINUTE: ${{ inputs.rate-per-minute }}
PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }}
HOST_COOLDOWN_MS: ${{ inputs.host-cooldown-ms }}
DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }}
CONFIRMATION: ${{ inputs.confirmation }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
steps:
- name: Require exact reusable-workflow configuration
working-directory: .
env:
DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
run: |
[[ "${MODE}" =~ ^(inspect|enable|pause|disable)$ ]]
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${EXPECTED_CONTROL_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
test -n "${DEPLOY_WIF}"
test -n "${DEPLOY_SERVICE_ACCOUNT}"
if [[ "${MODE}" =~ ^(inspect|enable)$ ]]; then
[[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
[[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
test -n "${GCP_REGION}"
test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
fi
case "${MODE}" in
inspect)
test -z "${CONFIRMATION}"
;;
enable)
test "${CONFIRMATION}" = ENABLE_REGIONAL_REHOMING
test "${RATE_PER_MINUTE}" = 10
[[ "${NOT_BEFORE}" =~ ^[1-9][0-9]*$ ]]
;;
pause)
test "${CONFIRMATION}" = PAUSE_REGIONAL_REHOMING
;;
disable)
test "${CONFIRMATION}" = DISABLE_REGIONAL_REHOMING
;;
esac
# Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
# and the provenance check fails closed on a commit a shallow clone left out.
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-node@v4
with:
node-version: 24
- id: google-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Apply emergency durable pause or disable before diagnostics
if: ${{ inputs.mode == 'pause' || inputs.mode == 'disable' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
- uses: pnpm/action-setup@v4
if: ${{ inputs.mode == 'enable' }}
with: { package_json_file: cloud/package.json }
- run: pnpm install --frozen-lockfile
if: ${{ inputs.mode == 'enable' }}
- name: Download fresh aggregate safety evidence
if: ${{ inputs.mode == 'enable' }}
uses: actions/download-artifact@v4
with:
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ github.workspace }}/relay-monitor-evidence
github-token: ${{ github.token }}
run-id: ${{ inputs.monitor-run-id }}
- name: Verify enable evidence provenance
if: ${{ inputs.mode == 'enable' }}
run: |
[[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
node dev/scripts/relay-monitor-evidence.mjs verify-authority \
--directory "${OUTPUT_DIRECTORY}" \
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
--run-id "${MONITOR_RUN_ID}" --run-attempt "${MONITOR_RUN_ATTEMPT}" \
--commit-sha "${GITHUB_SHA}" --mode dry-run \
--required-migration-policy strict
- name: Reject previously consumed enable safety evidence
if: ${{ inputs.mode == 'enable' }}
env:
GH_TOKEN: ${{ github.token }}
run: |
MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
COUNT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
--jq '.total_count')"
test "${COUNT}" = 0
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
- name: Verify exact serving and rollback director identities
if: ${{ inputs.mode == 'inspect' || inputs.mode == 'enable' }}
env:
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
run: |
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
SERVING_REVISION="$(jq -er \
'[.status.traffic[] | select((.percent // 0) > 0)] |
if length == 1 and .[0].percent == 100 then .[0].revisionName
else error("director does not have one serving revision") end' \
<<< "${SERVICE_JSON}")"
ROLLBACK_REVISION="$(jq -er \
'[.status.traffic[] | select(.tag == "selector-rollback")] |
if length == 1 then .[0].revisionName else error("rollback tag missing") end' \
<<< "${SERVICE_JSON}")"
verify_revision() {
local revision="$1" expected_digest="$2"
local json
json="$(gcloud run revisions describe "${revision}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
test "$(jq -r '.spec.serviceAccountName' <<< "${json}")" = \
"${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
test "$(jq -r '.spec.containers[0].image | split("@") | last' <<< "${json}")" = \
"${expected_digest}"
test "$(jq -r '[.spec.containers[0].env[] | select(.name ==
"ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT") | .value] | if length == 1
then .[0] else empty end' <<< "${json}")" = "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
test "$(jq -r '[.spec.containers[0].env[] | select(.name ==
"ORCA_RELAY_REHOME_AUDIENCE") | .value] | if length == 1 then .[0]
else empty end' <<< "${json}")" = "${REHOME_AUDIENCE}"
}
verify_revision "${SERVING_REVISION}" "${DIRECTOR_IMAGE_DIGEST}"
verify_revision "${ROLLBACK_REVISION}" "${ROLLBACK_IMAGE_DIGEST}"
- name: Seal 24-hour aggregate region observation evidence
if: ${{ inputs.mode == 'enable' }}
run: |
mkdir -p "${RUNNER_TEMP}/relay-region-observation"
# 6 director instances x 120 samples/hour x 25h = 18000; a clipped
# read empties the oldest hourly buckets and fails the seal.
gcloud logging read \
'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND jsonPayload.event="orca_relay_runtime_metrics" AND jsonPayload.role="director"' \
--project "${GCP_PROJECT_ID}" --freshness=25h --limit=30000 --format=json \
| node dev/scripts/relay-region-observation-evidence.mjs create \
--commit-sha "${GITHUB_SHA}" \
--director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \
--selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--control-generation "${EXPECTED_CONTROL_GENERATION}" \
> "${RUNNER_TEMP}/relay-region-observation/evidence.json"
- name: Upload sealed 24-hour aggregate region evidence
if: ${{ inputs.mode == 'enable' }}
uses: actions/upload-artifact@v4
with:
name: relay-region-observation-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/relay-region-observation/evidence.json
retention-days: 90
if-no-files-found: error
- name: Verify sealed 24-hour enable authority
if: ${{ inputs.mode == 'enable' }}
run: |
node dev/scripts/relay-region-observation-evidence.mjs verify \
--file "${RUNNER_TEMP}/relay-region-observation/evidence.json" \
--commit-sha "${GITHUB_SHA}" \
--director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \
--selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--control-generation "${EXPECTED_CONTROL_GENERATION}"
- name: Recheck every aggregate safety signal before enable
if: ${{ inputs.mode == 'enable' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
pnpm incident:relay-preflight -- \
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json"
- name: Seal single-use enable safety authority
if: ${{ inputs.mode == 'enable' }}
run: |
MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
mkdir -p "${RUNNER_TEMP}/relay-rehome-enable-authority"
printf '%s\n' "${GITHUB_RUN_ID}" \
> "${RUNNER_TEMP}/relay-rehome-enable-authority/${MARKER_NAME}"
- name: Consume enable safety evidence before durable mutation
if: ${{ inputs.mode == 'enable' }}
uses: actions/upload-artifact@v4
with:
name: relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
path: ${{ runner.temp }}/relay-rehome-enable-authority/relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
retention-days: 90
if-no-files-found: error
- name: Inspect regional rehome control
if: ${{ inputs.mode == 'inspect' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
- name: Apply exact durable regional rehome enable
if: ${{ inputs.mode == 'enable' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
- name: Read fresh aggregate completion and abort evidence
run: |
gcloud logging read \
'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND textPayload:"[orca-relay] regional rehome inventory"' \
--project "${GCP_PROJECT_ID}" --freshness=15m --limit=20 --format=json \
| node dev/scripts/relay-rehome-aggregate-evidence.mjs --max-age-ms 900000 \
| tee "${RUNNER_TEMP}/relay-rehome-inventory.json"
- name: Publish aggregate control evidence
run: |
{
echo '### Regional rehome control'
jq -r '"- mode: `\(.mode)`\n- generation: `\(.control.generation)`\n- enabled: `\(.control.enabled)`"' \
"${RUNNER_TEMP}/relay-rehome-control.json"
jq -r '"- active: `\(.active)`\n- awaiting receipt: `\(.awaitingReceipt)`\n- target registered: `\(.targetRegistered)`\n- completed (24h): `\(.completedLast24Hours)`\n- aborted (24h): `\(.abortedLast24Hours)`\n- host not arrived (24h): `\(.hostNotArrivedLast24Hours // "not reported")`\n- oldest active age (ms): `\(.oldestActiveAgeMs // "none")`"' \
"${RUNNER_TEMP}/relay-rehome-inventory.json"
} >> "${GITHUB_STEP_SUMMARY}"
- name: Fail closed after an unsuccessful enable run
if: ${{ failure() && inputs.mode == 'enable' && steps.google-auth.outcome == 'success' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode recover-enable --director-origin "${DIRECTOR_ORIGIN}" \
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
--confirmation RECOVER_FAILED_REGIONAL_REHOME_ENABLE \
| tee "${RUNNER_TEMP}/relay-rehome-enable-recovery.json"
jq -e \
'.mode == "recover-enable" and .control.enabled == false' \
"${RUNNER_TEMP}/relay-rehome-enable-recovery.json" >/dev/null
@@ -0,0 +1,112 @@
name: Operate Relay Production Rehome
on:
workflow_dispatch:
inputs:
mode:
description: Inspect or apply the durable regional-rehome switch
required: true
default: inspect
type: choice
options: [inspect, enable, pause, disable]
director-image-digest:
description: Exact immutable serving director digest
required: true
type: string
rollback-image-digest:
description: Exact immutable selector-rollback director digest
required: true
type: string
expected-selector-generation:
description: Exact admission selector generation
required: true
type: string
expected-existing-only-cells:
description: Exact existing-only membership, or none
required: true
type: string
expected-migration-only-cells:
description: Exact migration-only membership, or none
required: true
type: string
expected-general-cells:
description: Exact general membership, or none
required: true
type: string
expected-control-generation:
description: Exact durable rehome generation
required: true
type: string
not-before:
description: Exact epoch milliseconds; ignored only by inspect
required: true
default: '0'
type: string
rate-per-minute:
description: Exact global host rate; initial enable is fixed at 10
required: true
default: '10'
type: string
preference-max-age-ms:
description: Maximum fresh preference age
required: true
default: '86400000'
type: string
host-cooldown-ms:
description: Minimum gap between two rehomes of the same host
required: true
default: '604800000'
type: string
drain-grace-ms:
description: Per-host source drain grace
required: true
default: '3600000'
type: string
monitor-run-id:
description: Fresh successful aggregate dry-run required only by enable
required: false
type: string
monitor-run-attempt:
description: Exact monitor attempt required only by enable
required: false
type: string
confirmation:
description: ENABLE_REGIONAL_REHOMING, PAUSE_REGIONAL_REHOMING, or DISABLE_REGIONAL_REHOMING
required: false
type: string
permissions:
actions: read
contents: read
id-token: write
concurrency:
group: production-cloud-sql-rollout
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
operate:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
uses: ./.github/workflows/cloud-operate-relay-production-rehome-job.yml
with:
mode: ${{ inputs.mode }}
director-image-digest: ${{ inputs.director-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-control-generation: ${{ inputs.expected-control-generation }}
not-before: ${{ inputs.not-before }}
rate-per-minute: ${{ inputs.rate-per-minute }}
preference-max-age-ms: ${{ inputs.preference-max-age-ms }}
host-cooldown-ms: ${{ inputs.host-cooldown-ms }}
drain-grace-ms: ${{ inputs.drain-grace-ms }}
confirmation: ${{ inputs.confirmation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
secrets: inherit
@@ -0,0 +1,327 @@
name: Operate Relay Production Reserve Watch
# Watches one cell already switched to reserve mode and flips it back to db on the first stop
# rule it trips (step 5 design §9). Manual only. dry-run evaluates and logs, writes nothing.
# An operator watches the run. A refused watch or a flip back that did not land raises an ::error::
# annotation, as the clock-skew monitor does, and in write mode logs the line the reserve-watch
# alert pages on. Refuses to start while a rollout holds the Cloud SQL rollout lease.
on:
workflow_dispatch:
inputs:
mode:
description: dry-run logs what a trip would write; write flips the cell back on a trip
required: true
default: dry-run
type: choice
options: [dry-run, write]
cell-id:
description: Exact cell id already in reserve mode, e.g. production-gce-c26
required: true
type: string
duration-minutes:
description: How long to watch, 5 to 300
required: true
default: '120'
type: string
flipped-at:
description: >-
When the reserve write applied, ISO with a zone, e.g. 2026-10-14T12:00:00Z. The 503
background is the median of the 10 minutes before it
required: false
default: ''
type: string
background-503-per-minute:
description: >-
Director 503s per minute before the flip; overrides the one measured before flipped-at.
One of the two is required
required: false
default: ''
type: string
unverifiable-limit:
description: Consecutive samples with any unreadable signal that count as a trip
required: true
default: '3'
type: string
confirmation:
description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
required: false
type: string
permissions:
contents: read
id-token: write
# One watch per cell; why not the flag workflow's group: cloud/docs/relay-workflows.md.
concurrency:
group: relay-production-reserve-watch-${{ inputs.cell-id }}
cancel-in-progress: false
defaults:
run:
# Explicit bash runs with pipefail, so the script's exit code survives the tee.
shell: bash
working-directory: cloud
env:
MODE: ${{ inputs.mode }}
CELL_ID: ${{ inputs.cell-id }}
CONFIRMATION: ${{ inputs.confirmation }}
GCP_PROJECT_ID: onorca-cloud
jobs:
watch:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
runs-on: ubuntu-latest
# Six 55-minute segments cover the longest watch; under the 360-minute job limit.
timeout-minutes: 350
environment: production
outputs:
state: ${{ steps.outcome.outputs.state }}
verdict: ${{ steps.outcome.outputs.verdict }}
env:
DURATION_MINUTES: ${{ inputs.duration-minutes }}
FLIPPED_AT: ${{ inputs.flipped-at }}
BACKGROUND_503_PER_MINUTE: ${{ inputs.background-503-per-minute }}
UNVERIFIABLE_LIMIT: ${{ inputs.unverifiable-limit }}
# One segment, after the auth step before it: fixed text, the inputs arrive as env vars.
WATCH_SEGMENT: |
set -euo pipefail
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
node dev/scripts/watch-relay-reserve-cell.mjs watch \
--mode "${MODE}" --cell-id "${CELL_ID}" \
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
--project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \
--director-service orca-cloud-relay --confirmation "${CONFIRMATION}" \
--duration-minutes "${DURATION_MINUTES}" --segment-minutes 55 \
--flipped-at "${FLIPPED_AT}" --background-503-per-minute "${BACKGROUND_503_PER_MINUTE}" \
--unverifiable-limit "${UNVERIFIABLE_LIMIT}" \
--state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}" \
| tee -a "${RUNNER_TEMP}/relay-reserve-watch.jsonl"
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-node@v4
with:
node-version: 24
- id: google-auth-1
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- id: watch-1
name: Watch segment 1 of 6
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-1.outputs.id_token }}
run: bash -c "${WATCH_SEGMENT}"
- id: google-auth-2
if: ${{ steps.watch-1.outputs.state == 'watching' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- id: watch-2
name: Watch segment 2 of 6
if: ${{ steps.watch-1.outputs.state == 'watching' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-2.outputs.id_token }}
run: bash -c "${WATCH_SEGMENT}"
- id: google-auth-3
if: ${{ steps.watch-2.outputs.state == 'watching' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- id: watch-3
name: Watch segment 3 of 6
if: ${{ steps.watch-2.outputs.state == 'watching' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-3.outputs.id_token }}
run: bash -c "${WATCH_SEGMENT}"
- id: google-auth-4
if: ${{ steps.watch-3.outputs.state == 'watching' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- id: watch-4
name: Watch segment 4 of 6
if: ${{ steps.watch-3.outputs.state == 'watching' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-4.outputs.id_token }}
run: bash -c "${WATCH_SEGMENT}"
- id: google-auth-5
if: ${{ steps.watch-4.outputs.state == 'watching' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- id: watch-5
name: Watch segment 5 of 6
if: ${{ steps.watch-4.outputs.state == 'watching' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-5.outputs.id_token }}
run: bash -c "${WATCH_SEGMENT}"
- id: google-auth-6
if: ${{ steps.watch-5.outputs.state == 'watching' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- id: watch-6
name: Watch segment 6 of 6
if: ${{ steps.watch-5.outputs.state == 'watching' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-6.outputs.id_token }}
run: bash -c "${WATCH_SEGMENT}"
- id: outcome
name: Record the watch outcome
if: ${{ always() }}
run: |
node dev/scripts/watch-relay-reserve-cell.mjs outcome \
--state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}"
# The documented flow writes reserve before dispatching, so a refusal can leave it unwatched.
- name: Alarm on a refused watch
if: ${{ always() && steps.outcome.outputs.state == 'refused' }}
env:
WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }}
run: |
node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind refused \
--mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${WATCH_VERDICT}"
- name: Audit summary
if: ${{ always() }}
env:
WATCH_STATE: ${{ steps.outcome.outputs.state }}
WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }}
run: |
{
echo "### Relay reserve watch: ${MODE} ${CELL_ID}"
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
echo "Outcome: ${WATCH_STATE}: ${WATCH_VERDICT}"
echo '```json'
tail -n 20 "${RUNNER_TEMP}/relay-reserve-watch.jsonl" 2>/dev/null || echo '(no output)'
echo '```'
} >> "${GITHUB_STEP_SUMMARY}"
flip-back:
needs: watch
# Missing data is never healthy: a watch that crashed, timed out or was cancelled (by an
# operator too) flips back like a trip. In dry-run this only plans the write.
if: ${{ always() && vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && needs.watch.result != 'skipped' && !contains(fromJSON('["completed","stopped","refused"]'), needs.watch.outputs.state) }}
runs-on: ubuntu-latest
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
timeout-minutes: 60
environment: production
# Only the run's per-cell group above, which a running run never yields (cancel-in-progress off).
# Every step runs on always() too, so an ordinary cancel cannot skip the switch-file write; a force-cancel
# can, and leaves the cell in reserve for the operator to flip back by hand. A cancel during the
# lease wait can still kill the process before Postgres records db: the file says db, Postgres reserve.
env:
VERDICT: ${{ needs.watch.outputs.verdict || format('the watch ended {0} without a verdict', needs.watch.result) }}
steps:
- uses: actions/checkout@v4
if: ${{ always() }}
with:
persist-credentials: false
- uses: actions/setup-node@v4
if: ${{ always() }}
with:
node-version: 24
- id: google-auth
if: ${{ always() }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
if: ${{ always() }}
with:
skip_install: true
- name: Flip the cell back to db, or preview it
if: ${{ always() }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
node dev/scripts/watch-relay-reserve-cell.mjs flip-back \
--mode "${MODE}" --cell-id "${CELL_ID}" \
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
--project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \
--confirmation "${CONFIRMATION}" --verdict "${VERDICT}" \
| tee "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"
# The flip-back line is logged only once the write (or its dry-run plan) finished; without
# it a missing token, a failed login or a silent cell left the cell in reserve, unwatched.
- name: Alarm when the flip back did not land
if: ${{ always() }}
run: |
if grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"; then
exit 0
fi
{
echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode"
echo "Flip it to db by hand with Operate Relay Production Cell Flags."
} >> "${GITHUB_STEP_SUMMARY}"
node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind flip-back-failed \
--mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}"
- name: Audit summary
if: ${{ always() }}
run: |
{
echo "### Relay reserve watch flip back: ${MODE} ${CELL_ID}"
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
echo "Verdict: ${VERDICT}"
echo '```json'
cat "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl" 2>/dev/null || echo '(no output)'
echo '```'
} >> "${GITHUB_STEP_SUMMARY}"
@@ -0,0 +1,127 @@
name: Operate Relay Staging Cell Flags
# The staging twin of Operate Relay Production Cell Flags: writes one staging cell's runtime
# switch object and waits for the cell to report it applied. One cell per run. dry-run writes
# nothing.
on:
workflow_dispatch:
inputs:
mode:
description: dry-run prints the change; write applies it
required: true
default: dry-run
type: choice
options: [dry-run, write]
cell-id:
description: Exact cell id, e.g. staging-gce-c2
required: true
type: string
break-glass:
description: >-
Only for a dead or unreachable reserve cell, with set admitMode=default. Typed as
BREAK GLASS <cell-id>; records db without the cell's read-back
required: false
default: ''
type: string
confirm-reserve:
description: Required for admitMode=reserve, typed as RESERVE <cell-id>
required: false
default: ''
type: string
set:
description: >-
Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a
key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck,
admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs,
reregisterInFlight
required: true
type: string
expected-generation:
description: Exact current object generation from a dry run, or 0 if none exists
required: true
type: string
confirmation:
description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
required: false
type: string
permissions:
contents: read
id-token: write
# Not relay-staging-mutation: a switch write must run while a staging deploy or load run is
# in flight, and it touches neither Cloud SQL nor a revision.
concurrency:
group: relay-staging-cell-flags
cancel-in-progress: false
defaults:
run:
# Explicit bash runs with pipefail, so the script's exit code survives the tee.
shell: bash
working-directory: cloud
jobs:
operate:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
runs-on: ubuntu-latest
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
timeout-minutes: 60
environment: staging
env:
MODE: ${{ inputs.mode }}
CELL_ID: ${{ inputs.cell-id }}
FLAG_CHANGES: ${{ inputs.set }}
EXPECTED_GENERATION: ${{ inputs.expected-generation }}
CONFIRMATION: ${{ inputs.confirmation }}
CONFIRM_RESERVE: ${{ inputs.confirm-reserve }}
BREAK_GLASS: ${{ inputs.break-glass }}
GCP_PROJECT_ID: onorca-cloud-staging
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 24
- id: google-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- name: Write or preview the cell's switch object
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
[[ "${CELL_ID}" =~ ^staging-gce-c[0-9]{1,3}$ ]]
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
node dev/scripts/operate-relay-cell-flags.mjs \
--mode "${MODE}" --cell-id "${CELL_ID}" \
--cell-origin "https://${CELL_ID#staging-gce-}.relay-staging.onorca.dev" \
--project-id "${GCP_PROJECT_ID}" \
--set "${FLAG_CHANGES}" \
--director-origin "https://relay-staging.onorca.dev" --confirm-reserve "${CONFIRM_RESERVE}" \
--break-glass "${BREAK_GLASS}" \
--expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-cell-flags.jsonl"
- name: Audit summary
if: ${{ always() }}
run: |
{
echo "### Relay staging cell flags: ${MODE} ${CELL_ID}"
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
echo '```json'
cat "${RUNNER_TEMP}/relay-cell-flags.jsonl" 2>/dev/null || echo '(no output)'
echo '```'
} >> "${GITHUB_STEP_SUMMARY}"
@@ -0,0 +1,103 @@
name: Power Relay Staging
on:
schedule:
# A zero-activity guard makes this a no-op when an internal test is still running.
- cron: '0 9 * * *'
workflow_dispatch:
inputs:
mode:
description: Inspect, wake, or sleep the staging Relay data plane
required: true
default: status
type: choice
options:
- status
- wake
- sleep
wake-cells:
description: Wake configured admission cells, or include disabled candidate cells
required: true
default: configured
type: choice
options:
- configured
- all
confirmation:
description: Enter WAKE_STAGING or SLEEP_STAGING for a manual mutation
required: false
type: string
permissions:
contents: read
id-token: write
concurrency:
group: relay-staging-mutation
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
power:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: staging
env:
POWER_MODE: ${{ github.event_name == 'schedule' && 'sleep' || inputs.mode }}
WAKE_CELLS: ${{ inputs.wake-cells || 'configured' }}
steps:
- uses: actions/checkout@v4
- id: google-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-staging-terraform-state
object: terraform/state/cloud-sql-rollout/staging.lock
- uses: hashicorp/setup-terraform@v3
with:
terraform_wrapper: false
- uses: actions/setup-node@v4
with:
node-version: 24
- name: Require explicit manual mutation confirmation
if: ${{ github.event_name == 'workflow_dispatch' && inputs.mode != 'status' }}
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: |
if [[ "${POWER_MODE}" = "wake" ]]; then
test "${CONFIRMATION}" = "WAKE_STAGING"
else
test "${CONFIRMATION}" = "SLEEP_STAGING"
fi
- name: Read reviewed staging topology
run: |
node dev/scripts/infra.mjs init --env staging
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
- name: Inspect or change staging power state
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/power-staging-relay.mjs \
--mode "${POWER_MODE}" \
--wake-cells "${WAKE_CELLS}" \
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json"
@@ -0,0 +1,497 @@
name: Prove Relay Asia Staging
on:
workflow_dispatch:
inputs:
mode:
description: prove runs the fixed C4 proof; load runs the generator with free-form arguments
required: true
default: prove
type: choice
options: [prove, load]
image-digest:
description: Prove only. Exact immutable Relay digest deployed on staging C4
required: false
type: string
selector-generation:
description: Prove only. Exact selector generation with C4 migration-only
required: false
type: string
promote-attempt-id:
description: Prove only. Durable unique C4 promotion attempt ID
required: false
type: string
rollback-attempt-id:
description: Prove only. Durable unique C4 rollback attempt ID
required: false
type: string
load-region:
description: >-
Load only. asia-east2 runs on the Asia load runner; asia-east2-hosted asks for asia-east2
from a hosted runner; us-central1 runs on a hosted runner
required: false
default: asia-east2
type: choice
options: [asia-east2, asia-east2-hosted, us-central1]
load-shards:
description: Load only. Generator processes, each with its own shard of the 32 load principals
required: false
default: '4'
type: choice
options: ['1', '2', '3', '4']
load-runners:
description: >-
Load only, hosted runners only. Runner machines, each with load-shards generators; one
hosted runner holds about 2,000 connections to one cell
required: false
default: '1'
type: choice
options: ['1', '2']
load-args:
description: >-
Load only. Per-shard generator arguments, e.g. --controls 150 --splices 15
--reconnect-max-seconds 30 --duration-seconds 900. Origins, region, principals and
sharding are set by the workflow
required: false
default: ''
type: string
confirmation:
description: Enter PROVE_ASIA_STAGING (prove) or LOAD_RELAY_STAGING (load)
required: true
type: string
permissions:
contents: read
id-token: write
# A load run drives traffic only, so it must not queue behind (or hold up) the staging deploys
# and switch writes it is there to exercise.
concurrency:
group: ${{ inputs.mode == 'load' && 'relay-staging-load' || 'relay-staging-mutation' }}
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
prove:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') && inputs.mode == 'prove' }}
runs-on: [self-hosted, linux, x64, relay-asia-east2-load]
timeout-minutes: 75
environment: staging
env:
GCP_PROJECT_ID: onorca-cloud-staging
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
AUTH_ORIGIN: https://auth-staging.onorca.dev
IMAGE_DIGEST: ${{ inputs.image-digest }}
INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }}
ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }}
steps:
- uses: actions/checkout@v4
- name: Validate the exact staging proof request
shell: bash
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: |
set -euo pipefail
test "${CONFIRMATION}" = PROVE_ASIA_STAGING
[[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
[[ "${INITIAL_SELECTOR_GENERATION}" =~ ^[1-9][0-9]*$ ]]
[[ "${PROMOTE_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
[[ "${ROLLBACK_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
test "${PROMOTE_ATTEMPT_ID}" != "${ROLLBACK_ATTEMPT_ID}"
- id: auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-staging-terraform-state
object: terraform/state/cloud-sql-rollout/staging.lock
- uses: actions/setup-node@v4
with:
node-version: 24
- uses: pnpm/action-setup@v4
with:
package_json_file: cloud/package.json
- name: Require the exact staging director image before promotion
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
runtime="$(curl --fail-with-body --max-time 30 --request POST \
"${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' --data '{"v":1}')"
test "$(jq -r '.role' <<< "${runtime}")" = director
test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}"
- name: Install exact load-harness dependencies
run: pnpm install --frozen-lockfile
- name: Build the Relay load-harness contract
run: pnpm --filter @orca-cloud/relay-contract build
- name: Promote only staging C4
id: promote
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment staging \
--mode promote \
--cell-ids staging-gce-c4 \
--expected-generation "${INITIAL_SELECTOR_GENERATION}" \
--attempt-id "${PROMOTE_ATTEMPT_ID}" \
--image-digest "${IMAGE_DIGEST}")"
generation="$(jq -er '.generation' <<< "${result}")"
test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = general
echo "generation=${generation}" >> "${GITHUB_OUTPUT}"
echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
- name: Run sharded two-horizon and mixed splice proofs
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof"
mkdir -p "${proof_dir}"
fd_limit="$(ulimit -n)"
if test "${fd_limit}" != unlimited; then
[[ "${fd_limit}" =~ ^[0-9]+$ ]]
test "${fd_limit}" -ge 4096
fi
run_phase() {
phase="$1"
controls="$2"
splices="$3"
pids=()
stop_shards() {
for pid in "${pids[@]}"; do kill "${pid}" 2>/dev/null || true; done
for pid in "${pids[@]}"; do wait "${pid}" 2>/dev/null || true; done
}
trap stop_shards EXIT
for shard in 0 1 2 3; do
slow=0
wedged=0
boundary_args=()
request_unit_args=()
if test "${phase}" = launch && test "${shard}" = 0; then
slow=4
wedged=1
fi
if test "${phase}" = launch && test "${shard}" = 0; then
boundary_args=(
--region-behavior-probes 1
--capacity-cell-id staging-gce-c4
--capacity-cell-origin https://c4.relay-staging.onorca.dev
--capacity-unobserved-bound 60
--rebind-probes 2
--skip-rebind-overflow-check
)
fi
node dev/scripts/load-relay-controls.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--auth-origin "${AUTH_ORIGIN}" \
--preferred-region asia-east2 \
--relay-asia-load-principals 32 \
--controls "${controls}" \
--splices "${splices}" \
--slow-reader-splices "${slow}" \
--wedged-reader-splices "${wedged}" \
--capacity-hard-cap 3000 \
"${boundary_args[@]}" \
"${request_unit_args[@]}" \
--phase-barrier-dir "${proof_dir}/${phase}-barrier" \
--aggregate-controls "$((controls * 4))" \
--aggregate-splices "$((splices * 4))" \
--aggregate-reader-splices "$([[ "${phase}" = launch ]] && echo 5 || echo 0)" \
--aggregate-reader-bytes "$([[ "${phase}" = launch ]] && echo 12582912 || echo 0)" \
--required-lease-horizons 2 \
--splice-ramp-seconds 120 \
--max-generator-rss-growth-mib 512 \
--ramp-seconds 180 \
--duration-seconds 210 \
--shard-count 4 \
--shard-index "${shard}" \
> "${proof_dir}/${phase}-${shard}.jsonl" &
pids+=("$!")
done
failed=0
for pid in "${pids[@]}"; do
if ! wait "${pid}"; then failed=1; break; fi
done
if test "${failed}" = 1; then
stop_shards
for shard in 0 1 2 3; do
jq -cer 'select(.event == "relay_load_progress" or .event == "relay_load_complete") |
{event, shardIndex, active, peakActive, connected, connectionFailures,
rampConnectionFailures, connectionFailuresByReason, unexpectedCloses,
protocolErrors, refreshErrors, socketErrors, elapsedSeconds}' \
"${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1 || true
done
trap - EXIT
return 1
fi
trap - EXIT
for shard in 0 1 2 3; do
jq -cer 'select(.event == "relay_load_complete")' \
"${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1
done | jq -s . > "${proof_dir}/${phase}.json"
}
run_phase launch 5 5
- name: Collect and validate aggregate staging proof evidence
env:
PROOF_STARTED_AT: ${{ steps.promote.outputs.started_at }}
shell: bash
run: |
set -euo pipefail
proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof"
ended_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
sleep 60
gcloud logging read \
"timestamp>=\"${PROOF_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \
--project "${GCP_PROJECT_ID}" --limit 20000 --format json \
> "${proof_dir}/runtime-metrics.json"
node dev/scripts/relay-asia-rollout-evidence.mjs create-staging \
--repository "${GITHUB_REPOSITORY}" \
--run-id "${GITHUB_RUN_ID}" \
--run-attempt "${GITHUB_RUN_ATTEMPT}" \
--commit-sha "${GITHUB_SHA}" \
--image-digest "${IMAGE_DIGEST}" \
--selector-generation "${{ steps.promote.outputs.generation }}" \
--started-at "${PROOF_STARTED_AT}" \
--ended-at "${ended_at}" \
--launch-report "${proof_dir}/launch.json" \
--logs-json "${proof_dir}/runtime-metrics.json" \
--output "${proof_dir}/evidence.json"
- name: Return staging C4 to migration-only
if: ${{ always() && steps.promote.outcome != 'skipped' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment staging \
--mode recover-promotion \
--cell-ids staging-gce-c4 \
--expected-generation "${INITIAL_SELECTOR_GENERATION}" \
--attempt-id "${PROMOTE_ATTEMPT_ID}" \
--image-digest "${IMAGE_DIGEST}")"
if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment staging \
--mode rollback \
--cell-ids staging-gce-c4 \
--expected-generation "${promoted_generation}" \
--attempt-id "${ROLLBACK_ATTEMPT_ID}" \
--image-digest "${IMAGE_DIGEST}")"
test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only
- name: Upload immutable staging readiness evidence
if: ${{ success() }}
uses: actions/upload-artifact@v4
with:
name: relay-asia-staging-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/relay-asia-staging-proof/evidence.json
if-no-files-found: error
retention-days: 7
recover:
if: ${{ always() && github.ref == 'refs/heads/main' && inputs.mode == 'prove' }}
needs: prove
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 15
environment: staging
env:
IMAGE_DIGEST: ${{ inputs.image-digest }}
INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }}
ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }}
steps:
- uses: actions/checkout@v4
- id: auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: actions/setup-node@v4
with:
node-version: 24
- name: Recover staging C4 with a fresh identity
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment staging \
--mode recover-promotion \
--cell-ids staging-gce-c4 \
--expected-generation "${INITIAL_SELECTOR_GENERATION}" \
--attempt-id "${PROMOTE_ATTEMPT_ID}" \
--image-digest "${IMAGE_DIGEST}")"
if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment staging \
--mode rollback \
--cell-ids staging-gce-c4 \
--expected-generation "${promoted_generation}" \
--attempt-id "${ROLLBACK_ATTEMPT_ID}" \
--image-digest "${IMAGE_DIGEST}")"
test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only
# The staging auth service mints relay load principals only for this workflow file, so the
# free-form staging load lives here rather than in a workflow of its own.
load:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') && inputs.mode == 'load' }}
runs-on: ${{ inputs.load-region == 'asia-east2' && fromJSON('["self-hosted","linux","x64","relay-asia-east2-load"]') || 'ubuntu-latest' }}
strategy:
# One runner's failure must not drop the other half of the load mid-run.
fail-fast: false
matrix:
runner: ${{ fromJSON(inputs.load-runners == '2' && '[0, 1]' || '[0]') }}
timeout-minutes: 180
environment: staging
env:
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
AUTH_ORIGIN: https://auth-staging.onorca.dev
LOAD_REGION: ${{ inputs.load-region }}
LOAD_SHARDS: ${{ inputs.load-shards }}
LOAD_RUNNERS: ${{ inputs.load-runners }}
LOAD_RUNNER_INDEX: ${{ matrix.runner }}
LOAD_ARGS: ${{ inputs.load-args }}
steps:
- uses: actions/checkout@v4
- name: Validate the staging load request
shell: bash
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: |
set -euo pipefail
test "${CONFIRMATION}" = LOAD_RELAY_STAGING
[[ "${LOAD_REGION}" =~ ^(asia-east2|asia-east2-hosted|us-central1)$ ]]
[[ "${LOAD_SHARDS}" =~ ^[1-4]$ ]]
[[ "${LOAD_RUNNERS}" =~ ^[12]$ ]]
# The Asia runner is one machine; a second matrix job would only queue behind the first.
if test "${LOAD_RUNNERS}" != 1 && test "${LOAD_REGION}" = asia-east2; then
echo "load-runners 2 needs a hosted runner" >&2
exit 1
fi
read -r -a args <<< "${LOAD_ARGS}"
for arg in "${args[@]}"; do
[[ "${arg}" =~ ^[A-Za-z0-9=:/._,-]+$ ]] || { echo "unsupported characters in load-args" >&2; exit 1; }
case "${arg}" in
--target-origin|--director-origin|--auth-origin|--signing-key-file|--preferred-region|\
--relay-asia-load-principals|--shard-count|--shard-index|--phase-barrier-dir)
echo "${arg} is set by the workflow" >&2
exit 1
;;
esac
done
- id: auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: actions/setup-node@v4
with:
node-version: 24
- uses: pnpm/action-setup@v4
with:
package_json_file: cloud/package.json
- name: Install exact load-harness dependencies
run: pnpm install --frozen-lockfile
- name: Build the Relay load-harness contract
run: pnpm --filter @orca-cloud/relay-contract build
- name: Run the sharded staging load
env:
# Probes that read director capacity need an admin token; it lasts about an hour.
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
load_dir="${RUNNER_TEMP}/relay-staging-load"
mkdir -p "${load_dir}"
fd_limit="$(ulimit -n)"
if test "${fd_limit}" != unlimited; then
[[ "${fd_limit}" =~ ^[0-9]+$ ]]
test "${fd_limit}" -ge 4096
fi
read -r -a args <<< "${LOAD_ARGS}"
pids=()
stop_shards() {
for pid in "${pids[@]}"; do kill "${pid}" 2>/dev/null || true; done
for pid in "${pids[@]}"; do wait "${pid}" 2>/dev/null || true; done
}
trap stop_shards EXIT
for ((shard = 0; shard < LOAD_SHARDS; shard++)); do
node dev/scripts/load-relay-controls.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--auth-origin "${AUTH_ORIGIN}" \
--preferred-region "${LOAD_REGION%-hosted}" \
--relay-asia-load-principals 32 \
--shard-count "$((LOAD_SHARDS * LOAD_RUNNERS))" \
--shard-index "$((LOAD_RUNNER_INDEX * LOAD_SHARDS + shard))" \
"${args[@]}" \
> "${load_dir}/shard-${shard}.jsonl" &
pids+=("$!")
done
failed=0
for pid in "${pids[@]}"; do
if ! wait "${pid}"; then failed=1; fi
done
trap - EXIT
for ((shard = 0; shard < LOAD_SHARDS; shard++)); do
jq -c 'select(.event == "relay_load_complete" or .event == "relay_load_progress")' \
"${load_dir}/shard-${shard}.jsonl" | tail -n 1 || true
done
exit "${failed}"
- name: Upload the load reports
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: relay-staging-load-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.runner }}
path: ${{ runner.temp }}/relay-staging-load/
if-no-files-found: warn
retention-days: 7
@@ -0,0 +1,926 @@
name: Prove Relay Staging Capacity
on:
workflow_dispatch:
inputs:
mode:
description: Verify or change C3 capacity, restore admission, or refresh empty Asia C4
required: true
default: verify
type: choice
options:
- verify
- apply
- restore-admission
- refresh-asia-c4-image
expected-hard-cap:
description: Exact cap declared for staging-gce-c3 in the reviewed staging tfvars
required: true
default: '600'
type: choice
options:
- '1000'
- '600'
expected-unobserved-bound:
description: Exact bound declared for staging-gce-c3 in the reviewed staging tfvars
required: true
default: '60'
type: choice
options:
- '60'
- '0'
confirmation:
description: Exact confirmation required for a mutation
required: false
type: string
expected-selector-generation:
description: Exact staging selector generation for a C4 image refresh
required: false
type: string
predecessor-image-digest:
description: Exact current C4 sha256 digest
required: false
type: string
target-image-digest:
description: Exact desired C4 sha256 digest
required: false
type: string
permissions:
contents: read
id-token: write
concurrency:
group: relay-staging-mutation
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
capacity:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && inputs.mode != 'refresh-asia-c4-image' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: staging
env:
GCP_PROJECT_ID: onorca-cloud-staging
GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
CELL_ORIGIN: https://c3.relay-staging.onorca.dev
# These drains are unpaced, but restart-safe still waits out the production pace window
# from the drain, the same one the same-cap drain uses.
RESTART_SAFE_PACE_WINDOW_MS: '300000'
TARGET_CELL_ID: staging-gce-c3
FALLBACK_CELL_ORIGIN: https://c2.relay-staging.onorca.dev
FALLBACK_CELL_ID: staging-gce-c2
CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
EXPECTED_HARD_CAP: ${{ inputs.expected-hard-cap }}
EXPECTED_UNOBSERVED_BOUND: ${{ inputs.expected-unobserved-bound }}
steps:
- uses: actions/checkout@v4
- id: google-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-staging-terraform-state
object: terraform/state/cloud-sql-rollout/staging.lock
- uses: hashicorp/setup-terraform@v3
if: ${{ inputs.mode != 'restore-admission' }}
with:
terraform_wrapper: false
- uses: actions/setup-node@v4
with:
node-version: 24
- name: Initialize the exact staging backend
if: ${{ inputs.mode != 'restore-admission' }}
run: node dev/scripts/infra.mjs init --env staging
- name: Require reviewed desired capacity and image
if: ${{ inputs.mode != 'restore-admission' }}
shell: bash
run: |
CAP_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_hard_cap"
BOUND_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_unobserved_bound"
IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image"
ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone"
DESIRED_CAP="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars <<< "${CAP_EXPRESSION}")"
DESIRED_BOUND="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars <<< "${BOUND_EXPRESSION}")"
DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars <<< "${IMAGE_EXPRESSION}" | jq -r '.')"
TARGET_ZONE="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars <<< "${ZONE_EXPRESSION}" | jq -r '.')"
MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
| jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars \
<<< 'local.relay_director_cells_json' | jq -r '.')"
CELL_ORIGIN="$(jq -r --arg cell "${TARGET_CELL_ID}" \
'.[] | select(.id == $cell) | .url' <<< "${DESIRED_CELLS_JSON}")"
test "${DESIRED_CAP}" = "${EXPECTED_HARD_CAP}"
test "${DESIRED_BOUND}" = "${EXPECTED_UNOBSERVED_BOUND}"
[[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]]
[[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]]
[[ "${MIG_NAME}" =~ ^[a-z0-9-]+$ ]]
test "${CELL_ORIGIN}" = "https://c3.relay-staging.onorca.dev"
jq -e \
--arg cell "${TARGET_CELL_ID}" \
--arg fallback "${FALLBACK_CELL_ID}" \
--argjson cap "${EXPECTED_HARD_CAP}" \
--argjson bound "${EXPECTED_UNOBSERVED_BOUND}" \
'(any(.[]; .id == $cell and .connectionHardCap == $cap and .connectionUnobservedBound == $bound)) and
(any(.[]; .id == $fallback and .connectionHardCap == 600 and .connectionUnobservedBound == 60))' \
<<< "${DESIRED_CELLS_JSON}" >/dev/null
echo "DESIRED_IMAGE=${DESIRED_IMAGE}" >> "${GITHUB_ENV}"
echo "TARGET_ZONE=${TARGET_ZONE}" >> "${GITHUB_ENV}"
echo "MIG_NAME=${MIG_NAME}" >> "${GITHUB_ENV}"
echo "CELL_ORIGIN=${CELL_ORIGIN}" >> "${GITHUB_ENV}"
echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" >> "${GITHUB_ENV}"
- name: Verify exact compatible director image
if: ${{ inputs.mode != 'restore-admission' }}
shell: bash
run: |
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
ACTIVE_REVISION="$(jq -r \
'[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \
<<< "${SERVICE_JSON}")"
test -n "${ACTIVE_REVISION}"
ACTIVE_IMAGE="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format='value(spec.containers[0].image)')"
test "${ACTIVE_IMAGE}" = "${DESIRED_IMAGE}"
echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}" >> "${GITHUB_ENV}"
- name: Require explicit transition confirmation
if: ${{ inputs.mode == 'apply' }}
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: test "${CONFIRMATION}" = "FENCE_AND_TRANSITION_STAGING_C3"
- name: Require explicit admission restore confirmation
if: ${{ inputs.mode == 'restore-admission' }}
env:
CONFIRMATION: ${{ inputs.confirmation }}
run: test "${CONFIRMATION}" = "RESTORE_STAGING_C2_C3_GENERAL"
- name: Require capacity identity and exact predecessor state
if: ${{ inputs.mode == 'apply' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
case "${EXPECTED_HARD_CAP}/${EXPECTED_UNOBSERVED_BOUND}" in
1000/0)
PREDECESSOR_C3_CAP=600
PREDECESSOR_C3_BOUND=60
;;
1000/60)
PREDECESSOR_C3_CAP=1000
PREDECESSOR_C3_BOUND=0
;;
600/60)
PREDECESSOR_C3_CAP=1000
PREDECESSOR_C3_BOUND=60
;;
*)
echo "Unsupported staging capacity transition" >&2
exit 1
;;
esac
ACTIVE_REVISION="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '
[.status.traffic[] | select((.percent // 0) > 0)] |
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test -n "${ACTIVE_REVISION}"
CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -cer '
[.spec.containers[0].env[]? |
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
if length == 1 then .[0] | fromjson else error("missing director topology") end')"
PREDECESSOR_CELLS_JSON="$(jq -ce \
--arg cell "${TARGET_CELL_ID}" \
--argjson cap "${PREDECESSOR_C3_CAP}" \
--argjson bound "${PREDECESSOR_C3_BOUND}" \
'map(if .id == $cell then . + {
connectionHardCap: $cap,
connectionUnobservedBound: $bound
} else . end)' <<< "${DESIRED_CELLS_JSON}")"
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${FALLBACK_CELL_ORIGIN}" \
--cell-id "${FALLBACK_CELL_ID}" \
--hard-cap 600 \
--unobserved-bound 60 \
--heartbeat fresh \
--admission either \
--draining forbidden \
--activity allowed
if jq -ne \
--argjson current "${CURRENT_CELLS_JSON}" \
--argjson expected "${DESIRED_CELLS_JSON}" \
'$current == $expected'; then
if node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh \
--admission general \
--draining forbidden \
--activity allowed; then
TRANSITION_PHASE=cell-active
elif node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh \
--admission migration-only \
--draining forbidden \
--activity allowed; then
TRANSITION_PHASE=cell-ready
else
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--heartbeat either \
--admission migration-only \
--draining required \
--activity restart-safe \
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
TRANSITION_PHASE=director-ready
fi
else
jq -ne \
--argjson current "${CURRENT_CELLS_JSON}" \
--argjson expected "${PREDECESSOR_CELLS_JSON}" \
'$current == $expected'
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${PREDECESSOR_C3_CAP}" \
--unobserved-bound "${PREDECESSOR_C3_BOUND}" \
--heartbeat fresh \
--admission either \
--draining either \
--activity allowed
TRANSITION_PHASE=predecessor
fi
echo "TRANSITION_PHASE=${TRANSITION_PHASE}" >> "${GITHUB_ENV}"
- name: Restore C2 as the safe placement fallback
if: ${{ inputs.mode == 'restore-admission' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${FALLBACK_CELL_ORIGIN}" \
--cell-id "${FALLBACK_CELL_ID}" \
--heartbeat fresh \
--admission either \
--draining forbidden \
--activity allowed
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--mode restore-fallback \
--general-cell-ids "${FALLBACK_CELL_ID}"
- name: Require a healthy non-draining C3 before restoring it
if: ${{ inputs.mode == 'restore-admission' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh \
--admission either \
--draining forbidden \
--activity allowed
- name: Require a healthy general C2 before isolating C3
if: ${{ inputs.mode == 'apply' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
if test "${TRANSITION_PHASE}" = cell-active; then
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--mode restore-fallback \
--general-cell-ids "${FALLBACK_CELL_ID}"
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${FALLBACK_CELL_ORIGIN}" \
--cell-id "${FALLBACK_CELL_ID}" \
--hard-cap 600 \
--unobserved-bound 60 \
--heartbeat fresh \
--admission general \
--draining forbidden \
--activity allowed
- name: Reversibly isolate and drain C3
if: ${{ inputs.mode == 'apply' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
if test "${TRANSITION_PHASE}" = cell-ready; then
exit 0
fi
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--mode isolate
- name: Verify restart-safe migration-only target
if: ${{ inputs.mode == 'apply' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
if test "${TRANSITION_PHASE}" = cell-ready; then
exit 0
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--heartbeat either \
--admission migration-only \
--draining required \
--activity restart-safe \
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
- name: Verify current capacity
if: ${{ inputs.mode == 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh \
--admission general \
--draining forbidden \
--activity allowed
- name: Deploy reviewed director topology and remove pre-protocol revisions
if: ${{ inputs.mode == 'apply' }}
run: |
if test "${TRANSITION_PHASE}" != predecessor; then
echo "DIRECTOR_CONFIG_CHANGED=false" >> "${GITHUB_ENV}"
exit 0
fi
RELEASE_ID="capacity-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
DEPLOY_RESULT="$(node dev/scripts/deploy-relay-blue-green.mjs \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--service "${DIRECTOR_SERVICE_NAME}" \
--image "${ACTIVE_IMAGE}" \
--role director \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--capacity-cell-id "${TARGET_CELL_ID}" \
--director-cells-json "${DESIRED_CELLS_JSON}" \
--prune-revisions true \
--release-id "${RELEASE_ID}")"
echo "${DEPLOY_RESULT}"
DIRECTOR_CONFIG_CHANGED="$(jq -r '.topologyChanged' <<< "${DEPLOY_RESULT}")"
[[ "${DIRECTOR_CONFIG_CHANGED}" =~ ^(true|false)$ ]]
echo "DIRECTOR_CONFIG_CHANGED=${DIRECTOR_CONFIG_CHANGED}" >> "${GITHUB_ENV}"
- name: Require fail-closed director transition
if: ${{ inputs.mode == 'apply' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
if test "${TRANSITION_PHASE}" = cell-ready; then
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh \
--admission migration-only \
--draining forbidden \
--activity allowed
exit 0
fi
HEARTBEAT_EXPECTATION=either
if test "${DIRECTOR_CONFIG_CHANGED}" = true; then
HEARTBEAT_EXPECTATION=stale
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat "${HEARTBEAT_EXPECTATION}" \
--admission migration-only \
--draining required \
--activity restart-safe \
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
- name: Plan and apply only the exact empty cell
if: ${{ inputs.mode == 'apply' }}
shell: bash
run: |
recreate_fixed_one_instance() {
local instance
instance="$(gcloud compute instance-groups managed list-instances \
"${MIG_NAME}" \
--project "${GCP_PROJECT_ID}" \
--zone "${TARGET_ZONE}" \
--format=json \
| jq -er '
if length == 1 and .[0].instanceStatus == "RUNNING" and
.[0].currentAction == "NONE"
then .[0].instance | split("/") | last
else error("capacity cell does not have one stable running instance") end')"
gcloud compute instance-groups managed recreate-instances \
"${MIG_NAME}" \
--instances "${instance}" \
--project "${GCP_PROJECT_ID}" \
--zone "${TARGET_ZONE}" \
--quiet
}
terraform -chdir=infra/terraform plan \
-var-file=environments/staging.tfvars \
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c3"]' \
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]' \
-out="${RUNNER_TEMP}/relay-capacity-cell.tfplan"
PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-capacity-cell.tfplan" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode cell \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--image "${DESIRED_IMAGE}")"
echo "${PLAN_RESULT}"
CELL_PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")"
[[ "${CELL_PLAN_CHANGES}" =~ ^(0|2)$ ]]
if test "${TRANSITION_PHASE}" = cell-ready; then
test "${CELL_PLAN_CHANGES}" = 0
elif test "${TRANSITION_PHASE}" = cell-active; then
test "${CELL_PLAN_CHANGES}" = 0
recreate_fixed_one_instance
elif test "${CELL_PLAN_CHANGES}" = 0; then
recreate_fixed_one_instance
else
terraform -chdir=infra/terraform apply \
-auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan"
fi
gcloud compute instance-groups managed wait-until \
"${MIG_NAME}" \
--stable \
--project "${GCP_PROJECT_ID}" \
--zone "${TARGET_ZONE}" \
--timeout 900
- name: Verify exact live cap and fresh matching heartbeat
if: ${{ inputs.mode == 'apply' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh \
--admission migration-only \
--draining forbidden \
--activity allowed
- name: Make C3 the only staging placement cell
if: ${{ inputs.mode == 'apply' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--mode activate
- name: Verify the sole general canary after transition
if: ${{ inputs.mode == 'apply' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh \
--admission general \
--draining forbidden \
--activity allowed
- name: Restore the reviewed C2 and C3 placement set
if: ${{ inputs.mode == 'restore-admission' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--mode restore \
--general-cell-ids staging-gce-c2,staging-gce-c3
- name: Verify restored C3 admission and capacity
if: ${{ inputs.mode == 'restore-admission' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh \
--admission general \
--draining forbidden \
--activity allowed
- name: Preserve C2 as the safe fallback after a failed transition
if: ${{ failure() && inputs.mode == 'apply' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
run: |
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" \
--mode restore-fallback \
--general-cell-ids "${FALLBACK_CELL_ID}"
refresh-asia-c4-image:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && github.ref == 'refs/heads/main' && inputs.mode == 'refresh-asia-c4-image' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 90
environment: staging
env:
GCP_PROJECT_ID: onorca-cloud-staging
GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
CELL_ORIGIN: https://c4.relay-staging.onorca.dev
TARGET_CELL_ID: staging-gce-c4
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }}
TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }}
APPROVED_PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7
steps:
- uses: actions/checkout@v4
- name: Require the exact bounded C4 refresh
env:
CONFIRMATION: ${{ inputs.confirmation }}
shell: bash
run: |
set -euo pipefail
test "${CONFIRMATION}" = REFRESH_STAGING_ASIA_C4_IMAGE
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
[[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
test "${PREDECESSOR_IMAGE_DIGEST}" != "${TARGET_IMAGE_DIGEST}"
test "${PREDECESSOR_IMAGE_DIGEST}" = "${APPROVED_PREDECESSOR_IMAGE_DIGEST}"
- id: google-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-staging-terraform-state
object: terraform/state/cloud-sql-rollout/staging.lock
- uses: hashicorp/setup-terraform@v3
with:
terraform_version: 1.15.8
terraform_wrapper: false
- uses: actions/setup-node@v4
with:
node-version: 24
- name: Initialize the exact staging backend
run: node dev/scripts/infra.mjs init --env staging
- name: Resolve the reviewed C4 image and shape
shell: bash
run: |
set -euo pipefail
cells="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars \
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
shape="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${cells}")"
test "$(jq -r '.hostname' <<< "${shape}")" = c4
test "$(jq -r '.region' <<< "${shape}")" = asia-east2
test "$(jq -r '.zone' <<< "${shape}")" = asia-east2-a
test "$(jq -r '.machine_type' <<< "${shape}")" = e2-standard-4
test "$(jq -r '.capacity_requests' <<< "${shape}")" = 6000
test "$(jq -r '.database_pool_max' <<< "${shape}")" = 10
test "$(jq -r '.connection_hard_cap' <<< "${shape}")" = 3000
test "$(jq -r '.connection_unobserved_bound' <<< "${shape}")" = 60
test "$(jq -r '.initially_enabled' <<< "${shape}")" = false
desired_image="$(jq -r '.image' <<< "${shape}")"
test "${desired_image}" = \
"us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${TARGET_IMAGE_DIGEST}"
served_digest="$(gcloud artifacts docker images describe "${desired_image}" \
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
test "${served_digest}" = "${TARGET_IMAGE_DIGEST}"
mig_name="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
| jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
test "${mig_name}" = orca-cloud-staging-relay-gce-c4
{
echo "DESIRED_IMAGE=${desired_image}"
echo "ROLLBACK_IMAGE=us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${PREDECESSOR_IMAGE_DIGEST}"
echo "TARGET_ZONE=asia-east2-a"
echo "MIG_NAME=${mig_name}"
} >> "${GITHUB_ENV}"
- name: Save, validate, and classify the exact C4 plan
id: plan
shell: bash
run: |
set -euo pipefail
plan="${RUNNER_TEMP}/relay-c4-image-refresh.tfplan"
terraform -chdir=infra/terraform plan \
-var-file=environments/staging.tfvars \
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
-out="${plan}"
result="$(terraform -chdir=infra/terraform show -json "${plan}" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
--unobserved-bound 60 --image "${DESIRED_IMAGE}" \
--rollback-image "${ROLLBACK_IMAGE}")"
case "$(jq -r '[.changes,.changeKind] | join(":")' <<< "${result}")" in
2:replacement) refresh_phase=predecessor ;;
0:none|*:obsolete-template-delete) refresh_phase=applied ;;
*:manager-convergence|*:replacement-with-obsolete-template) refresh_phase=converging ;;
*) exit 1 ;;
esac
{
echo "PLAN_CHANGES=$(jq -r '.changes' <<< "${result}")"
echo "REFRESH_PHASE=${refresh_phase}"
} >> "${GITHUB_ENV}"
- id: state-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Verify the exact selector and current C4 state
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.state-auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \
--expected-generation '' --expected-membership-sha256 '' --attempt-id '' \
--image-digest "${TARGET_IMAGE_DIGEST}")"
test "$(jq -r '.generation' <<< "${inspect}")" = "${EXPECTED_SELECTOR_GENERATION}"
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \
migration-only
expected_digests="${TARGET_IMAGE_DIGEST}"
if test "${REFRESH_PHASE}" = predecessor; then
expected_digests="${PREDECESSOR_IMAGE_DIGEST}"
elif test "${REFRESH_PHASE}" = converging; then
expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
fi
if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \
"${CELL_ORIGIN}/v1/admin/runtime-status" \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' --data '{"v":1}')"; then
current_digest="$(jq -er '.imageDigest' <<< "${runtime}")"
case ",${expected_digests}," in
*,"${current_digest}",*) ;;
*) exit 1 ;;
esac
source_incarnation=''
draining=forbidden
if test "${REFRESH_PHASE}" != applied; then
status="$(curl --fail-with-body --max-time 30 --request POST \
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' \
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
source_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")"
[[ "${source_incarnation}" =~ ^[0-9a-f-]{36}$ ]]
if test "$(jq -r '.draining' <<< "${runtime}")" = true; then
draining=required
fi
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
--heartbeat fresh --admission migration-only --draining "${draining}" \
--activity quiescent --expected-image-digests "${expected_digests}"
runtime_available=true
else
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \
--admission migration-only --draining either --activity restart-safe \
--timeout-ms 300000
source_incarnation=''
runtime_available=false
fi
{
echo "MIG_STABLE_AT_MS=0"
echo "MUTATION_STARTED=false"
echo "REPLACEMENT_STARTED_AT_MS=0"
echo "RUNTIME_AVAILABLE=${runtime_available}"
echo "SOURCE_INCARNATION=${source_incarnation}"
} >> "${GITHUB_ENV}"
- id: fence-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Fence C4 and prove it stayed empty before replacement
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
if test "${REFRESH_PHASE}" = applied; then exit 0; fi
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
if test "${RUNTIME_AVAILABLE}" = false; then exit 0; fi
node dev/scripts/prepare-relay-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --mode isolate
fence_digests="${PREDECESSOR_IMAGE_DIGEST}"
if test "${REFRESH_PHASE}" = converging; then
fence_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
--heartbeat fresh --admission migration-only --draining required \
--activity quiescent --expected-image-digests "${fence_digests}"
- name: Apply the exact saved C4 plan
shell: bash
run: |
set -euo pipefail
if test "${PLAN_CHANGES}" = 0 && test "${RUNTIME_AVAILABLE}" = true; then exit 0; fi
if test "${REFRESH_PHASE}" = applied && test "${RUNTIME_AVAILABLE}" = false; then
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
fi
if test "${REFRESH_PHASE}" != applied; then
replacement_started_at_ms="$(date -u +%s%3N)"
echo "REPLACEMENT_STARTED_AT_MS=${replacement_started_at_ms}" >> "${GITHUB_ENV}"
fi
if test "${PLAN_CHANGES}" != 0; then
terraform -chdir=infra/terraform apply -auto-approve \
"${RUNNER_TEMP}/relay-c4-image-refresh.tfplan"
fi
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
if test "${RUNTIME_AVAILABLE}" = false && test "${REFRESH_PHASE}" = applied; then
instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \
| jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and
.[0].currentAction == "NONE" then .[0].instance | split("/") | last
else error("C4 is not one stable running instance") end')"
gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \
--instances "${instance}" --project "${GCP_PROJECT_ID}" \
--zone "${TARGET_ZONE}" --quiet
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
fi
echo "MIG_STABLE_AT_MS=$(date -u +%s%3N)" >> "${GITHUB_ENV}"
- id: post-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Verify new C4 incarnation, image, and unchanged isolation
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
--heartbeat fresh --admission migration-only --draining forbidden \
--activity quiescent --expected-image-digests "${TARGET_IMAGE_DIGEST}" \
--timeout-ms 240000
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
--expected-membership-sha256 '' --attempt-id '' \
--image-digest "${TARGET_IMAGE_DIGEST}")"
test "$(jq -r '.generation' <<< "${result}")" = "${EXPECTED_SELECTOR_GENERATION}"
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \
migration-only
for _ in $(seq 1 36); do
status="$(curl --fail-with-body --max-time 30 --request POST \
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' \
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
-ge "${MIG_STABLE_AT_MS}"; then break; fi
sleep 5
done
target_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")"
test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
-ge "${MIG_STABLE_AT_MS}"
if test "${REFRESH_PHASE}" != applied; then
if test -n "${SOURCE_INCARNATION}"; then
test "${target_incarnation}" != "${SOURCE_INCARNATION}"
fi
test "$(jq -r '.status.runtime.startedAt' <<< "${status}")" \
-ge "${REPLACEMENT_STARTED_AT_MS}"
fi
- name: Require an empty targeted Terraform readback
shell: bash
run: |
set -euo pipefail
plan="${RUNNER_TEMP}/relay-c4-image-readback.tfplan"
terraform -chdir=infra/terraform plan \
-var-file=environments/staging.tfvars \
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
-out="${plan}"
result="$(terraform -chdir=infra/terraform show -json "${plan}" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
--unobserved-bound 60 --image "${DESIRED_IMAGE}" \
--rollback-image "${ROLLBACK_IMAGE}")"
test "$(jq -r '.changes' <<< "${result}")" = 0
@@ -0,0 +1,133 @@
name: Publish Relay Production Image
on:
workflow_dispatch:
inputs:
mode:
description: Publish a new production image or mirror an existing immutable image to staging
required: true
default: publish
type: choice
options: [publish, mirror-staging]
image-digest:
description: Exact existing production digest for mirror-staging mode
required: false
type: string
confirmation:
description: Enter MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING for mirror-staging mode
required: false
type: string
permissions:
contents: read
id-token: write
concurrency:
group: publish-relay-production
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
publish:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: production
env:
GCP_PROJECT_ID: onorca-cloud
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
REPOSITORY_ID: orca-cloud
IMAGE_NAME: relay
PUBLISH_MODE: ${{ inputs.mode }}
MIRROR_DIGEST: ${{ inputs.image-digest }}
MIRROR_CONFIRMATION: ${{ inputs.confirmation }}
steps:
- uses: actions/checkout@v4
- name: Validate the exact publish request before authentication
shell: bash
run: |
set -euo pipefail
if test "${PUBLISH_MODE}" = mirror-staging; then
[[ "${MIRROR_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
test "${MIRROR_CONFIRMATION}" = MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING
else
test "${PUBLISH_MODE}" = publish
test -z "${MIRROR_DIGEST}"
test -z "${MIRROR_CONFIRMATION}"
fi
- uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: docker/setup-buildx-action@v3
- name: Configure Docker auth
run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
- name: Build and publish immutable image
if: ${{ inputs.mode == 'publish' }}
run: |
IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${GITHUB_SHA}"
docker build -f apps/relay/Dockerfile -t "${IMAGE_TAG}" .
docker push "${IMAGE_TAG}"
DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')"
test -n "${DIGEST}"
IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${DIGEST}"
{
echo '### Terraform candidate image'
echo
echo "\`${IMAGE}\`"
echo
echo 'Declare this digest on a distinct disabled candidate cell in a reviewed Terraform PR.'
} >> "${GITHUB_STEP_SUMMARY}"
- name: Build and publish immutable fence broker
if: ${{ inputs.mode == 'publish' }}
run: |
IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker:sha-${GITHUB_SHA}"
docker build \
--build-arg "ORCA_RELAY_FENCE_IMAGE_COMMIT=${GITHUB_SHA}" \
-f apps/relay-fence-broker/Dockerfile \
-t "${IMAGE_TAG}" .
docker push "${IMAGE_TAG}"
DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')"
test -n "${DIGEST}"
IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker@${DIGEST}"
{
echo
echo '### Terraform fence broker image'
echo
echo "\`${IMAGE}\`"
} >> "${GITHUB_STEP_SUMMARY}"
- name: Mirror the exact production manifest to staging
if: ${{ inputs.mode == 'mirror-staging' }}
shell: bash
run: |
set -euo pipefail
source_image="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${MIRROR_DIGEST}"
target_tag="${GCP_REGION}-docker.pkg.dev/onorca-cloud-staging/${REPOSITORY_ID}/${IMAGE_NAME}:production-${MIRROR_DIGEST#sha256:}"
source_digest="$(gcloud artifacts docker images describe "${source_image}" \
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
test "${source_digest}" = "${MIRROR_DIGEST}"
docker pull "${source_image}"
docker tag "${source_image}" "${target_tag}"
docker push "${target_tag}"
target_digest="$(gcloud artifacts docker images describe "${target_tag}" \
--project onorca-cloud-staging --format='value(image_summary.digest)')"
test "${target_digest}" = "${MIRROR_DIGEST}"
{
echo '### Mirrored Relay image'
echo
printf 'Production and staging now resolve the same immutable digest: %s.\n' \
"${MIRROR_DIGEST}"
} >> "${GITHUB_STEP_SUMMARY}"
+568
View File
@@ -0,0 +1,568 @@
name: Deploy Push Gateway Production
on:
workflow_dispatch:
inputs:
source_sha:
description: Full reviewed commit SHA to build (feature may remain unmerged)
required: true
type: string
confirmation:
description: Enter DEPLOY_PUSH_GATEWAY to shift production traffic
required: true
type: string
permissions:
contents: read
id-token: write
# Serialize push traffic changes independently of Relay and the shared database.
concurrency:
group: production-push-rollout
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
deploy:
if: >-
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
github.ref == 'refs/heads/main' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: production
env:
GCP_PROJECT_ID: onorca-cloud
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
SERVICE_NAME: orca-cloud-push
REPOSITORY_ID: orca-cloud
IMAGE_NAME: push
PUSH_ORIGIN: https://push.onorca.dev
PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud.iam.gserviceaccount.com
# Scaling the serving revision must already hold, matching push_min_instances and
# push_max_instances. Terraform owns both, and the candidate inherits them from the
# service, so this deploy never passes a scaling flag: doing so would write a
# Terraform-owned field that `lifecycle.ignore_changes` does not cover, and a later
# `push_max_instances` raise would then be reverted by every deploy. These two values
# are the expected shape, asserted before the candidate is created and again on the
# candidate itself, so a deploy that would change the gateway's Cloud SQL draw fails.
PUSH_MIN_INSTANCES: 1
PUSH_MAX_INSTANCES: 2
CONFIRMATION: ${{ inputs.confirmation }}
SOURCE_SHA: ${{ inputs.source_sha }}
steps:
- uses: actions/checkout@v4
- name: Require the explicit deploy confirmation
shell: bash
run: |
set -euo pipefail
test "${CONFIRMATION}" = DEPLOY_PUSH_GATEWAY
[[ "${SOURCE_SHA}" =~ ^[a-f0-9]{40}$ ]]
# Keep the workflow and rollout lease on main; only the Docker build uses candidate code.
- name: Fetch the immutable gateway source
shell: bash
run: |
set -euo pipefail
git fetch --no-tags origin "${SOURCE_SHA}"
test "$(git rev-parse FETCH_HEAD)" = "${SOURCE_SHA}"
mkdir -p "${RUNNER_TEMP}/push-source"
git -C "${GITHUB_WORKSPACE}" archive "${SOURCE_SHA}" cloud \
| tar -x -C "${RUNNER_TEMP}/push-source"
- uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT }}
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: docker/setup-buildx-action@v3
- name: Configure Docker auth
run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
# Building an image does not need the deployment lease.
- name: Build and publish the immutable gateway image
shell: bash
run: |
set -euo pipefail
image_tag="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${SOURCE_SHA}"
docker buildx build --push --platform linux/amd64 --provenance=false --metadata-file "${RUNNER_TEMP}/push-image.json" \
-f "${RUNNER_TEMP}/push-source/cloud/apps/push/Dockerfile" \
-t "${image_tag}" "${RUNNER_TEMP}/push-source/cloud"
digest="$(jq -er '."containerimage.digest"' "${RUNNER_TEMP}/push-image.json")"
[[ "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
echo "IMAGE=${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${digest}" \
>> "${GITHUB_ENV}"
echo "IMAGE_DIGEST=${digest}" >> "${GITHUB_ENV}"
# Refuse older images before they ever boot against production.
- name: Require image support for inert validation
shell: bash
run: |
set -euo pipefail
docker run --rm --network none --entrypoint node "${IMAGE}" --input-type=module -e '
import { loadPushConfig } from "./apps/push/dist/config.js";
const env = { ORCA_PUSH_PUBLIC_URL: "https://push.onorca.dev", ORCA_PUSH_MODE: "validation", ORCA_PUSH_FCM_PROJECT_ID: "onorca-cloud" };
if (loadPushConfig(env).mode !== "validation") throw new Error("validation_mode_unsupported");
let rejected = false;
try { loadPushConfig({ ...env, ORCA_PUSH_MODE: "invalid" }); } catch { rejected = true; }
if (!rejected) throw new Error("validation_mode_not_fail_closed");
'
# Held across the deploy, not just a separate schema step: the gateway opens its pool and
# applies its schema while the new revision starts, so the revision is the schema step.
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/push-rollout/production.lock
# Why: the candidate inherits the serving revision's scaling. A serving revision that has
# drifted below the floor would hand the candidate a cold start on every notification, and
# one that has drifted above the ceiling would hand it a larger Cloud SQL draw than the
# rollout lease was taken for. Refuse to inherit either rather than latch it.
- name: Record the serving revision and require its Terraform-owned scaling
shell: bash
run: |
set -euo pipefail
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test -n "${serving}"
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
if test "${revisions}" != "${serving}"; then
echo 'Retire leftover revisions under the rollout lease before deploying; three pools are the limit.' >&2
exit 1
fi
floor="$(gcloud run revisions describe "${serving}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
if [[ "${floor:-0}" -lt "${PUSH_MIN_INSTANCES}" ]]; then
echo "serving revision ${serving} holds ${floor:-0} minimum instances," \
"below ${PUSH_MIN_INSTANCES}; deploying would inherit and latch it." >&2
echo "Restore the floor first: gcloud run services update ${SERVICE_NAME}" \
"--region ${GCP_REGION} --min-instances=${PUSH_MIN_INSTANCES}" >&2
exit 1
fi
ceiling="$(gcloud run revisions describe "${serving}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
test "${ceiling}" = "${PUSH_MAX_INSTANCES}"
echo "serving revision ${serving} holds ${floor} minimum and ${ceiling} maximum instances"
echo "ROLLBACK_REVISION=${serving}" >> "${GITHUB_ENV}"
gcloud run revisions describe "${serving}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
> "${RUNNER_TEMP}/push-rollback-revision.json"
image="$(jq -er '.status.imageDigest' "${RUNNER_TEMP}/push-rollback-revision.json")"
[[ "${image}" =~ @sha256:[a-f0-9]{64}$ ]]
echo "ROLLBACK_IMAGE=${image}" >> "${GITHUB_ENV}"
jq -e 'all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE" or .value == "active")' \
"${RUNNER_TEMP}/push-rollback-revision.json" > /dev/null
# Validation has no schema writes, HTTP mutations, worker, or pruners; tags alone do not
# isolate background consumers from production.
- name: Deploy the candidate revision with no traffic
shell: bash
run: |
set -euo pipefail
tag="c${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
{
echo "VALIDATION_DEPLOY_ATTEMPTED=true"
echo "VALIDATION_REVISION=${SERVICE_NAME}-${tag}"
echo "VALIDATION_TAG=${tag}"
echo "CANDIDATE_TAG=${tag}"
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
} >> "${GITHUB_ENV}"
gcloud run deploy "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--image "${IMAGE}" \
--tag "${tag}" \
--revision-suffix "${tag}" \
--no-traffic \
--update-env-vars ORCA_PUSH_MODE=validation \
--quiet
candidate="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -er --arg tag "${tag}" \
'[.status.traffic[] | select(.tag == $tag)]
| if length == 1 then .[0] else error("tagged candidate is not unique") end')"
test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
echo "CANDIDATE_URL=$(jq -r '.url' <<< "${candidate}")" >> "${GITHUB_ENV}"
# A tagged revision is directly addressable and sits outside the service-wide cap, so the
# candidate and the serving revision each draw up to the ceiling during the probe window.
# Successor creation later requires three revision pools; assert the inherited ceiling.
- name: Require the candidate to serve the exact image and inherited scaling
shell: bash
run: |
set -euo pipefail
served="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format='value(spec.containers[0].image)')"
test "${served}" = "${IMAGE}"
test "${CANDIDATE_REVISION}" != "${ROLLBACK_REVISION}"
candidate_ceiling="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
test "${candidate_ceiling}" = "${PUSH_MAX_INSTANCES}"
- name: Probe the candidate readiness endpoint
shell: bash
run: |
set -euo pipefail
[[ "${CANDIDATE_URL}" =~ ^https://[^/]+$ ]]
for attempt in $(seq 1 30); do
code="$(curl -sS -o "${RUNNER_TEMP}/push-ready.json" -w '%{http_code}' \
--max-time 10 "${CANDIDATE_URL}/ready" || true)"
if test "${code}" = 200; then
jq -e . < "${RUNNER_TEMP}/push-ready.json" > /dev/null
curl --fail --silent --show-error --max-time 10 "${CANDIDATE_URL}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "validation"' > /dev/null
echo "candidate ${CANDIDATE_REVISION} is ready after ${attempt} attempt(s)"
exit 0
fi
echo "attempt ${attempt}: /ready returned ${code}"
sleep 5
done
echo "candidate ${CANDIDATE_REVISION} never reported ready" >&2
exit 1
# Why: a gateway that boots and answers /ready can still be unable to send. This proves the
# runtime account's FCM grant end to end without delivering anything: validate_only stops
# Google before any push, and the deliberately invalid token means a healthy credential
# answers INVALID_ARGUMENT. PERMISSION_DENIED is the failure this step exists to catch.
#
# Only the four verdicts below are conclusive. A 429, a 5xx, or a transport failure says
# nothing about the credential, so it is retried rather than treated as either answer; a
# denied credential still fails on the first attempt, without burning the retries.
- name: Prove the runtime identity can reach FCM
shell: bash
run: |
set -euo pipefail
token="$(gcloud auth print-access-token \
--impersonate-service-account "${PUSH_RUNTIME_SERVICE_ACCOUNT}")"
test -n "${token}"
echo "::add-mask::${token}"
body='{"validate_only":true,"message":{"token":"orca-push-deploy-probe-invalid-token","notification":{"title":"Orca","body":"deploy probe"}}}'
for attempt in $(seq 1 5); do
code="$(curl -sS -o "${RUNNER_TEMP}/push-fcm.json" -w '%{http_code}' --max-time 20 \
-X POST "https://fcm.googleapis.com/v1/projects/${GCP_PROJECT_ID}/messages:send" \
-H "Authorization: Bearer ${token}" \
-H 'Content-Type: application/json' \
--data "${body}" || true)"
status="$(jq -r '.error.status // empty' < "${RUNNER_TEMP}/push-fcm.json" || true)"
echo "attempt ${attempt}: FCM validate-only send returned HTTP ${code} status ${status:-OK}"
if test "${status}" = PERMISSION_DENIED || test "${status}" = INVALID_ARGUMENT ||
test "${code}" = 401 || test "${code}" = 403; then
break
fi
sleep 5
done
if test "${status}" = PERMISSION_DENIED || test "${code}" = 401 || test "${code}" = 403; then
echo "the push runtime identity cannot send through FCM" >&2
exit 1
fi
test "${status}" = INVALID_ARGUMENT
# Cloud Run requires a successor before the latest revision can be deleted.
- name: Retire inert validation and activate the verified image
shell: bash
run: |
set -euo pipefail
tag="a${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
{
echo "CANDIDATE_TAG=${tag}"
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
echo "ACTIVATION_ATTEMPTED=true"
} >> "${GITHUB_ENV}"
# This is the production-effect boundary: schema, pruners and workers start here.
gcloud run deploy "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--image "${IMAGE}" --tag "${tag}" --revision-suffix "${tag}" \
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--remove-tags "${VALIDATION_TAG}" --quiet
gcloud run revisions delete "${VALIDATION_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
echo "VALIDATION_RETIRED=true" >> "${GITHUB_ENV}"
revision="$(gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
jq -e --arg image "${IMAGE}" --arg account "${PUSH_RUNTIME_SERVICE_ACCOUNT}" \
--arg ceiling "${PUSH_MAX_INSTANCES}" --arg floor "${PUSH_MIN_INSTANCES}" \
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" '
def shape: del(.containers[0].image) |
.containers[0].env = ((.containers[0].env // []) |
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
.spec.containers[0].image == $image and .spec.serviceAccountName == $account and
all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
(.spec | shape) == ($prior[0].spec | shape) and
.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
(.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber)' \
<<< "${revision}" > /dev/null
candidate="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -er --arg tag "${tag}" '[.status.traffic[] | select(.tag == $tag)]
| if length == 1 then .[0] else error("active candidate is not unique") end')"
test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
url="$(jq -er '.url' <<< "${candidate}")"
[[ "${url}" =~ ^https://[^/]+$ ]]
curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
curl --fail --silent --show-error --max-time 10 "${url}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
- name: Shift all traffic to the verified candidate
shell: bash
run: |
set -euo pipefail
echo "TRAFFIC_SHIFT_ATTEMPTED=true" >> "${GITHUB_ENV}"
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--to-revisions "${CANDIDATE_REVISION}=100" \
--quiet
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test "${serving}" = "${CANDIDATE_REVISION}"
echo "TRAFFIC_SHIFTED=true" >> "${GITHUB_ENV}"
# Why: the summary is written before the origin check, not after it. Once traffic has
# moved, the rollback target is the single thing an operator needs, and a summary that only
# appeared on success would be missing in exactly the run that needs it.
- name: Publish the rollout summary
if: ${{ always() && env.CANDIDATE_REVISION != '' && env.ROLLBACK_REVISION != '' }}
shell: bash
run: |
set -euo pipefail
{
echo '### Push gateway rollout'
echo
echo "Source: ${SOURCE_SHA}"
echo
echo "Revision: \`${CANDIDATE_REVISION}\`"
echo
echo "Image: \`${IMAGE_DIGEST}\`"
echo "Known-good image: \`${ROLLBACK_IMAGE}\`"
echo
echo "Recovery: deploy \`${ROLLBACK_IMAGE}\` as a new revision with" \
"\`--remove-env-vars ORCA_PUSH_MODE --no-traffic --tag <unique-recovery-tag> --revision-suffix <unique-recovery-suffix>\`."
echo 'Verify its exact digest, configuration, readiness and active mode, then promote and check the public origin.'
echo 'Only then remove obsolete tags and delete rejected/previous revisions; never delete the latest revision.'
echo 'The previous revision is retired after public checks; retain this immutable image for recovery under the rollout lease.'
echo "Activation attempted: ${ACTIVATION_ATTEMPTED:-false}; traffic rollback cannot undo schema or deliveries."
} >> "${GITHUB_STEP_SUMMARY}"
- name: Verify the public origin after the shift
shell: bash
run: |
set -euo pipefail
for attempt in $(seq 1 30); do
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \
"${PUSH_ORIGIN}/ready" || true)"
if test "${code}" = 200; then
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' > /dev/null
echo "ROLLOUT_VERIFIED=true" >> "${GITHUB_ENV}"
echo "${PUSH_ORIGIN} is ready after ${attempt} attempt(s)"
exit 0
fi
echo "attempt ${attempt}: ${PUSH_ORIGIN}/ready returned ${code}"
sleep 5
done
echo "${PUSH_ORIGIN} never reported ready after the shift" >&2
exit 1
# Why: everything after the shift runs with production on the candidate. A failure there
# is not a failure to deploy, it is a live gateway that has to go back, so the traffic move
# is undone here rather than left to whoever reads the run.
- name: Roll traffic back to the previous revision
if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
shell: bash
run: |
set -euo pipefail
test -n "${ROLLBACK_REVISION:-}"
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--to-revisions "${ROLLBACK_REVISION}=100" \
--quiet
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test "${serving}" = "${ROLLBACK_REVISION}"
echo "TRAFFIC_ROLLED_BACK=true" >> "${GITHUB_ENV}"
{
echo
echo '### Push gateway rolled back'
echo
echo "Traffic returned to \`${ROLLBACK_REVISION}\`; the candidate" \
"\`${CANDIDATE_REVISION}\` no longer serves HTTP; deletion below must stop its workers."
} >> "${GITHUB_STEP_SUMMARY}"
# Deleting a revision does not restore the service template that Terraform reconciles.
- name: Restore the known-good service template
if: ${{ (failure() || cancelled()) && env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
shell: bash
run: |
set -euo pipefail
test "${TRAFFIC_SHIFT_ATTEMPTED:-false}" != true || test "${TRAFFIC_ROLLED_BACK:-false}" = true
test -n "${ROLLBACK_IMAGE}"
# A partial activation may leave validation plus active; free one slot before recovery.
latest="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format='value(status.latestCreatedRevisionName)')"
test -n "${latest}"
if test "${VALIDATION_RETIRED:-false}" != true && test "${latest}" != "${VALIDATION_REVISION}"; then
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--filter "metadata.name=${VALIDATION_REVISION}" --format='value(metadata.name)')"
if test -n "${existing}"; then
test "${existing}" = "${VALIDATION_REVISION}"
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--remove-tags "${VALIDATION_TAG}" --quiet
gcloud run revisions delete "${VALIDATION_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
fi
fi
tag="r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
echo "RECOVERY_TAG=${tag}" >> "${GITHUB_ENV}"
echo "TEMPLATE_RECOVERY_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
echo "Template recovery attempted: ${SERVICE_NAME}-${tag}, image ${ROLLBACK_IMAGE}." \
>> "${GITHUB_STEP_SUMMARY}"
# Known-good schema/workers can run here even though HTTP stays on the old revision.
gcloud run deploy "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--image "${ROLLBACK_IMAGE}" --revision-suffix "${tag}" --tag "${tag}" \
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
> "${RUNNER_TEMP}/push-recovered-service.json"
gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
> "${RUNNER_TEMP}/push-recovery-revision.json"
jq -e --arg image "${ROLLBACK_IMAGE}" --arg serving "${ROLLBACK_REVISION}" \
--arg floor "${PUSH_MIN_INSTANCES}" --arg ceiling "${PUSH_MAX_INSTANCES}" \
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" \
--slurpfile recovered "${RUNNER_TEMP}/push-recovery-revision.json" '
def shape: del(.containers[0].image) |
.containers[0].env = ((.containers[0].env // []) |
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
.spec.template.spec.containers[0].image == $image and
all(.spec.template.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
$recovered[0].spec.containers[0].image == $image and
all($recovered[0].spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
($recovered[0].spec | shape) == ($prior[0].spec | shape) and
.spec.template.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
(.spec.template.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber) and
([.status.traffic[] | select((.percent // 0) > 0)] |
length == 1 and .[0].revisionName == $serving and .[0].percent == 100)
' "${RUNNER_TEMP}/push-recovered-service.json" > /dev/null
echo "TEMPLATE_RESTORED=true" >> "${GITHUB_ENV}"
echo 'Known-good image and normal mode restored; previous revision still serves HTTP.' \
>> "${GITHUB_STEP_SUMMARY}"
- name: Promote and verify the known-good recovery revision
if: ${{ always() && env.TEMPLATE_RESTORED == 'true' }}
shell: bash
run: |
set -euo pipefail
url="$(jq -er --arg tag "${RECOVERY_TAG}" \
'.status.traffic[] | select(.tag == $tag) | .url' "${RUNNER_TEMP}/push-recovered-service.json")"
[[ "${url}" =~ ^https://[^/]+$ ]]
curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
curl --fail --silent --show-error --max-time 10 "${url}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--to-revisions "${TEMPLATE_RECOVERY_REVISION}=100" --quiet
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test "${serving}" = "${TEMPLATE_RECOVERY_REVISION}"
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/ready" | jq -e '.ok == true'
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
echo "RECOVERY_VERIFIED=true" >> "${GITHUB_ENV}"
echo "Recovery revision ${TEMPLATE_RECOVERY_REVISION} now serves the known-good image." \
>> "${GITHUB_STEP_SUMMARY}"
- name: Delete the rejected candidate revision
if: ${{ always() && env.RECOVERY_VERIFIED == 'true' }}
shell: bash
run: |
set -euo pipefail
if test -z "${CANDIDATE_REVISION:-}"; then
echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
exit 0
fi
if test -n "${CANDIDATE_TAG:-}"; then
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--remove-tags "${CANDIDATE_TAG}" \
--quiet
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
fi
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--filter "metadata.name=${CANDIDATE_REVISION}" --format='value(metadata.name)')"
if test -n "${existing}"; then
test "${existing}" = "${CANDIDATE_REVISION}"
gcloud run revisions delete "${CANDIDATE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
fi
echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
echo "candidate revision ${CANDIDATE_REVISION} is absent"
# Public checks commit the serving revision; cleanup failures must not roll it back.
- name: Retire previous consumers after public checks
if: ${{ always() && (env.ROLLOUT_VERIFIED == 'true' || (env.RECOVERY_VERIFIED == 'true' && env.CANDIDATE_DELETED == 'true')) }}
shell: bash
run: |
set -euo pipefail
serving="${CANDIDATE_REVISION}"
if test "${RECOVERY_VERIFIED:-false}" = true; then
serving="${TEMPLATE_RECOVERY_REVISION}"
fi
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --clear-tags --quiet
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
while IFS= read -r revision; do
test -n "${revision}" || continue
test "${revision}" != "${serving}" || continue
case "${revision}" in
"${ROLLBACK_REVISION}"|"${VALIDATION_REVISION}"|"${CANDIDATE_REVISION}") ;;
*) echo "Unexpected revision ${revision}; manual retirement required." >&2; exit 1 ;;
esac
gcloud run revisions delete "${revision}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
done <<< "${revisions}"
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
echo 'Obsolete revision resources retired; verify actual SQL session drain operationally.' \
>> "${GITHUB_STEP_SUMMARY}"
- name: Drop the candidate traffic tag
if: always()
shell: bash
run: |
set -euo pipefail
test -n "${CANDIDATE_TAG:-}" || exit 0
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--remove-tags "${CANDIDATE_TAG}" \
--quiet
@@ -0,0 +1,413 @@
name: Recover Relay Staging C4 Image
on:
workflow_run:
workflows: [Prove Relay Staging Capacity]
types: [completed]
workflow_dispatch:
inputs:
confirmation:
description: Enter RECOVER_STAGING_ASIA_C4_IMAGE
required: true
type: string
permissions:
actions: read
contents: read
id-token: write
defaults:
run:
working-directory: cloud
jobs:
gate:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event_name == 'workflow_dispatch' || (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion != 'success')) }}
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 5
outputs:
recover: ${{ steps.trigger.outputs.recover }}
steps:
- name: Bind recovery to the exact failed C4 job
working-directory: .
id: trigger
env:
CONFIRMATION: ${{ inputs.confirmation }}
GH_TOKEN: ${{ github.token }}
SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
SOURCE_RUN_EVENT: ${{ github.event.workflow_run.event }}
shell: bash
run: |
set -euo pipefail
if test "${GITHUB_EVENT_NAME}" = workflow_dispatch; then
test "${CONFIRMATION}" = RECOVER_STAGING_ASIA_C4_IMAGE
echo "recover=true" >> "${GITHUB_OUTPUT}"
exit 0
fi
test "${SOURCE_RUN_EVENT}" = workflow_dispatch
[[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
jobs="$(gh api --paginate \
"repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")"
count="$(jq -s '[.[].jobs[] | select(.name == "refresh-asia-c4-image" and
(.conclusion == "failure" or .conclusion == "cancelled" or
.conclusion == "timed_out"))] | length' <<< "${jobs}")"
if test "${count}" = 0; then
echo "recover=false" >> "${GITHUB_OUTPUT}"
exit 0
fi
test "${count}" = 1
echo "recover=true" >> "${GITHUB_OUTPUT}"
recover:
needs: gate
if: ${{ needs.gate.outputs.recover == 'true' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 90
environment: staging
concurrency:
group: relay-staging-mutation
cancel-in-progress: false
env:
GCP_PROJECT_ID: onorca-cloud-staging
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
CELL_ORIGIN: https://c4.relay-staging.onorca.dev
TARGET_CELL_ID: staging-gce-c4
TARGET_ZONE: asia-east2-a
MIG_NAME: orca-cloud-staging-relay-gce-c4
PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7
TARGET_IMAGE_DIGEST: sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563
steps:
- uses: actions/checkout@v4
- id: auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
- uses: google-github-actions/setup-gcloud@v2
with:
skip_install: true
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-staging-terraform-state
object: terraform/state/cloud-sql-rollout/staging.lock
- uses: hashicorp/setup-terraform@v3
with:
terraform_version: 1.15.8
terraform_wrapper: false
- uses: actions/setup-node@v4
with:
node-version: 24
- name: Initialize the exact staging backend
run: node dev/scripts/infra.mjs init --env staging
- id: preflight-auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Inspect the exact C4 recovery state
id: preflight
timeout-minutes: 3
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.preflight-auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \
--expected-generation '' --expected-membership-sha256 '' --attempt-id '' \
--image-digest "${PREDECESSOR_IMAGE_DIGEST}")"
generation="$(jq -er '.generation' <<< "${inspect}")"
[[ "${generation}" =~ ^(0|[1-9][0-9]*)$ ]]
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \
migration-only
echo "selector_generation=${generation}" >> "${GITHUB_OUTPUT}"
if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \
"${CELL_ORIGIN}/v1/admin/runtime-status" \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' --data '{"v":1}')"; then
current_digest="$(jq -er '.imageDigest' <<< "${runtime}")"
[[ "${current_digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
status="$(curl --fail-with-body --max-time 30 --request POST \
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' \
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
jq -e '.draining | type == "boolean"' <<< "${runtime}" >/dev/null
{
echo "runtime_available=true"
echo "current_digest=${current_digest}"
echo "draining=$(jq -r '.draining' <<< "${runtime}")"
echo "ready=$(jq -r '.status.runtime.ready == true' <<< "${status}")"
} >> "${GITHUB_OUTPUT}"
else
echo "runtime_available=false" >> "${GITHUB_OUTPUT}"
fi
- name: Classify both exact recovery end states
id: recovery-plan
timeout-minutes: 10
shell: bash
run: |
set -euo pipefail
image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay"
predecessor_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}"
target_image="${image_repository}@${TARGET_IMAGE_DIGEST}"
served_digest="$(gcloud artifacts docker images describe "${predecessor_image}" \
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
test "${served_digest}" = "${PREDECESSOR_IMAGE_DIGEST}"
cells="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars \
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].image' <<< "${cells}")" = \
"${target_image}"
target_plan="${RUNNER_TEMP}/relay-c4-image-target.tfplan"
terraform -chdir=infra/terraform plan \
-var-file=environments/staging.tfvars -lock-timeout=5m \
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
-out="${target_plan}"
target_result="$(terraform -chdir=infra/terraform show -json "${target_plan}" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
--unobserved-bound 60 --image "${target_image}" \
--rollback-image "${predecessor_image}")"
target_state="$(jq -r '[.changes,.changeKind] | join(":")' <<< "${target_result}")"
case "${target_state}" in
0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;;
*) exit 1 ;;
esac
recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
--arg image "${predecessor_image}" '.[$cell].image = $image' <<< "${cells}")"
jq -n --argjson cells "${recovery_cells}" \
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-recovery.tfvars.json"
plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan"
terraform -chdir=infra/terraform plan \
-var-file=environments/staging.tfvars \
-var-file="${RUNNER_TEMP}/relay-c4-recovery.tfvars.json" \
-lock-timeout=5m \
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
-out="${plan}"
result="$(terraform -chdir=infra/terraform show -json "${plan}" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
--unobserved-bound 60 --image "${predecessor_image}" \
--rollback-image "${target_image}")"
changes="$(jq -r '.changes' <<< "${result}")"
change_kind="$(jq -r '.changeKind' <<< "${result}")"
case "${changes}:${change_kind}" in
0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;;
*) exit 1 ;;
esac
mig="$(gcloud compute instance-groups managed describe "${MIG_NAME}" \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)"
mig_stable="$(jq -r '.status.isStable == true and .status.versionTarget.isReached == true' \
<<< "${mig}")"
instances="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)"
instance_stable="$(jq -r 'length == 1 and .[0].instanceStatus == "RUNNING" and
.[0].currentAction == "NONE"' <<< "${instances}")"
action=rollback-predecessor
recovery_digest="${PREDECESSOR_IMAGE_DIGEST}"
if test "${{ steps.preflight.outputs.runtime_available }}" = true && \
test "${{ steps.preflight.outputs.ready }}" = true && \
test "${{ steps.preflight.outputs.draining }}" = false && \
test "${mig_stable}" = true && test "${instance_stable}" = true; then
if test "${{ steps.preflight.outputs.current_digest }}" = "${TARGET_IMAGE_DIGEST}" && \
test "${target_state}" = 0:none; then
action=verify-target
recovery_digest="${TARGET_IMAGE_DIGEST}"
elif test "${{ steps.preflight.outputs.current_digest }}" = \
"${PREDECESSOR_IMAGE_DIGEST}" && test "${changes}:${change_kind}" = 0:none; then
action=verify-predecessor
fi
fi
{
echo "changes=${changes}"
echo "change_kind=${change_kind}"
echo "action=${action}"
echo "recovery_digest=${recovery_digest}"
} >> "${GITHUB_OUTPUT}"
- id: fence-auth
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Fence C4 before predecessor recovery
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
timeout-minutes: 6
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }}
shell: bash
run: |
set -euo pipefail
expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
current_digest="${{ steps.preflight.outputs.current_digest }}"
if test -n "${current_digest}" && [[ ",${expected_digests}," != *",${current_digest},"* ]]; then
expected_digests="${expected_digests},${current_digest}"
fi
if curl --silent --show-error --fail-with-body --max-time 30 --request POST \
"${CELL_ORIGIN}/v1/admin/drain" \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' --data '{"v":1,"graceMs":0}' \
>/dev/null; then
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
--heartbeat fresh --admission migration-only --draining required \
--activity quiescent \
--expected-image-digests "${expected_digests}" \
--timeout-ms 240000
else
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \
--admission migration-only --draining either --activity restart-safe \
--timeout-ms 240000
fi
- name: Apply and stabilize the saved predecessor plan
id: apply
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
timeout-minutes: 20
env:
CHANGES: ${{ steps.recovery-plan.outputs.changes }}
CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }}
shell: bash
run: |
set -euo pipefail
plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan"
if test "${CHANGES}" != 0; then
terraform -chdir=infra/terraform apply -auto-approve "${plan}"
fi
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}"
- name: Restart only when the plan did not replace C4
id: restore
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
timeout-minutes: 20
env:
APPLY_STABLE_AT_MS: ${{ steps.apply.outputs.stable_at_ms }}
CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }}
shell: bash
run: |
set -euo pipefail
if [[ "${CHANGE_KIND}" =~ ^(replacement|replacement-with-obsolete-template|manager-convergence)$ ]]; then
echo "stable_at_ms=${APPLY_STABLE_AT_MS}" >> "${GITHUB_OUTPUT}"
exit 0
fi
instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \
| jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and
.[0].currentAction == "NONE" then .[0].instance | split("/") | last
else error("C4 is not one stable running instance") end')"
gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \
--instances "${instance}" --project "${GCP_PROJECT_ID}" \
--zone "${TARGET_ZONE}" --quiet
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}"
- name: Require an empty selected-image recovery readback
timeout-minutes: 8
env:
RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }}
shell: bash
run: |
set -euo pipefail
image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay"
recovery_image="${image_repository}@${RECOVERY_DIGEST}"
other_image="${image_repository}@${TARGET_IMAGE_DIGEST}"
if test "${RECOVERY_DIGEST}" = "${TARGET_IMAGE_DIGEST}"; then
other_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}"
fi
cells="$(terraform -chdir=infra/terraform console \
-var-file=environments/staging.tfvars \
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" --arg image "${recovery_image}" \
'.[$cell].image = $image' <<< "${cells}")"
jq -n --argjson cells "${recovery_cells}" \
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-readback.tfvars.json"
readback="${RUNNER_TEMP}/relay-c4-image-recovery-readback.tfplan"
terraform -chdir=infra/terraform plan \
-var-file=environments/staging.tfvars \
-var-file="${RUNNER_TEMP}/relay-c4-readback.tfvars.json" \
-lock-timeout=5m \
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
-out="${readback}"
readback_result="$(terraform -chdir=infra/terraform show -json "${readback}" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
--unobserved-bound 60 --image "${recovery_image}" \
--rollback-image "${other_image}")"
test "$(jq -r '.changes' <<< "${readback_result}")" = 0
- id: verify-auth
if: ${{ always() }}
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
token_format: id_token
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
id_token_include_email: true
- name: Verify the recovered image and unchanged isolation
if: ${{ always() && steps.verify-auth.outcome == 'success' }}
timeout-minutes: 8
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.verify-auth.outputs.id_token }}
SELECTOR_GENERATION: ${{ steps.preflight.outputs.selector_generation }}
STABLE_AT_MS: ${{ steps.restore.outputs.stable_at_ms }}
RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }}
shell: bash
run: |
set -euo pipefail
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
--heartbeat fresh --admission migration-only --draining forbidden \
--activity quiescent --expected-image-digests "${RECOVERY_DIGEST}" \
--timeout-ms 240000
stable_at_ms="${STABLE_AT_MS:-0}"
for _ in $(seq 1 36); do
status="$(curl --fail-with-body --max-time 30 --request POST \
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
--header 'Content-Type: application/json' \
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
-ge "${stable_at_ms}"; then break; fi
sleep 5
done
test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
-ge "${stable_at_ms}"
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
--environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \
--expected-generation "${SELECTOR_GENERATION}" \
--expected-membership-sha256 '' --attempt-id '' \
--image-digest "${RECOVERY_DIGEST}")"
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \
migration-only
@@ -0,0 +1,60 @@
name: Requeue Relay Staging C4 Recovery
on:
workflow_run:
workflows: [Recover Relay Staging C4 Image]
types: [completed]
permissions:
actions: write
contents: read
concurrency:
group: relay-staging-c4-recovery-requeue
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
requeue:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion == 'cancelled') }}
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 5
steps:
- name: Requeue only a cancelled protected recovery job
working-directory: .
env:
GH_TOKEN: ${{ github.token }}
SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
shell: bash
run: |
set -euo pipefail
[[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
jobs="$(gh api --paginate \
"repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")"
count="$(jq -s '[.[].jobs[] | select(.name == "recover" and
.conclusion == "cancelled" and .started_at == null)] | length' <<< "${jobs}")"
if test "${count}" = 0; then exit 0; fi
test "${count}" = 1
runs="$(gh api \
"repos/${GITHUB_REPOSITORY}/actions/workflows/cloud-recover-relay-staging-c4-image.yml/runs?branch=main&per_page=100")"
active=0
while IFS= read -r run_id; do
active_jobs="$(gh api --paginate \
"repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?filter=latest")"
if jq -se '
([.[].jobs[] | select(.name == "gate" and .conclusion == "success")] | length) == 1 and
([.[].jobs[] | select(.name == "recover" and .status != "completed")] | length) == 1
' <<< "${active_jobs}" >/dev/null; then
active=1
break
fi
done < <(jq -r --arg source "${SOURCE_RUN_ID}" \
'.workflow_runs[] | select((.id | tostring) != $source and
.status != "completed") | .id' <<< "${runs}")
if test "${active}" != 0; then exit 0; fi
gh workflow run cloud-recover-relay-staging-c4-image.yml \
--repo "${GITHUB_REPOSITORY}" --ref main \
-f confirmation=RECOVER_STAGING_ASIA_C4_IMAGE
+153
View File
@@ -0,0 +1,153 @@
name: Cloud Verify
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- cloud/**
- src/packages/process-host/**
- .github/workflows/cloud-*.yml
- .github/actions/cloud-sql-rollout-lease/**
push:
branches: [main]
paths:
- cloud/**
- src/packages/process-host/**
- .github/workflows/cloud-*.yml
- .github/actions/cloud-sql-rollout-lease/**
permissions:
contents: read
concurrency:
group: cloud-verify-${{ github.ref }}
cancel-in-progress: true
defaults:
run:
working-directory: cloud
jobs:
# Public-repository hosted runners preserve Blacksmith allowance for macOS.
security:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
name: Secret scan
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Pull Gitleaks image
id: gitleaks-image
background: true
run: docker pull zricethezav/gitleaks@sha256:cdbb7c955abce02001a9f6c9f602fb195b7fadc1e812065883f695d1eeaba854
- name: Pull TruffleHog image
id: trufflehog-image
background: true
run: docker pull trufflesecurity/trufflehog@sha256:5dc064868ba7933601b5cbaea6954954d524ddd5dc6222a9667acea70068bf7d
# Scan every ancestor of HEAD without downloading unrelated branch/tag histories.
- name: Fetch complete scan history
working-directory: .
run: |
git fetch --no-tags --unshallow origin "$GITHUB_SHA"
test "$(git rev-parse --is-shallow-repository)" = false
- wait: [gitleaks-image, trufflehog-image]
- name: Scan Cloud history reachable from HEAD with Gitleaks
id: history-scan
background: true
run: >-
docker run --rm
--volume "${GITHUB_WORKSPACE}:/repo:ro"
zricethezav/gitleaks@sha256:cdbb7c955abce02001a9f6c9f602fb195b7fadc1e812065883f695d1eeaba854
git /repo --config /repo/cloud/.gitleaks.toml
--log-opts="HEAD -- cloud :(glob).github/workflows/cloud-*.yml .github/actions/cloud-sql-rollout-lease"
- name: Scan the single-commit Cloud snapshot with TruffleHog
run: >-
docker run --rm
--volume "${GITHUB_WORKSPACE}:/repo:ro"
trufflesecurity/trufflehog@sha256:5dc064868ba7933601b5cbaea6954954d524ddd5dc6222a9667acea70068bf7d
filesystem /repo --no-verification --fail
--include-paths=/repo/cloud/.trufflehog-include-paths.txt
--exclude-paths=/repo/cloud/.trufflehog-exclude-paths.txt
- wait: history-scan
# Compiles the workspace. No Postgres service: nothing here reaches a
# database, and the service container costs ~13s of startup.
build:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
package_json_file: cloud/package.json
- uses: actions/setup-node@v4
with:
node-version: 24
- run: pnpm install --frozen-lockfile
- run: pnpm build
- run: pnpm typecheck
# Runs in parallel with build. `pnpm test` compiles the workspace
# packages it needs through pretest hooks, so it does not depend on
# `pnpm build` having run.
test:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: ubuntu-22.04
services:
postgres:
image: postgres:16-alpine
env:
POSTGRES_DB: orca_relay_test
POSTGRES_PASSWORD: relay_test
POSTGRES_USER: relay_test
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U relay_test -d orca_relay_test"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
ORCA_BACKGROUND_LAUNCH: '1'
ORCA_PUSH_TEST_DATABASE_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test
ORCA_RELAY_TEST_POSTGRES_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
package_json_file: cloud/package.json
- uses: actions/setup-node@v4
with:
node-version: 24
- run: pnpm install --frozen-lockfile
- run: pnpm test
# Fork pull requests reach this job, so it never configures a backend, never plans, and never
# holds a credential. Only the relay root ships here; foundation and apps stay private.
terraform:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- uses: hashicorp/setup-terraform@v3
with:
terraform_version: 1.15.8
- run: terraform -chdir=infra/terraform fmt -check -recursive
- run: terraform -chdir=infra/terraform init -backend=false -input=false
- run: terraform -chdir=infra/terraform validate
+44 -79
View File
@@ -2,21 +2,23 @@ name: Computer-use e2e
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- '.github/workflows/computer-e2e.yml'
- 'config/electron-builder.config.cjs'
- 'config/scripts/build-computer-macos.mjs'
- 'config/scripts/build-native-for-platform.mjs'
- 'config/scripts/build-native-for-platform.test.mjs'
- 'config/scripts/pnpm-cli-invocation.mjs'
- 'config/scripts/build-windows-cli-launcher.mjs'
- 'config/scripts/windows-rust-executable-build.mjs'
- 'config/scripts/build-windows-cli-launcher.test.mjs'
- 'config/scripts/computer-e2e-workflow.test.mjs'
- 'config/scripts/macos-computer-helper-owner-loss-benchmark.mjs'
- 'config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs'
- 'config/scripts/macos-computer-helper-owner-loss-metrics.mjs'
- 'config/scripts/macos-computer-helper-owner-loss-processes.mjs'
- 'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs'
- 'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs'
- 'config/scripts/computer-use-modifier-safety.test.mjs'
- 'config/scripts/computer-use-skill-guidance.test.mjs'
- 'config/scripts/computer-use-smoke.mjs'
- 'config/scripts/computer-use-smoke.test.mjs'
- 'config/scripts/daemon-boot-smoke.mjs'
@@ -29,6 +31,15 @@ on:
- 'electron.vite.config.ts'
- 'config/build-plugins/**'
- 'src/main/daemon/**'
- '!src/main/daemon/**/*.test.ts'
- '!src/main/daemon/daemon-init-dependency-mocks.ts'
- '!src/main/daemon/daemon-init-test-harness.ts'
- '!src/main/daemon/daemon-init-fresh-import.ts'
- '!src/main/daemon/daemon-init-mock-types.ts'
- '!src/main/daemon/serialize-grid-roundtrip.ts'
- '!src/main/daemon/mock-descendant-sweep.ts'
- '!src/main/daemon/daemon-host-relocation.test-fixture.ts'
- '!src/main/daemon/daemon-endpoint-reachability-test-harness.ts'
- 'src/main/providers/local-pty-provider.ts'
- 'src/main/providers/windows-shell-args.ts'
- 'src/main/providers/windows-shell-preflight-runtime.windows.test.ts'
@@ -42,12 +53,8 @@ on:
- 'src/main/runtime/rpc/dispatcher.ts'
- 'src/main/runtime/rpc/errors.ts'
- 'src/main/runtime/rpc/methods/computer*.ts'
- 'src/main/ssh/ssh-remote-cli-launcher.ts'
- 'src/main/ssh/ssh-remote-cli-launcher.test.ts'
- 'src/shared/computer-use-*.ts'
- 'tests/e2e/computer-linux.e2e.ts'
- 'tests/e2e/computer-mac.e2e.ts'
- 'tests/e2e/computer-mac-safari.e2e.ts'
- 'tests/e2e/computer-windows.e2e.ts'
- 'tests/e2e/computer-windows-store.e2e.ts'
- 'tests/e2e/helpers/computer-cli-driver.ts'
@@ -57,9 +64,13 @@ on:
schedule:
- cron: '0 6 * * *'
concurrency:
group: computer-e2e-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
native-smoke:
if: github.event_name == 'pull_request'
if: github.event_name == 'pull_request' && github.event.pull_request.draft != true
permissions:
contents: read
strategy:
@@ -71,36 +82,19 @@ jobs:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: actions/setup-node@v6
with:
node-version-file: package.json
- uses: pnpm/action-setup@v6
with:
run_install: false
- if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y python3 python3-gi gir1.2-atspi-2.0 at-spi2-core gedit xvfb xclip xdotool
# Why: pnpm's bundled node-gyp can ship gyp_main.py without execute
# permission on Linux runners; node-pty's install fallback then fails
# before this smoke job can exercise the native package.
- name: Use external node-gyp to avoid pnpm's bundled copy (Linux only)
if: runner.os == 'Linux'
run: |
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- run: pnpm install --frozen-lockfile
run: sudo apt-get update && sudo apt-get install -y python3 python3-gi gir1.2-atspi-2.0 at-spi2-core
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
# Why: without --config, bare vitest ignores config/vitest.config.ts (there
# is no root config) and falls back to the 5s default timeout with no
# Windows worker cap, so the real csc.exe launcher-compile tests time out
# on hosted Windows. Use the shared config so this job matches pnpm test.
- run: >-
pnpm vitest run --config config/vitest.config.ts
config/scripts/build-windows-cli-launcher.test.mjs
src/main/ssh/ssh-remote-cli-launcher.test.ts
config/scripts/computer-e2e-workflow.test.mjs
config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs
config/scripts/macos-computer-helper-owner-loss-processes.test.mjs
config/scripts/computer-use-modifier-safety.test.mjs
config/scripts/computer-use-skill-guidance.test.mjs
config/scripts/computer-use-smoke.test.mjs
src/main/computer/computer-provider-lifecycle.test.ts
src/main/computer/computer-provider-unavailable-message.test.ts
@@ -123,7 +117,6 @@ jobs:
src/cli/handlers/computer-action-routing.test.ts
src/cli/handlers/computer-action-validation.test.ts
src/cli/handlers/computer-state-formatting.test.ts
src/cli/specs/computer.test.ts
src/cli/index.test.ts
src/main/runtime/rpc/dispatcher-computer-errors.test.ts
src/main/runtime/rpc/errors.test.ts
@@ -133,7 +126,7 @@ jobs:
src/shared/remote-runtime-client.test.ts
- run: pnpm verify:computer-native
- run: pnpm build:cli
- run: pnpm build:electron-vite
- run: pnpm run build:electron-vite:parallel
# Why: boot the BUILT daemon-entry under plain Node the way production
# forks it. v1.4.129-rc.1 shipped a daemon that exited at module load
# (leaked electron require) while every other check passed; this fails
@@ -150,8 +143,10 @@ jobs:
- name: Windows daemon workspace-close repro
if: runner.os == 'Windows'
run: node config/scripts/windows-daemon-workspace-close-repro.mjs
# Hosted macOS runners cannot receive persistent Accessibility/Screen Recording
# grants. Keep the real native build/owner-loss checks on every trigger instead.
mac-native-owner-smoke:
if: github.event_name == 'pull_request'
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: macos-15
permissions:
contents: read
@@ -159,45 +154,25 @@ jobs:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: actions/setup-node@v6
- uses: ./.github/actions/install-node-dependencies
with:
node-version-file: package.json
- uses: pnpm/action-setup@v6
with:
run_install: false
- run: pnpm install --frozen-lockfile
native-runtime: electron
- name: Owner-loss benchmark process cleanup
run: >-
pnpm vitest run
config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs
config/scripts/macos-computer-helper-owner-loss-processes.test.mjs
# Why: the parallel launcher's cancellation tests are darwin-only and no
# other PR job runs on macOS.
- name: Parallel native build launcher cancellation
run: >-
pnpm vitest run --config config/vitest.config.ts
config/scripts/build-native-for-platform.test.mjs
- name: Authenticated helper owner-loss smoke
run: pnpm bench:macos-computer-helper-owner-loss --expect reaped --trials 1
- name: Swift tests and signed universal helper verification
run: pnpm verify:computer-native
mac:
# macOS Accessibility and Screen Recording require user-granted TCC entries.
# Keep this on manual/scheduled permission-bearing runners instead of PR CI.
if: github.event_name != 'pull_request'
runs-on: macos-15
env:
ORCA_COMPUTER_E2E: '1'
steps:
- uses: actions/checkout@v6
- uses: actions/setup-node@v6
with:
node-version-file: package.json
- uses: pnpm/action-setup@v6
with:
run_install: false
- run: pnpm install --frozen-lockfile
- run: pnpm build:computer-macos
- run: pnpm verify:computer-native
- run: pnpm build:cli
- run: pnpm build:electron-vite
- run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-mac.e2e.ts tests/e2e/computer-mac-safari.e2e.ts
linux:
if: github.event_name != 'pull_request'
runs-on: ubuntu-22.04
@@ -206,23 +181,15 @@ jobs:
ACCESSIBILITY_ENABLED: '1'
steps:
- uses: actions/checkout@v6
- uses: actions/setup-node@v6
with:
node-version-file: package.json
- uses: pnpm/action-setup@v6
with:
run_install: false
persist-credentials: false
- run: sudo apt-get update && sudo apt-get install -y build-essential python3 python3-gi gir1.2-atspi-2.0 gedit at-spi2-core xvfb xclip xdotool
# Why: keep scheduled Linux e2e on the same native install path as PR
# smoke and pr.yml's verify job.
- name: Use external node-gyp to avoid pnpm's bundled copy (Linux only)
run: |
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- run: pnpm install --frozen-lockfile
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- run: pnpm verify:computer-native
- run: pnpm build:cli
- run: pnpm build:electron-vite
- run: pnpm run build:electron-vite:parallel
- run: xvfb-run --auto-servernum dbus-run-session -- pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-linux.e2e.ts
windows:
@@ -232,14 +199,12 @@ jobs:
ORCA_COMPUTER_E2E: '1'
steps:
- uses: actions/checkout@v6
- uses: actions/setup-node@v6
with:
node-version-file: package.json
- uses: pnpm/action-setup@v6
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
run_install: false
- run: pnpm install --frozen-lockfile
native-runtime: electron
- run: pnpm verify:computer-native
- run: pnpm build:cli
- run: pnpm build:electron-vite
- run: pnpm run build:electron-vite:parallel
- run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-store.e2e.ts
+24 -3
View File
@@ -43,9 +43,9 @@ jobs:
node-version-file: package.json
- name: Setup pnpm
uses: pnpm/action-setup@v6
uses: pnpm/setup@v2
with:
run_install: false
install: false
- name: Install dependencies
run: pnpm install --frozen-lockfile
@@ -57,7 +57,28 @@ jobs:
uses: actions/cache@v4
with:
path: dist/win-unpacked
key: win-unpacked-${{ hashFiles('src/**', 'config/**', 'package.json', 'pnpm-lock.yaml') }}
# mobile/ is in the key because beforePack requires out/mobile-web, whose bytes are the
# page built from the mobile install and the page trees below; a mobile-only change must
# miss this cache, not reuse a stale installer. config/** already covers the builder, the
# verifier and the manifest writer.
key: >-
win-unpacked-${{ hashFiles(
'src/**',
'config/**',
'package.json',
'pnpm-lock.yaml',
'mobile/package.json',
'mobile/pnpm-lock.yaml',
'mobile/app/**',
'mobile/src/**',
'mobile/web-entry/**'
) }}
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules. Gated with the
# build it feeds, so a cache hit does not pay for an install nothing consumes.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.cache-unpacked.outputs.cache-hit != 'true'
- name: Build unpacked app
if: steps.cache-unpacked.outputs.cache-hit != 'true'
+87 -16
View File
@@ -70,7 +70,39 @@ env:
DAILY_RETAIN_COUNT: 30
jobs:
# Why: the app and the orcad template it ships must come from one commit, so resolve main once.
resolve-main:
if: github.repository == 'stablyai/orca'
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
head_sha: ${{ steps.resolve.outputs.head_sha }}
steps:
- name: Resolve main
id: resolve
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
head_sha="$(gh api "repos/$GITHUB_REPOSITORY/commits/main" --jq .sha)"
[[ "$head_sha" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::Could not resolve main"; exit 1; }
echo "head_sha=$head_sha" >>"$GITHUB_OUTPUT"
# As release-cut does: every SSH host runs orcad, deployed from the template this build ships.
orcad-template:
needs: resolve-main
permissions:
contents: read
uses: ./.github/workflows/node-server-tests.yml
with:
ref: ${{ needs.resolve-main.outputs.head_sha }}
build_template: true
build-daily-mac:
needs:
- resolve-main
- orcad-template
if: github.repository == 'stablyai/orca'
outputs:
tag: ${{ steps.release.outputs.tag }}
@@ -89,8 +121,9 @@ jobs:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
fetch-depth: 0
ref: ${{ needs.resolve-main.outputs.head_sha }}
# Version helpers only read HEAD; published versions come from git tags.
fetch-depth: 1
# Why: this job only reads stablyai/orca and never pushes; every write
# goes to the daily repo through a minted App token passed by env.
# Not persisting the checkout credential shrinks the blast radius if a
@@ -145,9 +178,9 @@ jobs:
- name: Setup pnpm
if: steps.freshness.outputs.should_build == 'true'
uses: pnpm/action-setup@v6
uses: pnpm/setup@v2
with:
run_install: false
install: false
- name: Setup Node.js
if: steps.freshness.outputs.should_build == 'true'
@@ -155,6 +188,23 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why: packaging bundles the pinned Node into the macOS terminal host. Keyed on the pin alone
# with no restore-keys, so a pin bump starts empty instead of carrying old archives forward.
- name: Locate the pinned Node runtime cache
if: steps.freshness.outputs.should_build == 'true'
shell: bash
run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV"
- name: Cache the pinned Node runtime
if: steps.freshness.outputs.should_build == 'true'
uses: actions/cache@v5
with:
path: ~/Library/Caches/orca-node-runtime
key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }}
- name: Cache electron-builder downloads
if: steps.freshness.outputs.should_build == 'true'
@@ -167,6 +217,8 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
if: steps.freshness.outputs.should_build == 'true'
uses: nick-fields/retry@v4
@@ -174,7 +226,12 @@ jobs:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.freshness.outputs.should_build == 'true'
# Why: signing is what makes a daily installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
@@ -206,22 +263,26 @@ jobs:
# number free", where a stranded draft still holds one.
names="$(gh release list --repo "$DAILY_REPO" --limit 200 --json name \
--jq '.[].name // empty')"
# Why the main repo's tags decide the base version rather than
# package.json: main's version only moves on `release:` commits, and
# stable patches are cut from release branches that never merge back, so
# package.json can sit several patches behind what users are running. A
# Why git tags, not GitHub releases: unpublishing a buggy cut deletes the
# GitHub release and leaves the tag. That dragged hourlies backwards so
# electron-updater stopped offering them; dailies would do the same. A
# separate token because GH_TOKEN above is the App's, scoped to the
# daily repo. Empty on failure — the script then falls back to
# package.json, which is stale but never wrong enough to fail a build.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh release list \
--repo "$GITHUB_REPOSITORY" --limit 100 --exclude-drafts \
--json tagName --jq '.[].tagName' || true)"
echo "Highest published tag seen: $(head -1 <<<"$published")"
main_tags="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \
"repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \
--jq '.[].ref | sub("^refs/tags/"; "")' || true)"
# Already-shipped channel tags are a second floor so unpublishing a
# buggy main release cannot drag this series below a daily already out.
channel_tags="$(gh release list --repo "$DAILY_REPO" --limit 200 --json tagName \
--jq '.[].tagName' || true)"
published="$main_tags"$'\n'"$channel_tags"
echo "Published version sources: $(grep -c . <<<"$main_tags" || true) main tags, $(grep -c . <<<"$channel_tags" || true) channel tags"
ORCA_PUBLISHED_VERSIONS="$published" ORCA_DAILY_RELEASE_NAMES="$names" \
node config/scripts/daily-build-version.mjs \
>"$RUNNER_TEMP/daily-identity.txt"
grep -E '^(version|build_number)=' "$RUNNER_TEMP/daily-identity.txt"
# Why check rather than trust: the checkout above pins `ref: main`, but a
# Why check rather than trust: the checkout above pins the resolved main commit, but a
# workflow_dispatch runs this file from whatever branch was dispatched. A
# branch that edits this step while main still has the old script yields
# an empty name and an untitled release — silent, and only visible once
@@ -242,6 +303,14 @@ jobs:
# silent, which is correct for unvetted dev artifacts.
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
# After the app build so nothing that cleans out/ can drop it; electron-builder ships it.
- name: Download the orcad deployment template
if: steps.freshness.outputs.should_build == 'true'
uses: actions/download-artifact@v8
with:
name: orcad-template
path: out/orcad-template
# Why a second mint: everything from here on writes to the daily repo, and
# the notary round trip inside the publish step can be tens of minutes. The
# token minted at the top has already spent install + build of its one hour
@@ -308,7 +377,6 @@ jobs:
echo "tag=$TAG" >>"$GITHUB_OUTPUT"
echo "notes_file=$notes_file" >>"$GITHUB_OUTPUT"
- name: Publish daily macOS artifacts
if: steps.freshness.outputs.should_build == 'true'
uses: nick-fields/retry@v4
@@ -320,13 +388,15 @@ jobs:
timeout_minutes: 45
max_attempts: 2
retry_wait_seconds: 30
command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_DAILY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always
command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_DAILY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always
env:
# Why: electron-builder's github publisher targets the repo named in the
# config; the token must therefore carry write access to orca-daily.
GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
ORCA_DAILY_BUILD_VERSION: ${{ steps.daily.outputs.version }}
ORCA_BUILD_COMMIT: ${{ steps.daily.outputs.commit }}
# beforePack and afterPack fail the package when the template is absent.
ORCA_REQUIRE_ORCAD_TEMPLATE: '1'
CSC_LINK: ${{ secrets.MAC_CERTS }}
CSC_KEY_PASSWORD: ${{ secrets.MAC_CERTS_PASSWORD }}
# Why all three: electron-builder's notarize step authenticates to the
@@ -538,3 +608,4 @@ jobs:
tag: ${{ needs.build-daily-mac.outputs.tag }}
ref: ${{ needs.build-daily-mac.outputs.head_sha }}
version: ${{ needs.build-daily-mac.outputs.version }}
orcad_template: true
+40 -4
View File
@@ -58,6 +58,11 @@ on:
description: Version to package, without the leading v
required: true
type: string
orcad_template:
description: Ship the orcad-template artifact the calling run built
required: false
type: boolean
default: false
workflow_dispatch:
inputs:
channel:
@@ -149,9 +154,12 @@ jobs:
fi
# Reachability is the trust test: GitHub serves PR-only commits by SHA,
# so resolving the object is not proof a branch or tag of this repo
# reaches it. Bare + tree:0 keeps this to the commit graph.
# reaches it. Bare + tree:0 keeps this to the commit graph; reftable
# because branches that differ only in casing cannot both be stored by
# the files backend on a case-insensitive runner disk, which fails the
# entire fetch rather than the one ref.
scratch="$RUNNER_TEMP/vet-requested-ref"
git init -q --bare "$scratch"
git init -q --bare --ref-format=reftable "$scratch"
git -C "$scratch" fetch -q --filter=tree:0 "$REPO_URL" '+refs/heads/*:refs/heads/*' '+refs/tags/*:refs/tags/*'
if ! git -C "$scratch" rev-parse --verify --quiet "$REQUESTED_SHA^{commit}" >/dev/null; then
echo "::error::Commit $REQUESTED_SHA is not in stablyai/orca."
@@ -172,6 +180,15 @@ jobs:
# (zizmor: artipacked).
persist-credentials: false
# Why: a relay-era ref (before #27215) needs the Windows relay addon this build no longer compiles.
- name: Refuse a ref that still ships the SSH relay
shell: bash
run: |
if [[ -e config/tsconfig.relay.json ]]; then
echo "::error::This ref predates #27215 and still ships the SSH relay; build a commit at or after 2b4b60a87e."
exit 1
fi
# Why a guard and not just a build: the workflow file comes from the
# dispatch ref, but the packaging config comes from the *built* commit. A
# branch cut before Windows dev builds landed has a config that ignores
@@ -191,15 +208,18 @@ jobs:
# pnpm must be on PATH before setup-node so setup-node can locate the store.
- name: Setup pnpm
uses: pnpm/action-setup@v6
uses: pnpm/setup@v2
with:
run_install: false
install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Caches the Electron binary and electron-builder's tool downloads (nsis,
# winCodeSign). Same key shape as release-cut's Windows leg.
@@ -216,6 +236,8 @@ jobs:
# Why retried: pnpm install triggers electron's postinstall, which pulls the
# Electron binary from GitHub release assets, and that CDN returns transient
# 504s often enough to lose a build to it.
# Why host-only: this job packages only for its own runner OS and
# architecture, so the default host-scoped install is deliberate.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
@@ -224,6 +246,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why the packaging check runs before the 20-minute build: it only needs
# node_modules, and a stale config should cost seconds rather than a build.
- name: Verify dev-channel packaging identity
@@ -239,6 +265,14 @@ jobs:
# is correct for unvetted artifacts. Same as the mac dev channels.
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
# After the app build so nothing that cleans out/ can drop it; electron-builder ships it.
- name: Download the orcad deployment template
if: inputs.orcad_template
uses: actions/download-artifact@v8
with:
name: orcad-template
path: out/orcad-template
# Why the token is minted here and not at the top: installation tokens live
# one hour and nothing before this point writes anything.
- name: Mint dev channel repo token
@@ -276,6 +310,8 @@ jobs:
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
ORCA_BUILD_COMMIT: ${{ inputs.ref }}
# beforePack and afterPack fail the package when the template is absent.
ORCA_REQUIRE_ORCAD_TEMPLATE: ${{ inputs.orcad_template && '1' || '' }}
# Why: electron-publish refuses to upload into a release published more
# than two hours ago (gitHubPublisher.getOrCreateRelease). The mac leg
# publishes the draft live as soon as *it* finishes, so a slow notary
+276
View File
@@ -0,0 +1,276 @@
name: Docs site
on:
# Stable desktop releases are the production publication boundary. The
# release gate below excludes mobile and prerelease tags from this trigger.
release:
types: [published]
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- 'docs/site/**'
- '.github/workflows/docs.yml'
workflow_dispatch:
inputs:
tag:
description: 'Stable desktop release tag to redeploy (vX.Y.Z)'
required: true
type: string
permissions:
contents: read
concurrency:
# Serialize production runs so a delayed older release cannot overwrite a
# newer deployment. PR checks may still run concurrently by pull request.
group: ${{ github.event_name == 'pull_request' && format('docs-pr-{0}', github.event.pull_request.number) || 'docs-production' }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
defaults:
run:
# The Vercel project is rooted at `.`; this package directory is the
# complete upload and build context.
working-directory: docs/site
jobs:
# This job deliberately has no deployment credentials and runs for every PR,
# including forks.
check:
name: Build and test
if: github.event_name == 'pull_request' && github.event.pull_request.draft != true
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout pull request
uses: actions/checkout@v6
with:
persist-credentials: false
- name: Setup pnpm
# v5 avoids the v6 bootstrap/shim regression when pinning pnpm 10.
uses: pnpm/action-setup@v5
with:
version: 10.34.6
package_json_file: docs/site/package.json
run_install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 22
# The repository root pins pnpm 12 while this isolated app pins 10;
# setup-node's cache probe runs at the root and would auto-switch.
package-manager-cache: false
- name: Install site dependencies
run: pnpm --ignore-workspace install --frozen-lockfile
- name: Run package tests
run: pnpm --ignore-workspace test
- name: Lint site
run: pnpm --ignore-workspace lint
- name: Typecheck site
run: pnpm --ignore-workspace exec tsc --noEmit --incremental false
- name: Build site
run: pnpm --ignore-workspace build
release_gate:
name: Authorize release
if: >-
github.repository == 'stablyai/orca' &&
(github.event_name == 'release' || github.event_name == 'workflow_dispatch')
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
deploy: ${{ steps.validate.outputs.deploy }}
commit_sha: ${{ steps.validate.outputs.commit_sha }}
steps:
- name: Validate stable desktop tag
id: validate
env:
EVENT_NAME: ${{ github.event_name }}
RELEASE_TAG: ${{ github.event.release.tag_name }}
INPUT_TAG: ${{ inputs.tag }}
RELEASE_PRERELEASE: ${{ github.event.release.prerelease }}
RELEASE_DRAFT: ${{ github.event.release.draft }}
RELEASE_AUTHOR: ${{ github.event.release.author.login }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
WORKFLOW_REF: ${{ github.ref }}
GH_TOKEN: ${{ github.token }}
shell: bash
working-directory: .
run: |
set -euo pipefail
tag="$INPUT_TAG"
[[ "$EVENT_NAME" == "release" ]] && tag="$RELEASE_TAG"
# Match the stable desktop format used by release-policy.yml. This
# intentionally rejects mobile-* and all -rc.* tags.
stable_tag=false
[[ "$tag" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] && stable_tag=true
if [[ "$stable_tag" != "true" ]]; then
echo "Release $tag is not a stable desktop release; skipping docs deployment."
echo "deploy=false" >> "$GITHUB_OUTPUT"
exit 0
fi
if [[ "$EVENT_NAME" == "release" ]]; then
authorized_ref=true
authorized_author=false
[[ "$RELEASE_AUTHOR" == "github-actions[bot]" ]] && authorized_author=true
release_state_ok=false
[[ "$RELEASE_PRERELEASE" == "false" && "$RELEASE_DRAFT" == "false" ]] && release_state_ok=true
else
authorized_ref=false
[[ "$WORKFLOW_REF" == "refs/heads/$DEFAULT_BRANCH" ]] && authorized_ref=true
release_json=''
for attempt in 1 2 3; do
if release_json="$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$tag" 2>/dev/null)"; then
break
fi
if [[ "$attempt" -eq 3 ]]; then
echo "::error::Release metadata for $tag was not available after 3 attempts."
exit 1
fi
sleep "$((attempt * 5))"
done
authorized_author=false
[[ "$(jq -r '.author.login' <<<"$release_json")" == "github-actions[bot]" ]] && authorized_author=true
release_state_ok=false
if [[ "$(jq -r '.tag_name' <<<"$release_json")" == "$tag" &&
"$(jq -r '.prerelease' <<<"$release_json")" == "false" &&
"$(jq -r '.draft' <<<"$release_json")" == "false" ]]; then
release_state_ok=true
fi
fi
# Resolve the tag to an immutable commit before handing it to the
# deployment job. This prevents a force-moved tag from changing the
# source between authorization and checkout.
tag_ref_json=''
for attempt in 1 2 3; do
if tag_ref_json="$(gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$tag" 2>/dev/null)"; then
break
fi
if [[ "$attempt" -eq 3 ]]; then
echo "::error::Git ref for $tag was not available after 3 attempts."
exit 1
fi
sleep "$((attempt * 5))"
done
tag_object_sha="$(jq -er '.object.sha' <<<"$tag_ref_json")"
tag_object_type="$(jq -er '.object.type' <<<"$tag_ref_json")"
commit_sha="$tag_object_sha"
if [[ "$tag_object_type" == 'tag' ]]; then
commit_sha="$(gh api "repos/$GITHUB_REPOSITORY/git/tags/$tag_object_sha" | jq -er '.object.sha')"
fi
if [[ ! "$commit_sha" =~ ^[0-9a-f]{40}$ ]]; then
echo "::error::Could not resolve $tag to a commit SHA."
exit 1
fi
echo "commit_sha=$commit_sha" >> "$GITHUB_OUTPUT"
# Stable tags cut before the docs package was merged are valid Orca
# releases, but cannot produce a docs deployment. Skip them before
# requesting the protected production environment.
docs_source_available=false
for attempt in 1 2 3; do
if gh api "repos/$GITHUB_REPOSITORY/contents/docs/site/package.json?ref=$commit_sha" >/dev/null 2>&1; then
docs_source_available=true
break
fi
[[ "$attempt" -eq 3 ]] || sleep "$((attempt * 5))"
done
if [[ "$docs_source_available" != "true" ]]; then
echo "Release $tag does not contain docs/site; skipping docs deployment."
echo "deploy=false" >> "$GITHUB_OUTPUT"
exit 0
fi
deploy=false
if [[ "$authorized_ref" == "true" && "$authorized_author" == "true" && "$release_state_ok" == "true" ]]; then
deploy=true
else
echo "Release $tag is not an authorized stable desktop release; skipping docs deployment."
fi
echo "deploy=$deploy" >> "$GITHUB_OUTPUT"
production:
name: Production
if: >-
(github.event_name == 'release' || github.event_name == 'workflow_dispatch') &&
needs.release_gate.result == 'success' &&
needs.release_gate.outputs.deploy == 'true'
needs: release_gate
runs-on: ubuntu-latest
timeout-minutes: 15
environment:
name: docs-production
url: https://www.onorca.dev/docs
env:
VERCEL_TELEMETRY_DISABLED: '1'
steps:
- name: Checkout released tag
uses: actions/checkout@v6
with:
ref: ${{ needs.release_gate.outputs.commit_sha }}
persist-credentials: false
- name: Setup pnpm
# v5 avoids the v6 bootstrap/shim regression when pinning pnpm 10.
uses: pnpm/action-setup@v5
with:
version: 10.34.6
package_json_file: docs/site/package.json
run_install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 22
package-manager-cache: false
- name: Install site dependencies
run: pnpm --ignore-workspace install --frozen-lockfile
- name: Verify Vercel credentials
env:
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
run: |
set -euo pipefail
test -n "${VERCEL_TOKEN:-}" || { echo '::error::VERCEL_TOKEN is not configured for docs-production.'; exit 1; }
test -n "${VERCEL_ORG_ID:-}" || { echo '::error::VERCEL_ORG_ID is not configured for docs-production.'; exit 1; }
test -n "${VERCEL_PROJECT_ID:-}" || { echo '::error::VERCEL_PROJECT_ID is not configured for docs-production.'; exit 1; }
- name: Pull Vercel production settings
env:
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
run: pnpm --ignore-workspace exec vercel pull --yes --non-interactive --environment=production
- name: Run package tests
run: pnpm --ignore-workspace test
- name: Lint site
run: pnpm --ignore-workspace lint
- name: Typecheck site
run: pnpm --ignore-workspace exec tsc --noEmit --incremental false
- name: Build production site
run: pnpm --ignore-workspace exec vercel build --prod --non-interactive
- name: Deploy production site
env:
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
run: pnpm --ignore-workspace exec vercel deploy --prebuilt --prod --yes --non-interactive
+407 -196
View File
@@ -17,22 +17,44 @@ on:
description: JSON array of changed specs; empty runs the full suite
required: false
type: string
run_changed_e2e:
description: PR detector found specs requiring the general changed-spec lane.
required: false
type: boolean
default: true
needs_build:
description: PR detector found an Electron consumer requiring build and native cache.
required: false
type: boolean
default: true
# Why: artifact names are unique per workflow run, and pr.yml calls this workflow twice.
artifact_suffix:
description: Appended to the build and trace artifact names so two calls in one run do not collide
required: false
type: string
default: ''
workflow_dispatch:
inputs:
ref:
description: Ref to check out (defaults to the workflow ref)
required: false
type: string
test_files:
description: JSON array of specs to run; empty runs the full suite
required: false
type: string
schedule:
# Why: GitHub cron uses UTC; these slots map to 10am and 3pm
# America/Phoenix for the default-branch E2E run.
- cron: '0 17,22 * * *'
# One complete daily reference run; targeted PR coverage remains unchanged.
- cron: '0 17 * * *'
jobs:
build:
name: build e2e app
if: inputs.test_files == '' || github.event_name != 'pull_request' || inputs.needs_build
runs-on: ubuntu-latest
timeout-minutes: 10
timeout-minutes: 15
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- name: Checkout
@@ -40,62 +62,113 @@ jobs:
with:
ref: ${{ inputs.ref || github.ref }}
# Why: the E2E build compiles native modules via node-gyp. Mirrors the
# install step in pr.yml's verify job so E2E doesn't hit missing-toolchain
# errors.
- name: Install native build tools
run: sudo apt-get update && sudo apt-get install -y build-essential python3
env:
ORCA_E2E_APT_PACKAGES: build-essential python3
run: &install_e2e_tools |
if [ "$ORCA_E2E_APT_PACKAGES" = 'build-essential python3' ] &&
command -v make >/dev/null && command -v g++ >/dev/null && command -v python3 >/dev/null; then
exit 0
fi
read -r -a packages <<< "$ORCA_E2E_APT_PACKAGES"
# Why apt-mirrors.txt: runner images resolve mirror+file: sources through it, so editing the lists alone leaves Azure in use.
for source in /etc/apt/apt-mirrors.txt /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
if [ -f "$source" ]; then
sudo sed -i 's|https*://azure\.archive\.ubuntu\.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' "$source"
fi
done
sudo tee /etc/apt/apt.conf.d/99-orca-e2e >/dev/null <<'APTCONF'
Acquire::http::Timeout "15";
Acquire::https::Timeout "15";
Acquire::Retries "1";
APTCONF
timeout 120 sudo apt-get update
timeout 300 sudo apt-get install -y "${packages[@]}" || timeout 300 sudo apt-get install -y --fix-missing "${packages[@]}"
# Why pnpm first: setup-node needs pnpm on PATH to locate the store it caches.
# Without that cache every E2E job re-downloaded the whole dependency set.
- name: Setup pnpm
uses: pnpm/action-setup@v6
# Why: the build's plain-Node daemon smoke load resolves node-pty.
- uses: ./.github/actions/install-node-dependencies
id: deps
with:
run_install: false
native-runtime: node
persist-native-cache: 'false'
cache-electron-package: 'true'
- name: Setup Node.js
uses: actions/setup-node@v6
# Save Node's ABI before the Electron preparation replaces the same build paths.
- name: Save compiled Node native modules
if: steps.deps.outputs.native-cache-key != '' && steps.deps.outputs.native-cache-hit != 'true'
uses: actions/cache/save@v5
with:
node-version-file: package.json
cache: pnpm
# Why: this job runs the same pnpm install path as pr.yml's verify
# job, so it needs the same pinned node-gyp override to avoid pnpm's
# broken bundled gyp_main.py on Linux.
- name: Use external node-gyp to avoid pnpm's bundled copy (Linux only)
if: runner.os == 'Linux'
run: |
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- name: Install dependencies
run: pnpm install --frozen-lockfile
path: ${{ steps.deps.outputs.native-cache-path }}
key: ${{ steps.deps.outputs.native-cache-key }}
# Why: building here avoids parallel builds inside Playwright globalSetup;
# paired-browser specs also need the standalone web bundle.
- name: Build E2E outputs
env:
VITE_EXPOSE_STORE: 'true'
run: |
npx electron-vite build --mode e2e
pnpm run build:web-from-renderer
pnpm run build:relay
ORCA_CI_MAIN_SOURCEMAPS: '0'
run: pnpm run build:electron-vite:parallel --mode e2e
# The CLI emits into out/main too; start only after Electron finishes clearing that tree.
- name: Build shared E2E CLI
id: e2e-cli
background: true
run: |
if node -e 'process.exit(require("./package.json").scripts["prepare:cli-output"] ? 0 : 1)'; then
pnpm run build:cli
else
echo "Older ref: let each consumer build and install its CLI."
fi
- name: Project shared E2E web client
run: pnpm run build:web-from-renderer
- wait: e2e-cli
- name: Check shared native runtime action
id: native-runtime-action
run: |
if [ -f .github/actions/prepare-native-runtime/action.yml ]; then
echo 'available=true' >> "$GITHUB_OUTPUT"
else
echo 'available=false' >> "$GITHUB_OUTPUT"
fi
# The orcad builds downstream refuse to run without the page; older refs predate the action.
- uses: ./.github/actions/build-mobile-web-bundle
if: hashFiles('.github/actions/build-mobile-web-bundle/action.yml') != ''
# Only the completed output upload overlaps the change from Node to Electron ABI.
- name: Upload E2E build output
id: e2e-build-upload
background: true
uses: actions/upload-artifact@v7
with:
name: e2e-build-out
name: e2e-build-out${{ inputs.artifact_suffix }}
path: out/
# Why: build-relay.mjs writes each relay's marker as `out/relay/<platform>/.version`,
# and upload-artifact drops dotfiles by default — consumers then fail SSH specs with
# "local relay build is missing its version marker".
include-hidden-files: true
retention-days: 1
if-no-files-found: error
- name: Prepare shared Electron native runtime
id: electron-native-runtime
if: steps.native-runtime-action.outputs.available == 'true' && steps.deps.outputs.node-version != ''
uses: ./.github/actions/prepare-native-runtime
with:
native-runtime: electron
node-version: ${{ steps.deps.outputs.node-version }}
- name: Prepare Electron natives for older refs
if: steps.native-runtime-action.outputs.available != 'true' || steps.deps.outputs.node-version == ''
uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- wait: e2e-build-upload
e2e:
name: e2e ${{ matrix.shard_name }}
needs: build
needs: [build]
if: inputs.test_files == ''
runs-on: ubuntu-latest
timeout-minutes: 30
@@ -103,26 +176,37 @@ jobs:
fail-fast: false
matrix:
include:
- shard: '1/10'
shard_name: 1-of-10
- shard: '2/10'
shard_name: 2-of-10
- shard: '3/10'
shard_name: 3-of-10
- shard: '4/10'
shard_name: 4-of-10
- shard: '5/10'
shard_name: 5-of-10
- shard: '6/10'
shard_name: 6-of-10
- shard: '7/10'
shard_name: 7-of-10
- shard: '8/10'
shard_name: 8-of-10
- shard: '9/10'
shard_name: 9-of-10
- shard: '10/10'
shard_name: 10-of-10
# Fourteen scheduled runs averaged 24.6 minutes per shard; shards 4
# and 9 repeatedly hit the 30-minute cap. A 12-way trial still left
# one 30-minute shard, so 14 gives the suite enough failure headroom.
- shard: '1/14'
shard_name: 1-of-14
- shard: '2/14'
shard_name: 2-of-14
- shard: '3/14'
shard_name: 3-of-14
- shard: '4/14'
shard_name: 4-of-14
- shard: '5/14'
shard_name: 5-of-14
- shard: '6/14'
shard_name: 6-of-14
- shard: '7/14'
shard_name: 7-of-14
- shard: '8/14'
shard_name: 8-of-14
- shard: '9/14'
shard_name: 9-of-14
- shard: '10/14'
shard_name: 10-of-14
- shard: '11/14'
shard_name: 11-of-14
- shard: '12/14'
shard_name: 12-of-14
- shard: '13/14'
shard_name: 13-of-14
- shard: '14/14'
shard_name: 14-of-14
steps:
- name: Checkout
@@ -130,50 +214,23 @@ jobs:
with:
ref: ${{ inputs.ref || github.ref }}
# Why: pnpm install rebuilds native modules, and those postinstall
# scripts still need the Linux toolchain even though this shard reuses
# the prebuilt Electron output.
- name: Install native build tools
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk python3 zsh
# Native cache misses need the compiler, Electron needs Xvfb, and paired
# Quick Open needs ripgrep. Install them in one apt transaction per shard.
- name: Install native build and headless UI tools
# The Azure archive took 16 minutes for one font package; bound setup
# separately so a slow mirror cannot consume the shard's test budget.
env: &e2e_tool_packages
ORCA_E2E_APT_PACKAGES: build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils xdotool
run: *install_e2e_tools
# Why: Electron on Linux needs an X display even when the app
# suppresses mainWindow.show() via ORCA_E2E_HEADLESS. xvfb provides a
# virtual framebuffer so Chromium can initialize without a real display.
- name: Install xvfb
run: sudo apt-get install -y xvfb
# Why pnpm first: setup-node needs pnpm on PATH to locate the store it caches.
# Without that cache every E2E job re-downloaded the whole dependency set.
- name: Setup pnpm
uses: pnpm/action-setup@v6
- uses: ./.github/actions/install-node-dependencies
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
# Why: this job runs the same pnpm install path as pr.yml's verify
# job, so it needs the same pinned node-gyp override to avoid pnpm's
# broken bundled gyp_main.py on Linux. Gate on runner.os matches
# release.yml so the invariant "this workaround is Linux-only" is
# consistent across all three workflows, even though this job
# currently pins runs-on: ubuntu-latest.
- name: Use external node-gyp to avoid pnpm's bundled copy (Linux only)
if: runner.os == 'Linux'
run: |
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- name: Install dependencies
run: pnpm install --frozen-lockfile
native-runtime: electron
- name: Download E2E build output
uses: actions/download-artifact@v8
with:
name: e2e-build-out
name: e2e-build-out${{ inputs.artifact_suffix }}
path: out/
# Why: the Electron suite is wall-clock constrained on OSS runners, but
@@ -184,14 +241,51 @@ jobs:
# artifact instead of starting five concurrent electron-vite builds.
# ORCA_E2E_FORWARD_APP_LOGS keeps startup failures visible when Electron
# launches but never creates a BrowserWindow.
- name: Balance E2E shard from timing evidence
env:
ORCA_BACKGROUND_LAUNCH: '1'
SKIP_BUILD: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
ORCA_E2E_WEB_CLIENT: '1'
run: |
mkdir -p ci-shards
pnpm exec playwright test --config tests/playwright.config.ts --project=electron-headless --list --reporter=json > ci-shards/discovery.json
export ORCA_SHARD_SOURCE_SHA="$(git rev-parse HEAD)"
node config/scripts/ci-e2e-shard-plan.mjs ci-shards/discovery.json '${{ matrix.shard }}' ci-shards
pnpm exec playwright test --config tests/playwright.config.ts --project=electron-headless --test-list=ci-shards/selected.txt --list --reporter=json > ci-shards/selected-discovery.json
node config/scripts/ci-e2e-shard-plan.mjs --verify ci-shards/assignment.json ci-shards/selected-discovery.json
- name: Run E2E tests (${{ matrix.shard_name }})
run: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --shard=${{ matrix.shard }}
env:
PLAYWRIGHT_JSON_OUTPUT_FILE: ci-shards/results.json
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 pnpm run test:e2e --test-list=ci-shards/selected.txt --reporter=list,json
- name: Summarize E2E failures
if: always()
continue-on-error: true
run: node config/scripts/ci-e2e-failure-summary.mjs ci-shards/results.json
- name: Upload E2E shard assignment
if: always()
# Diagnostic upload outages must not change the test verdict.
continue-on-error: true
uses: actions/upload-artifact@v7
with:
name: e2e-shard-${{ matrix.shard_name }}-attempt-${{ github.run_attempt }}
path: ci-shards/
retention-days: 14
if-no-files-found: warn
# The frame benchmark needs a mapped window, which the headless shards exclude.
- name: Run worktree first-paint benchmark
if: matrix.shard == '1/14'
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm exec playwright test tests/e2e/worktree-switch-first-paint.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1
# Why: Playwright retains traces/screenshots only on failure. Uploading
# them as an artifact makes post-mortem debugging on CI possible without
# re-running locally.
- name: Upload Playwright traces
if: failure()
if: failure() || cancelled()
uses: actions/upload-artifact@v7
with:
name: playwright-traces-${{ matrix.shard_name }}
@@ -201,11 +295,9 @@ jobs:
changed-e2e:
name: changed e2e specs
needs: build
if: inputs.test_files != ''
needs: [build]
if: inputs.test_files != '' && (github.event_name != 'pull_request' || inputs.run_changed_e2e)
runs-on: ubuntu-latest
# Why 45: pr.yml now maps SSH source edits onto Docker-backed specs, so this lane can
# pay a container image build plus ~22 serial SSH tests on top of the changed specs.
timeout-minutes: 45
steps:
@@ -217,139 +309,258 @@ jobs:
- name: Install native build and headless UI tools
# Why ripgrep: Quick Open's bounded host-side search requires rg instead of an
# unbounded inventory fallback; the paired fixture exercises that real boundary.
# Why openssh-client: the Docker-SSH fixture shells out to ssh/ssh-keygen, and this
# lane now receives those specs from pr.yml's SSH source mapping.
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh
env: *e2e_tool_packages
run: *install_e2e_tools
# Why pnpm first: setup-node needs pnpm on PATH to locate the store it caches.
# Without that cache every E2E job re-downloaded the whole dependency set.
- name: Setup pnpm
uses: pnpm/action-setup@v6
- uses: ./.github/actions/install-node-dependencies
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
- name: Use external node-gyp to avoid pnpm's bundled copy
run: |
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- name: Install dependencies
run: pnpm install --frozen-lockfile
native-runtime: electron
- name: Download E2E build output
uses: actions/download-artifact@v8
with:
name: e2e-build-out
name: e2e-build-out${{ inputs.artifact_suffix }}
path: out/
- name: Run changed E2E specs
env:
TEST_FILES_JSON: ${{ inputs.test_files }}
run: |
mapfile -t TEST_FILES < <(jq -r '.[] | select(
. != "tests/e2e/ssh-startup-exec-readiness.spec.ts" and
. != "tests/e2e/paired-startup-exec-readiness.spec.ts"
)' <<<"$TEST_FILES_JSON")
if [ -f config/scripts/ci-e2e-job-selection.mjs ]; then
printf '%s\n' "$TEST_FILES_JSON" | node config/scripts/ci-e2e-job-selection.mjs > "$RUNNER_TEMP/general-e2e-specs"
else
# Dispatching an older ref retains its existing dedicated-owner exclusions.
jq -r '.[] | select(
. != "tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts" and
. != "tests/e2e/terminal-ibus-hangul-native.spec.ts" and
. != "tests/e2e/orcad-serve-mode-switch.spec.ts" and
. != "tests/e2e/workspace-layout-oracle-headless.spec.ts" and
. != "tests/e2e/windows-missing-appdata-startup.spec.ts" and
. != "tests/e2e/file-explorer-copy-file-windows.spec.ts" and
(startswith("tests/e2e/ssh-orcad-") | not)
)' <<<"$TEST_FILES_JSON" > "$RUNNER_TEMP/general-e2e-specs"
fi
mapfile -t TEST_FILES < "$RUNNER_TEMP/general-e2e-specs"
if [ "${#TEST_FILES[@]}" -eq 0 ]; then
echo "Changed startup-readiness specs are owned by the dedicated live lane."
echo "Changed specs are all owned by dedicated lanes."
exit 0
fi
E2E_ENV=(SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay")
# Second clause: a spec that reads ORCA_E2E_SSH_DOCKER test.skip()s itself without it, so
# naming only one trigger silently skipped every other Docker-SSH spec in this lane.
# The first clause stays because that spec needs Docker without referencing the variable.
if printf '%s\n' "${TEST_FILES[@]}" | grep -qx 'tests/e2e/ephemeral-vm-provisioned-root.spec.ts' \
|| grep -l 'ORCA_E2E_SSH_DOCKER' "${TEST_FILES[@]}" >/dev/null 2>&1; then
E2E_ENV+=(ORCA_E2E_SSH_DOCKER=1)
fi
E2E_ENV=(SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1)
E2E_PROJECT_ARGS=()
if grep -l '@headful' "${TEST_FILES[@]}" >/dev/null; then
E2E_PROJECT_ARGS+=(--project=electron-headful)
fi
xvfb-run --auto-servernum env "${E2E_ENV[@]}" \
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env "${E2E_ENV[@]}" \
pnpm run test:e2e "${TEST_FILES[@]}" --workers=1 "${E2E_PROJECT_ARGS[@]}"
- name: Upload Playwright traces
if: failure()
if: failure() || cancelled()
uses: actions/upload-artifact@v7
with:
name: playwright-traces-changed
name: playwright-traces-changed${{ inputs.artifact_suffix }}
path: test-results/
retention-days: 7
if-no-files-found: ignore
ssh-docker-watcher-isolation:
name: ssh docker watcher isolation
needs: build
if: >-
inputs.test_files == '' ||
contains(inputs.test_files, 'tests/e2e/ssh-startup-exec-readiness.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/paired-startup-exec-readiness.spec.ts')
ssh-browser-network-route:
name: ssh browser network route
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts')
runs-on: ubuntu-latest
# Why 35: the parking, retention, startup-exec, and paired parity specs run
# serially on isolated Electron/SSH fixtures after watcher isolation.
timeout-minutes: 35
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@v6
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.ref }}
- name: Install native build and headless UI tools
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 xvfb zsh
# Why pnpm first: setup-node needs pnpm on PATH to locate the store it caches.
# Without that cache every E2E job re-downloaded the whole dependency set.
- name: Setup pnpm
uses: pnpm/action-setup@v6
- uses: ./.github/actions/install-node-dependencies
with:
run_install: false
native-runtime: node
- name: Install SSH client
run: sudo apt-get update && sudo apt-get install -y openssh-client
- name: Run Docker SSH browser network route journeys
env:
ORCA_BACKGROUND_LAUNCH: '1'
ORCA_RUN_DOCKER_SSH_BROWSER_E2E: '1'
run: node_modules/.bin/vitest run --config config/vitest.config.ts tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts
- name: Setup Node.js
uses: actions/setup-node@v6
orcad-serve-mode-switch:
name: orca serve Electron/orcad mode switch (D7)
needs: [build]
if: >-
inputs.test_files == '' ||
contains(inputs.test_files, 'tests/e2e/orcad-serve-mode-switch.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/workspace-layout-oracle-headless.spec.ts')
runs-on: ubuntu-latest
timeout-minutes: 45
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
node-version-file: package.json
cache: pnpm
# Why: same Linux-only node-gyp pin as build/e2e jobs so the workaround
# stays consistent across workflows even while this job is ubuntu-latest.
- name: Use external node-gyp to avoid pnpm's bundled copy (Linux only)
if: runner.os == 'Linux'
run: |
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Download E2E build output
uses: actions/download-artifact@v8
ref: ${{ inputs.ref || github.ref }}
persist-credentials: false
- name: Install headless tools
run: sudo apt-get update && sudo apt-get install -y build-essential ripgrep xvfb openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
name: e2e-build-out
native-runtime: electron
- uses: actions/download-artifact@v8
with:
name: e2e-build-out${{ inputs.artifact_suffix }}
path: out/
# Why: this is the release-path proof that the deployed Linux relay keeps
# its PTY and explorer live across a real watcher SIGSEGV.
- name: Run Docker SSH watcher isolation E2E
run: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-watcher-isolation
# Why always(): this lane gates SSH parking/retention plus startup-exec
# readiness across live SSH, headed paired, and headless serve topologies.
- name: Run Docker SSH terminal parking + startup readiness E2E
if: always()
run: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-terminal-parking
- name: Upload watcher isolation traces
# The packaged orcad slot the T6-11 launcher runs: its own node-pty under the pinned Node.
# The template is what `orca serve`'s default selection materializes that slot from.
- name: Build this runner's orcad slot and template
run: |
slot="$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)"
pnpm build:orcad-prebuilds
pnpm build:orcad-prebuilds --require-slots "$slot"
pnpm build:orcad
node config/scripts/build-orcad-template.mjs --targets "$slot"
- name: Switch serve hosts on one profile
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/orcad-serve-mode-switch.spec.ts')
env:
SKIP_BUILD: '1'
ORCA_E2E_ORCAD_SERVE: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/orcad-serve-mode-switch.spec.ts --project=electron-headless --workers=1
# The layout oracle's headless scenarios run on both serve hosts, so they need this job's orcad.
- name: Layout oracle on headless serve (orcad and Electron)
if: >-
!cancelled() &&
(inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/workspace-layout-oracle-headless.spec.ts'))
env:
SKIP_BUILD: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/workspace-layout-oracle-headless.spec.ts --project=electron-headless --workers=1
- uses: actions/upload-artifact@v7
if: failure()
uses: actions/upload-artifact@v7
with:
name: playwright-traces-ssh-docker-watcher-isolation
name: orcad-serve-mode-switch-traces${{ inputs.artifact_suffix }}
path: test-results/
retention-days: 7
if-no-files-found: ignore
# Every managed-orcad SSH spec on a Docker host, including #24979's relay-era conversion. Needs the
# orcad template for the fixture's target (Debian, linux-x64-glibc), which only this job builds.
orcad-ssh-docker:
name: ssh on managed orcad (Docker ${{ matrix.shard }})
needs: [build]
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-orcad-')
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
shard: ['1/2', '2/2']
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.ref }}
persist-credentials: false
# COMPAT(legacy-relay-bridge): the bridge spec builds 1.4.224's relay from its tagged sources.
- name: Fetch the 1.4.224 relay sources
run: git fetch --no-tags --depth=1 origin +refs/tags/v1.4.224:refs/tags/v1.4.224
- name: Install headless tools
run: sudo apt-get update && sudo apt-get install -y build-essential ripgrep xvfb openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- uses: actions/download-artifact@v8
with:
name: e2e-build-out${{ inputs.artifact_suffix }}
path: out/
# Built to a path the app does not look at by default, so the relay-era phase has no template.
- name: Build the linux-x64-glibc orcad template
run: |
pnpm build:orcad-prebuilds
pnpm build:orcad-prebuilds --require-slots linux-x64-glibc
node config/scripts/build-orcad-template.mjs --targets linux-x64-glibc
mv out/orcad-template "$RUNNER_TEMP/orcad-convert-template"
# The Windows CLI matrix collects here too and skips: it needs a Windows host descriptor.
- name: Run managed SSH specs on a Docker host
env:
SKIP_BUILD: '1'
ORCA_E2E_SSH_DOCKER: '1'
ORCA_E2E_ORCAD_CONVERT_HOST: docker
ORCA_E2E_FORWARD_APP_LOGS: '1'
SHARD: ${{ matrix.shard }}
run: |
export ORCA_E2E_ORCAD_CONVERT_TEMPLATE="$RUNNER_TEMP/orcad-convert-template"
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-orcad- --project=electron-headless --workers=1 --shard="$SHARD"
- uses: actions/upload-artifact@v7
if: failure()
with:
name: orcad-ssh-docker-traces-${{ strategy.job-index }}${{ inputs.artifact_suffix }}
path: test-results/
retention-days: 7
if-no-files-found: ignore
# D7 on Windows: both hosts share <userData>\daemon and so one daemon pipe. Built here rather than
# from the Linux e2e build because the slot, template and addons are Windows-native. Also runs the
# missing-AppData startup check and the explorer file-copy check, which need the same Windows e2e build.
orcad-serve-mode-switch-windows:
name: orca serve on Windows (D7 mode switch, missing AppData)
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/orcad-serve-mode-switch.spec.ts') || contains(inputs.test_files, 'tests/e2e/windows-missing-appdata-startup.spec.ts') || contains(inputs.test_files, 'tests/e2e/file-explorer-copy-file-windows.spec.ts')
runs-on: windows-2022
timeout-minutes: 45
env:
NODE_OPTIONS: --max-old-space-size=4096
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.ref }}
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- uses: ./.github/actions/prepare-orcad-prebuilds
with:
resolve-windows-cache: 'true'
restore-windows-cache: 'true'
- uses: ./.github/actions/prepare-windows-cli-launcher
with:
arches: x64
- uses: ./.github/actions/build-mobile-web-bundle
- name: Build the e2e app, CLI, orcad slot and template
shell: bash
env:
ORCA_REUSE_PREPARED_WINDOWS_SECURITY: 'true'
run: |
pnpm run build:wsl-guest
pnpm exec electron-vite build --mode e2e
pnpm run build:cli
pnpm build:orcad
node config/scripts/build-windows-process-tree-addon.mjs --arch=x64 --reuse-prepared-runtime
node config/scripts/build-orcad-template.mjs --targets win32-x64
# A profile-less Windows session has no roaming AppData; Electron 43 used to crash natively there.
- name: Start serve with no AppData folder
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/windows-missing-appdata-startup.spec.ts')
env:
SKIP_BUILD: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
run: pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/windows-missing-appdata-startup.spec.ts --project=electron-headless --workers=1
# Explorer pastes only a real CF_HDROP, which needs the Windows clipboard and shell.
- name: Copy a file from the explorer
if: ${{ !cancelled() && (inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/file-explorer-copy-file-windows.spec.ts')) }}
env:
SKIP_BUILD: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
run: pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/file-explorer-copy-file-windows.spec.ts --project=electron-headless --workers=1
- name: Switch serve hosts on one profile
if: ${{ !cancelled() && (inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/orcad-serve-mode-switch.spec.ts')) }}
env:
SKIP_BUILD: '1'
ORCA_E2E_ORCAD_SERVE: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
ORCA_E2E_PRESERVE_PROFILE_LOGS_DIR: ${{ github.workspace }}\test-results\profile-logs
run: pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/orcad-serve-mode-switch.spec.ts --project=electron-headless --workers=1
- uses: actions/upload-artifact@v7
if: failure()
with:
name: orcad-serve-mode-switch-windows-traces${{ inputs.artifact_suffix }}
path: test-results/
retention-days: 7
if-no-files-found: ignore
@@ -0,0 +1,27 @@
name: Git command termination runtime
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- 'src/main/git/command-runner/spawned-command-tree-kill*'
- '.github/workflows/git-command-termination-runtime.yml'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: git-command-termination-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
windows-exit:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: windows-latest
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Verify exited native child does not trigger taskkill
run: node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/git/command-runner/spawned-command-tree-kill.test.ts
+15 -7
View File
@@ -43,9 +43,9 @@ jobs:
node-version-file: package.json
- name: Setup pnpm
uses: pnpm/action-setup@v6
uses: pnpm/setup@v2
with:
run_install: false
install: false
# Why: Linux golden E2E uses the same native install path as PR/release CI,
# which needs pnpm to bypass its non-executable bundled gyp_main.py.
@@ -66,27 +66,29 @@ jobs:
- name: Run golden E2E tests on Linux
if: runner.os == 'Linux'
run: |
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e -- tests/e2e/golden-core-flows.spec.ts
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e tests/e2e/golden-core-flows.spec.ts
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:workspace-session-golden
if [ -f tests/e2e/golden-fresh-profile-terminal.spec.ts ]; then
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e -- tests/e2e/golden-fresh-profile-terminal.spec.ts tests/e2e/golden-shell-command.spec.ts
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e tests/e2e/golden-fresh-profile-terminal.spec.ts tests/e2e/golden-shell-command.spec.ts
fi
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:posix-profile-index-golden
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:agent-tui-golden
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:tab-bar-agent-launch-golden
xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:source-control-golden
- name: Run golden E2E tests on macOS
if: runner.os == 'macOS'
run: |
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e -- tests/e2e/golden-core-flows.spec.ts
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e tests/e2e/golden-core-flows.spec.ts
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:workspace-session-golden
if [ -f tests/e2e/golden-fresh-profile-terminal.spec.ts ]; then
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e -- tests/e2e/golden-fresh-profile-terminal.spec.ts tests/e2e/golden-shell-command.spec.ts
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e tests/e2e/golden-fresh-profile-terminal.spec.ts tests/e2e/golden-shell-command.spec.ts
fi
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:terminal-rendering-golden
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:posix-profile-index-golden
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:agent-tui-golden
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:tab-bar-agent-launch-golden
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run --if-present test:e2e:source-control-golden
- name: Run golden E2E tests on Windows
@@ -96,11 +98,17 @@ jobs:
$env:SKIP_BUILD = '1'
$env:ORCA_E2E_FORWARD_APP_LOGS = '1'
pnpm run --if-present test:e2e:workspace-session-golden
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
pnpm run --if-present test:e2e:windows-fresh-startup-golden
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
pnpm run --if-present test:e2e:tab-bar-agent-launch-golden
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
if (Test-Path tests/e2e/golden-fresh-profile-terminal.spec.ts) {
pnpm run test:e2e -- tests/e2e/golden-fresh-profile-terminal.spec.ts tests/e2e/golden-shell-command.spec.ts
pnpm run test:e2e tests/e2e/golden-fresh-profile-terminal.spec.ts tests/e2e/golden-shell-command.spec.ts
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
}
pnpm run --if-present test:e2e:source-control-golden
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
- name: Upload Playwright traces
if: failure()
+4 -5
View File
@@ -132,11 +132,10 @@ jobs:
print(src)
PY
# Why: reuse the existing buf0-bot GitHub App (also used by
# track-community-prs.yaml) instead of minting a new PAT. The app is
# installed org-wide, so it has access to stablyai/homebrew-orca
# automatically. GITHUB_TOKEN cannot push cross-repo; a short-lived
# installation token can.
# Why: reuse the existing buf0-bot GitHub App instead of minting a new
# PAT. The app is installed org-wide, so it has access to
# stablyai/homebrew-orca automatically. GITHUB_TOKEN cannot push
# cross-repo; a short-lived installation token can.
- name: Generate buf0-bot token
id: app-token
uses: actions/create-github-app-token@v3
+118 -58
View File
@@ -26,7 +26,7 @@ name: Hourly macOS Dev Build
# HOURLY_RELEASE_APP_ID the App's numeric id
# HOURLY_RELEASE_APP_PRIVATE_KEY the App's .pem private key
#
# Installation tokens live one hour, which is why this mints twice. Install and
# Installation tokens live one hour, so the build job mints twice. Install and
# build need no token at all, and notarization can hold the publish step for tens
# of minutes; minting again once the build is done starts the clock at the first
# call that actually uses it rather than burning a third of it on `pnpm install`.
@@ -60,33 +60,15 @@ env:
HOURLY_RETAIN_COUNT: 72
jobs:
build-hourly-mac:
# Avoid occupying the limited Mac pool when main has not moved.
preflight:
if: github.repository == 'stablyai/orca'
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
tag: ${{ steps.release.outputs.tag }}
version: ${{ steps.hourly.outputs.version }}
should_build: ${{ steps.freshness.outputs.should_build }}
head_sha: ${{ steps.freshness.outputs.head_sha }}
published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }}
runs-on: blacksmith-6vcpu-macos-15
# Why 150: it must exceed the worst case the retry budgets below can produce
# (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or
# the job is killed mid-retry and no cleanup step runs at all. A typical run
# is far shorter — this is the notary queue's tail, not its median.
timeout-minutes: 150
env:
NODE_OPTIONS: --max-old-space-size=4096
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
fetch-depth: 0
# Why: this job only reads stablyai/orca and never pushes; every write
# goes to the hourly repo through a minted App token passed by env.
# Not persisting the checkout credential shrinks the blast radius if a
# build step is compromised (zizmor: artipacked).
persist-credentials: false
- name: Mint hourly repo token
id: app_token
uses: actions/create-github-app-token@v2
@@ -95,18 +77,19 @@ jobs:
private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }}
owner: stablyai
repositories: orca-hourly
permission-contents: read
# Why: main is often idle overnight. Rebuilding an unchanged commit burns a
# runner hour and adds a redundant tag to the retention window.
- name: Check whether main moved since the last hourly
id: freshness
shell: bash
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
MAIN_REPO_TOKEN: ${{ github.token }}
FORCED: ${{ github.event_name == 'workflow_dispatch' && inputs.force }}
run: |
set -euo pipefail
head_sha="$(git rev-parse HEAD)"
head_sha="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api "repos/$GITHUB_REPOSITORY/commits/main" --jq .sha)"
[[ "$head_sha" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::Could not resolve main"; exit 1; }
echo "head_sha=$head_sha" >>"$GITHUB_OUTPUT"
if [[ "$FORCED" == "true" ]]; then
echo "should_build=true" >>"$GITHUB_OUTPUT"
@@ -133,21 +116,84 @@ jobs:
echo "main moved to $head_sha (last hourly built $last_sha); building."
fi
- name: Setup pnpm
if: steps.freshness.outputs.should_build == 'true'
uses: pnpm/action-setup@v6
# As release-cut does: every SSH host runs orcad, deployed from the template this build ships.
orcad-template:
needs: preflight
if: needs.preflight.outputs.should_build == 'true'
permissions:
contents: read
uses: ./.github/workflows/node-server-tests.yml
with:
ref: ${{ needs.preflight.outputs.head_sha }}
build_template: true
build-hourly-mac:
needs:
- preflight
- orcad-template
if: needs.preflight.outputs.should_build == 'true'
outputs:
tag: ${{ steps.release.outputs.tag }}
version: ${{ steps.hourly.outputs.version }}
head_sha: ${{ needs.preflight.outputs.head_sha }}
published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }}
runs-on: blacksmith-6vcpu-macos-15
# Why 150: it must exceed the worst case the retry budgets below can produce
# (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or
# the job is killed mid-retry and no cleanup step runs at all. A typical run
# is far shorter — this is the notary queue's tail, not its median.
timeout-minutes: 150
env:
NODE_OPTIONS: --max-old-space-size=4096
steps:
- name: Checkout
uses: actions/checkout@v6
with:
run_install: false
ref: ${{ needs.preflight.outputs.head_sha }}
# Version helpers only read HEAD; published versions come from git tags.
fetch-depth: 1
# Why: this job only reads stablyai/orca and never pushes; every write
# goes to the hourly repo through a minted App token passed by env.
# Not persisting the checkout credential shrinks the blast radius if a
# build step is compromised (zizmor: artipacked).
persist-credentials: false
- name: Mint hourly repo token
id: app_token
uses: actions/create-github-app-token@v2
with:
app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }}
private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }}
owner: stablyai
repositories: orca-hourly
- name: Setup pnpm
uses: pnpm/setup@v2
with:
install: false
- name: Setup Node.js
if: steps.freshness.outputs.should_build == 'true'
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why: packaging bundles the pinned Node into the macOS terminal host. Keyed on the pin alone
# with no restore-keys, so a pin bump starts empty instead of carrying old archives forward.
- name: Locate the pinned Node runtime cache
shell: bash
run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV"
- name: Cache the pinned Node runtime
uses: actions/cache@v5
with:
path: ~/Library/Caches/orca-node-runtime
key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }}
- name: Cache electron-builder downloads
if: steps.freshness.outputs.should_build == 'true'
uses: actions/cache@v5
with:
path: |
@@ -157,19 +203,23 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
if: steps.freshness.outputs.should_build == 'true'
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: signing is what makes an hourly installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
- name: Verify macOS signing environment
if: steps.freshness.outputs.should_build == 'true'
run: node config/scripts/verify-macos-release-env.mjs
env:
CSC_LINK: ${{ secrets.MAC_CERTS }}
@@ -180,7 +230,6 @@ jobs:
- name: Compute hourly version
id: hourly
if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
@@ -196,22 +245,30 @@ jobs:
# number free", where a stranded draft still holds one.
names="$(gh release list --repo "$HOURLY_REPO" --limit 200 --json name \
--jq '.[].name // empty')"
# Why the main repo's tags decide the base version rather than
# package.json: main's version only moves on `release:` commits, and
# stable patches are cut from release branches that never merge back, so
# package.json can sit several patches behind what users are running. A
# separate token because GH_TOKEN above is the App's, scoped to the
# hourly repo. Empty on failure — the script then falls back to
# package.json, which is stale but never wrong enough to fail a build.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh release list \
--repo "$GITHUB_REPOSITORY" --limit 100 --exclude-drafts \
--json tagName --jq '.[].tagName' || true)"
echo "Highest published tag seen: $(head -1 <<<"$published")"
# Why git tags, not GitHub releases: unpublishing a buggy cut deletes the
# GitHub release and leaves the tag. On 2026-09-14 we deleted v1.4.202's
# release for a bug; hourlies had already climbed to 1.4.203, then
# `gh release list` fell back to v1.4.201 and the next hourlies shipped
# as 1.4.202-hourly — which electron-updater will not install over
# 1.4.203-hourly or over the still-tagged 1.4.202. A separate token
# because GH_TOKEN above is the App's, scoped to the hourly repo. Empty
# on failure — the script then falls back to package.json, which is
# stale but never wrong enough to fail a build.
main_tags="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \
"repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \
--jq '.[].ref | sub("^refs/tags/"; "")' || true)"
# Already-shipped channel tags are a second floor: even if main's tag
# list is empty this run, a 1.4.203-hourly already out must not be
# followed by a 1.4.202-hourly.
channel_tags="$(gh release list --repo "$HOURLY_REPO" --limit 200 --json tagName \
--jq '.[].tagName' || true)"
published="$main_tags"$'\n'"$channel_tags"
echo "Published version sources: $(grep -c . <<<"$main_tags" || true) main tags, $(grep -c . <<<"$channel_tags" || true) channel tags"
ORCA_PUBLISHED_VERSIONS="$published" ORCA_HOURLY_RELEASE_NAMES="$names" \
node config/scripts/hourly-build-version.mjs \
>"$RUNNER_TEMP/hourly-identity.txt"
grep -E '^(version|build_number)=' "$RUNNER_TEMP/hourly-identity.txt"
# Why check rather than trust: the checkout above pins `ref: main`, but a
# Why check rather than trust: the checkout above pins the resolved main commit, but a
# workflow_dispatch runs this file from whatever branch was dispatched. A
# branch that edits this step while main still has the old script yields
# an empty name and an untitled release — silent, and only visible once
@@ -223,7 +280,6 @@ jobs:
cat "$RUNNER_TEMP/hourly-identity.txt" >>"$GITHUB_OUTPUT"
- name: Build app
if: steps.freshness.outputs.should_build == 'true'
run: pnpm build:release
env:
NODE_OPTIONS: --max-old-space-size=4096
@@ -232,6 +288,13 @@ jobs:
# silent, which is correct for unvetted dev artifacts.
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
# After the app build so nothing that cleans out/ can drop it; electron-builder ships it.
- name: Download the orcad deployment template
uses: actions/download-artifact@v8
with:
name: orcad-template
path: out/orcad-template
# Why a second mint: everything from here on writes to the hourly repo, and
# the notary round trip inside the publish step can be tens of minutes. The
# token minted at the top has already spent install + build of its one hour
@@ -239,7 +302,6 @@ jobs:
# part the full budget.
- name: Re-mint hourly repo token for publish
id: app_token_publish
if: steps.freshness.outputs.should_build == 'true'
uses: actions/create-github-app-token@v2
with:
app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }}
@@ -249,13 +311,12 @@ jobs:
- name: Create hourly release
id: release
if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
TAG: v${{ steps.hourly.outputs.version }}
NAME: ${{ steps.hourly.outputs.name }}
SHA: ${{ steps.freshness.outputs.head_sha }}
SHA: ${{ needs.preflight.outputs.head_sha }}
run: |
set -euo pipefail
# Kept at 12 even though the title shows 7: the freshness check above
@@ -290,9 +351,7 @@ jobs:
including back to Stable, works in-app from there."
echo "tag=$TAG" >>"$GITHUB_OUTPUT"
- name: Publish hourly macOS artifacts
if: steps.freshness.outputs.should_build == 'true'
uses: nick-fields/retry@v4
with:
# Why 45 like the release pipeline: an attempt is pack + notarize +
@@ -302,13 +361,15 @@ jobs:
timeout_minutes: 45
max_attempts: 2
retry_wait_seconds: 30
command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_HOURLY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always
command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_HOURLY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always
env:
# Why: electron-builder's github publisher targets the repo named in the
# config; the token must therefore carry write access to orca-hourly.
GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
ORCA_HOURLY_BUILD_VERSION: ${{ steps.hourly.outputs.version }}
ORCA_BUILD_COMMIT: ${{ steps.hourly.outputs.commit }}
# beforePack and afterPack fail the package when the template is absent.
ORCA_REQUIRE_ORCAD_TEMPLATE: '1'
CSC_LINK: ${{ secrets.MAC_CERTS }}
CSC_KEY_PASSWORD: ${{ secrets.MAC_CERTS_PASSWORD }}
# Why all three: electron-builder's notarize step authenticates to the
@@ -323,7 +384,6 @@ jobs:
# release missing that manifest is a tag the picker offers and the download
# 404s on, so fail loudly instead of leaving a broken entry.
- name: Verify update manifest published
if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
@@ -353,7 +413,6 @@ jobs:
# means the picker can never offer a release whose assets are incomplete.
- name: Publish the verified release
id: publish_live
if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
@@ -466,3 +525,4 @@ jobs:
tag: ${{ needs.build-hourly-mac.outputs.tag }}
ref: ${{ needs.build-hourly-mac.outputs.head_sha }}
version: ${{ needs.build-hourly-mac.outputs.version }}
orcad_template: true
+1 -1
View File
@@ -12,7 +12,7 @@ permissions:
jobs:
apply-os-label:
runs-on: ubuntu-latest
runs-on: ubuntu-slim
timeout-minutes: 5
steps:
- name: Apply OS label from issue form
@@ -24,9 +24,9 @@ jobs:
node-version-file: package.json
- name: Setup pnpm
uses: pnpm/action-setup@v6
uses: pnpm/setup@v2
with:
run_install: false
install: false
# Why: mirrors pr.yml so native module rebuilds do not use pnpm's
# non-executable bundled gyp_main.py on Linux runners.
@@ -43,12 +43,9 @@ jobs:
fi
- name: Install dependencies
# Why: pnpm 10.24's frozen headless fast path can fail on fresh Ubuntu
# runners while creating the root node_modules. Use the normal resolver
# path, then verify package metadata stayed unchanged.
run: |
pnpm install --no-frozen-lockfile --prefer-frozen-lockfile=false
git diff --exit-code package.json pnpm-lock.yaml
pnpm install --frozen-lockfile
git diff --exit-code package.json pnpm-lock.yaml pnpm-workspace.yaml
- name: Start Weston
run: |
+50
View File
@@ -0,0 +1,50 @@
name: macOS updater regression tests
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- '.github/workflows/macos-updater-tests.yml'
- 'src/main/macos-update-running-instances*'
- 'src/main/updater*'
- 'src/main/updater/**'
- 'src/main/startup/main-process-quit*'
- 'src/main/window/main-window-state-lifecycle*'
- 'src/main/window/dashboard-popout-window*'
- 'src/packages/process-host/**'
- 'src/shared/update-status-types.ts'
- 'pnpm-lock.yaml'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: macos-updater-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
updater:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: macos-15
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
- name: Exercise native application registry and update shutdown
env:
ORCA_BACKGROUND_LAUNCH: '1'
run: >-
pnpm exec vitest run --config config/vitest.config.ts
src/main/macos-update-running-instances.test.ts
src/main/macos-update-running-instances.integration.test.ts
src/main/updater.mac-install.test.ts
src/main/updater.headless-serve-install.test.ts
src/main/updater-mac-quit-guard.test.ts
src/main/startup/main-process-quit-update-veto.test.ts
src/main/window/main-window-state-lifecycle.test.ts
src/main/window/dashboard-popout-window.test.ts
+207 -6
View File
@@ -30,6 +30,10 @@ jobs:
permissions:
contents: write
outputs:
publish_release: ${{ steps.release.outputs.publish_release }}
version: ${{ steps.release.outputs.version }}
defaults:
run:
working-directory: mobile
@@ -44,9 +48,9 @@ jobs:
node-version: 24
- name: Setup pnpm
uses: pnpm/action-setup@v6
uses: pnpm/setup@v2
with:
run_install: false
install: false
- name: Install dependencies
run: pnpm install --frozen-lockfile
@@ -70,11 +74,44 @@ jobs:
- name: Build Android release APK
run: cd android && ./gradlew assembleRelease
# Why: an install that fails with a missing certificate or a package-parse error is
# usually a download that died near the end, and the signature block sits in the last
# ~100 KB. Publishing the size and digest is what lets a reporter tell a truncated
# download apart from a bad build without anyone re-deriving them from the asset.
- name: Checksum the APK
id: apk
run: |
set -euo pipefail
shopt -s nullglob
apks=(android/app/build/outputs/apk/release/*.apk)
if [ "${#apks[@]}" -ne 1 ]; then
echo "Expected exactly one release APK, found ${#apks[@]}" >&2
exit 1
fi
apk="${apks[0]}"
bytes="$(wc -c < "$apk" | tr -d ' ')"
sha256="$(sha256sum "$apk" | cut -d ' ' -f 1)"
# A sibling file in `sha256sum -c` format, so verifying a download is one command
# and no retyped digest. The asset globs below match *.apk and skip it. The ` *`
# marker is binary mode: Git Bash's sha256sum reads text mode as a licence to
# translate line endings while hashing, which would fail on a valid APK.
printf '%s *%s\n' "$sha256" "$(basename "$apk")" > "$apk.sha256"
{
echo "checksum=$apk.sha256"
echo "bytes=$bytes"
echo "sha256=$sha256"
} >> "$GITHUB_OUTPUT"
echo "APK is $bytes bytes, sha256 $sha256"
- name: Upload APK artifact
uses: actions/upload-artifact@v7
with:
name: orca-mobile-apk
path: mobile/android/app/build/outputs/apk/release/*.apk
path: |
mobile/android/app/build/outputs/apk/release/*.apk
mobile/android/app/build/outputs/apk/release/*.apk.sha256
- name: Ensure GitHub release tag
if: steps.release.outputs.publish_release == 'true' && !startsWith(github.ref, 'refs/tags/mobile-android-v')
@@ -97,18 +134,182 @@ jobs:
run: |
set -euo pipefail
tag="${{ steps.release.outputs.tag }}"
notes_file="$RUNNER_TEMP/android-release-notes.md"
# printf rather than a heredoc: YAML block indentation would leak into the Markdown.
# Both platforms' commands are named because a reader on macOS has no sha256sum.
verification_section() {
printf '\n### Verify your download\n\n- Size: `%s` bytes\n- SHA-256: `%s`\n\nA "no certificate" or "problem parsing the package" install failure is usually a truncated download — check the size first, then `sha256sum -c app-release.apk.sha256` (`shasum -a 256 -c` on macOS).\n' \
'${{ steps.apk.outputs.bytes }}' '${{ steps.apk.outputs.sha256 }}'
}
# The section is always last, so dropping from its heading to EOF leaves the
# generated notes intact. \r* because a body round-tripped through the API has CRLFs.
drop_verification_section() {
sed '/^### Verify your download\r*$/,$d'
}
# Why: reuse the desktop release path's character-safe truncation so a multi-byte
# character cannot be split at the cap. The section's own length is reserved out of
# that cap, because appending after truncating would push a near-limit body past
# GitHub's API limit and fail the call — on the upload path, after --clobber has
# already replaced the assets. `wc -c` counts bytes, so the section's multi-byte
# characters over-reserve, which errs toward a shorter body.
write_notes_with_verification() {
NOTES_FILE="$notes_file" \
NOTES_RESERVE="$(verification_section | wc -c | tr -d ' ')" \
NOTES_MODULE="$GITHUB_WORKSPACE/config/scripts/create-draft-release.mjs" \
node --input-type=module -e '
const { readFileSync, writeFileSync } = await import("node:fs")
const { pathToFileURL } = await import("node:url")
const { MAX_RELEASE_BODY_LENGTH, truncateReleaseBody } = await import(
pathToFileURL(process.env.NOTES_MODULE).href
)
const file = process.env.NOTES_FILE
const max = MAX_RELEASE_BODY_LENGTH - Number(process.env.NOTES_RESERVE)
writeFileSync(file, truncateReleaseBody(readFileSync(file, "utf8"), max))
'
verification_section >> "$notes_file"
}
if gh release view "$tag" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
gh release upload "$tag" \
--repo "$GITHUB_REPOSITORY" \
--clobber \
android/app/build/outputs/apk/release/*.apk
android/app/build/outputs/apk/release/*.apk \
"${{ steps.apk.outputs.checksum }}"
# Why the body is rewritten too: --clobber replaced the APK, so a digest left
# over from the previous build now describes a file nobody can download, and a
# reader comparing against it would reject a good APK.
gh release view "$tag" --repo "$GITHUB_REPOSITORY" --json body --jq .body \
| drop_verification_section > "$notes_file"
write_notes_with_verification
gh release edit "$tag" --repo "$GITHUB_REPOSITORY" --notes-file "$notes_file"
else
# Why: release tags live on side branches, so GitHub's automatic
# previous-tag detection reaches back several releases; that body
# already exceeds the 125000-character API limit and grows each
# release. Pin the comparison base and cap the size.
previous_tag="$(
gh release list --repo "$GITHUB_REPOSITORY" --limit 200 --json tagName --jq '.[].tagName' \
| grep '^mobile-android-v' | grep -Fxv "$tag" | sort -V | tail -1 || true
)"
if [ -n "$previous_tag" ]; then
# Why: gh writes the JSON error body to stdout on an HTTP error, so a
# non-empty file is not proof of success — gate on exit status.
if ! gh api "repos/$GITHUB_REPOSITORY/releases/generate-notes" -X POST \
-f tag_name="$tag" \
-f target_commitish="$GITHUB_SHA" \
-f previous_tag_name="$previous_tag" \
--jq .body > "$notes_file"; then
: > "$notes_file"
fi
fi
if [ ! -s "$notes_file" ]; then
printf 'Orca Mobile Android %s\n' "$tag" > "$notes_file"
fi
write_notes_with_verification
gh release create "$tag" \
--repo "$GITHUB_REPOSITORY" \
--title "Orca Mobile Android $tag" \
--prerelease \
--latest=false \
--generate-notes \
android/app/build/outputs/apk/release/*.apk
--notes-file "$notes_file" \
android/app/build/outputs/apk/release/*.apk \
"${{ steps.apk.outputs.checksum }}"
fi
# Why a separate job: the release is already published here, so a failed bump only leaves
# the site's APK download on the previous release and must not fail the release run.
bump-apk-download:
needs: android-build
if: needs.android-build.outputs.publish_release == 'true'
runs-on: ubuntu-latest
continue-on-error: true
permissions:
contents: read
env:
VERSION: ${{ needs.android-build.outputs.version }}
SITE_REPO: stablyai/orca-marketing-website
BRANCH: bump/android-apk-${{ needs.android-build.outputs.version }}
steps:
- name: Checkout orca
uses: actions/checkout@v6
with:
sparse-checkout: config/scripts
# Why the buf0-bot app, as in homebrew-bump.yml: GITHUB_TOKEN cannot push to another
# repo, and a PR it opened would not run the site's checks.
- name: Generate buf0-bot token
id: app-token
uses: actions/create-github-app-token@v3
with:
app-id: 2590194
private-key: ${{ secrets.BUFO_BOT_PRIVATE_KEY }}
owner: stablyai
repositories: orca-marketing-website
permission-contents: write
permission-pull-requests: write
- name: Checkout marketing site
uses: actions/checkout@v6
with:
repository: stablyai/orca-marketing-website
path: site
token: ${{ steps.app-token.outputs.token }}
- name: Open and merge the version bump PR
working-directory: site
env:
GH_TOKEN: ${{ steps.app-token.outputs.token }}
run: |
set -euo pipefail
node ../config/scripts/bump-android-apk-download-version.mjs src/lib/downloadTargets.ts "$VERSION"
if git diff --quiet; then
echo "The site's main branch already has Android APK $VERSION or newer." >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
title="Point the Android APK download at $VERSION"
git config user.name "buf0-bot[bot]"
git config user.email "252831055+buf0-bot[bot]@users.noreply.github.com"
git checkout -b "$BRANCH"
git commit -am "$title"
# Why --force: a rerun rebuilds the branch on the site's current main.
git push --force origin "$BRANCH"
# Why: a rerun of the same release reuses its open PR instead of opening another.
pr="$(gh pr list --repo "$SITE_REPO" --head "$BRANCH" --state open --json url --jq '.[].url')"
if [ -z "$pr" ]; then
pr="$(gh pr create --repo "$SITE_REPO" --base main --head "$BRANCH" --title "$title" \
--body "Opened by the [Android release run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID) for mobile-android-v$VERSION. Merging points /download/android and every APK link on the site at the new APK.")"
fi
echo "Android APK download bump: $pr" >> "$GITHUB_STEP_SUMMARY"
# Why wait for checks: they push a deploy commit onto the branch (merging first fails with
# "Head branch was modified"), and a failed site build must not merge. Main's push deploys.
sleep 20
gh pr checks "$pr" --repo "$SITE_REPO" --watch --interval 15 || true
failed="$(gh pr checks "$pr" --repo "$SITE_REPO" --json bucket --jq 'map(select(.bucket == "fail")) | length' 2>/dev/null || true)"
# The deploy commit is [skip ci], so the final head may report no checks at all.
if [ "${failed:-0}" != "0" ]; then
echo "::warning::The site's checks failed on $pr; not merging. Fix and merge it by hand."
exit 1
fi
for _ in 1 2 3; do
if gh pr merge "$pr" --repo "$SITE_REPO" --squash --delete-branch; then
echo "Merged; the site deploys from main." >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
sleep 30
done
echo "::warning::Could not merge $pr; merge it by hand to update /download/android."
exit 1
- name: Report a failed bump
if: failure()
run: |
echo "::warning::The release is published, but the site's Android APK download was not bumped to $VERSION."
printf '### Android APK download not bumped\n\nThe release is published, but /download/android still serves the previous APK. Rerun this job, or set `ANDROID_RELEASE_VERSION` in `src/lib/downloadTargets.ts` of %s to `%s` by hand.\n' "$SITE_REPO" "$VERSION" >> "$GITHUB_STEP_SUMMARY"
+6 -3
View File
@@ -64,9 +64,9 @@ jobs:
node-version: 24
- name: Setup pnpm
uses: pnpm/action-setup@v6
uses: pnpm/setup@v2
with:
run_install: false
install: false
- name: Install dependencies
run: pnpm install --frozen-lockfile
@@ -94,10 +94,13 @@ jobs:
run: node -e 'const fs = require("node:fs"); const { expo } = require("./app.json"); fs.appendFileSync(process.env.GITHUB_OUTPUT, `version=${expo.version}\nbuild_number=${expo.ios.buildNumber}\n`)'
- name: Expo prebuild
env:
ORCA_IOS_APS_ENVIRONMENT: production
run: npx expo prebuild --platform ios --no-install
- name: Install CocoaPods
run: npx pod-install ios
working-directory: mobile/ios
run: bundle exec pod install
# Why: `-allowProvisioningUpdates` + the App Store Connect API key can
# create/refresh provisioning profiles, but it cannot recreate the
+234 -27
View File
@@ -7,25 +7,108 @@ on:
- synchronize
- reopened
- ready_for_review
- converted_to_draft
paths:
- 'mobile/**'
# Why: the mobile terminal link parsers are conformance-tested against
# these shared fixtures; desktop-side fixture edits must re-run this suite.
- 'src/shared/terminal-file-link-conformance.ts'
# Why all of src/shared: mobile imports hundreds of its modules, directly and through the RPC
# recordings and the host params check, so any shared edit can move a golden or break mobile's
# typecheck. Why the root lockfile: those modules resolve their packages from the root install.
- 'src/shared/**'
# Mobile's Node RPC recording tools import this package.
- 'src/packages/process-host/**'
- 'pnpm-lock.yaml'
# Mobile launch contracts exercise the real host dispatcher and durable receipt store.
- 'src/main/agent-launch/**'
- 'src/main/runtime/rpc/**'
- 'src/main/runtime/runtime-rpc/**'
- 'src/main/runtime/runtime-rpc.ts'
- 'src/main/runtime/device-registry.ts'
- 'src/main/runtime/orca-runtime.ts'
- 'src/main/runtime/agent-session-*.ts'
- 'src/main/native-chat/agent-session-wire/**'
- 'src/main/runtime/structured-attention-original-delivery.test-fixture.ts'
# Why: this job holds the only checks that load the Fastfile, so edits to
# it or to the release workflow it guards must re-run them.
- '.github/workflows/mobile.yml'
- 'config/mobile-ci.vitest.config.mjs'
- 'config/scripts/run-mobile-ci-tests.mjs'
- 'config/oxlint-code-quality-native-plugins.json'
- 'config/.bun-version'
- '.github/actions/install-node-dependencies/**'
- '.github/actions/restore-pnpm-verification/**'
- '.github/workflows/mobile-ios-release.yml'
- 'config/scripts/mobile-release-check-scope*'
- 'config/scripts/mobile-test-change-scope*'
- 'config/scripts/pr-code-change-scope.mjs'
- 'config/scripts/pr-package-test-paths.mjs'
- 'config/scripts/pr-layout-oracle-route.mjs'
# Desktop suites do not feed mobile; keep fixtures and test/toolchain config inputs.
- '!src/shared/**/*.test.[jt]s'
- '!src/shared/**/*.test.[jt]sx'
- '!src/shared/**/*.test.[cm][jt]s'
- '!src/packages/process-host/**/*.test.[jt]s'
- '!src/packages/process-host/**/*.test.[jt]sx'
- '!src/packages/process-host/**/*.test.[cm][jt]s'
- '!src/main/**/*.test.[jt]s'
- '!src/main/**/*.test.[jt]sx'
- '!src/main/**/*.test.[cm][jt]s'
# Why main too: two pull requests can each pass against their own base and disagree once both
# land, and `verify` never runs on main. The same source paths as above, less the CI inputs only
# `verify` reads.
push:
branches:
- main
paths:
- 'mobile/**'
- 'src/shared/**'
- 'src/packages/process-host/**'
- 'pnpm-lock.yaml'
- 'src/main/agent-launch/**'
- 'src/main/runtime/rpc/**'
- 'src/main/runtime/runtime-rpc/**'
- 'src/main/runtime/runtime-rpc.ts'
- 'src/main/runtime/device-registry.ts'
- 'src/main/runtime/orca-runtime.ts'
- 'src/main/runtime/agent-session-*.ts'
- 'src/main/native-chat/agent-session-wire/**'
- 'src/main/runtime/structured-attention-original-delivery.test-fixture.ts'
- '.github/workflows/mobile.yml'
- 'config/mobile-ci.vitest.config.mjs'
- 'config/scripts/run-mobile-ci-tests.mjs'
- 'config/.bun-version'
# Desktop suites do not feed mobile; keep fixtures and test/toolchain config inputs.
- '!src/shared/**/*.test.[jt]s'
- '!src/shared/**/*.test.[jt]sx'
- '!src/shared/**/*.test.[cm][jt]s'
- '!src/packages/process-host/**/*.test.[jt]s'
- '!src/packages/process-host/**/*.test.[jt]sx'
- '!src/packages/process-host/**/*.test.[cm][jt]s'
- '!src/main/**/*.test.[jt]s'
- '!src/main/**/*.test.[jt]sx'
- '!src/main/**/*.test.[cm][jt]s'
concurrency:
# Per commit on main, not per branch. GitHub cancels any PENDING run in a group when a new one
# queues, whatever `cancel-in-progress` says, so one shared main group drops the middle merge of
# three -- and a merge that breaks main there is exactly what `main-tests` checks for.
group: mobile-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
jobs:
verify:
runs-on: ubuntu-latest
name: Mobile verify
if: github.event_name == 'pull_request' && github.event.pull_request.draft != true
# Why ARM: 209s of this job is Vitest and nothing here needs x86: no Android SDK, emulator, gradle,
# Hermes or Watchman, no docker, and no artifacts. The Gemfile.lock lists the generic `ruby`
# platform, so frozen bundler installs without an aarch64-linux entry.
runs-on: ubuntu-24.04-arm
env:
# Why: an unfrozen bundler silently re-resolves when Gemfile.lock drifts
# from the Gemfile, which is how the release jobs could land on different
# fastlane versions in the first place. Fail here instead.
BUNDLE_FROZEN: 'true'
ORCA_MOBILE_TYPECHECK_COMPILER: 'native'
defaults:
run:
@@ -34,52 +117,115 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
fetch-depth: 2
- uses: ./.github/actions/install-node-dependencies
with:
cache-pnpm-store: 'false'
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Detect Ruby release inputs
id: ruby-scope
shell: bash
working-directory: .
run: |
# Keep deletions when release files move into an application directory.
if ! git diff --name-only --no-renames -z HEAD^1 HEAD > "$RUNNER_TEMP/mobile-release-changes"; then
echo 'should_run=true' >> "$GITHUB_OUTPUT"
elif ! node config/scripts/mobile-release-check-scope.mjs "$RUNNER_TEMP/mobile-release-changes"; then
echo 'should_run=true' >> "$GITHUB_OUTPUT"
fi
# bundler-cache installs mobile/Gemfile.lock, so this job is also what
# proves the pinned fastlane the release workflow depends on still
# resolves — before a release run finds out.
- name: Setup Ruby and fastlane
id: ruby-tools
background: true
if: steps.ruby-scope.outputs.should_run != 'false'
uses: ruby/setup-ruby@v1
with:
ruby-version: '3.3'
bundler-cache: true
working-directory: mobile
- name: Setup pnpm
uses: pnpm/action-setup@v6
with:
run_install: false
# Why: the mobile typecheck imports shared types from ../src/shared, and
# some of those files import runtime deps (tweetnacl, ws) resolved from
# the repo-root node_modules. Without a root install, tsc fails with
# "Cannot find module 'tweetnacl'/'ws'". Mobile is a separate pnpm project
# (not in the root workspace), so this is a distinct install.
# --ignore-scripts skips the root postinstall (Electron native-module
# rebuild) which is irrelevant to a type-only check and would only add
# time and failure surface on this ubuntu mobile runner.
- name: Install root dependencies
working-directory: .
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Install dependencies
run: pnpm install --frozen-lockfile
# Why here: pr.yml runs these import rules before mobile's install, when mobile's tsconfig
# (which extends expo's) can't load, so its import cycles were never seen. Zero tolerance.
- name: Lint mobile imports with mobile dependencies installed
working-directory: .
run: node node_modules/oxlint/bin/oxlint --config config/oxlint-code-quality-native-plugins.json mobile --deny-warnings
# Call installed tools so pnpm's dependency refresh cannot race between checks.
- name: Typecheck
run: pnpm typecheck
id: production-types
background: true
env:
GOMAXPROCS: '2'
run: node scripts/typecheck-mobile.mjs
# Why a ratchet and not the raw typecheck: mobile/tsconfig.json excludes test files, so until
# tsconfig.test.json existed nothing checked them, and at introduction 127 of the 632 had
# drifted. This fails when a test file that checks today stops checking, when a test leaves
# the program, and on @ts-nocheck; the baseline may only shrink.
- name: Typecheck tests (ratchet)
id: test-types
background: true
env:
GOMAXPROCS: '2'
run: node scripts/check-tests-typecheck-ratchet.mjs
# This includes the bridged replay of the whole recording corpus, which used to be a second
# step of its own behind RPC_FOUNDATION_BRIDGE=1. A gate nobody can forget to set is the point:
# it fails when a divergence class grows, when a divergence lands in no class at all, or when
# one of the 103 goldens inside the C1 page closure changes the verdict it is pinned to. It is
# ~3 min of test time on its own, and Vitest runs it on a worker beside the rest of the suite,
# so folding it in costs a fraction of that in wall time and one step less to skip.
- name: Detect mobile test inputs
id: test-scope
shell: bash
working-directory: .
run: |
if ! git diff --name-only --no-renames -z HEAD^1 HEAD > "$RUNNER_TEMP/mobile-test-changes"; then
echo 'should_run=true' >> "$GITHUB_OUTPUT"
elif ! node config/scripts/mobile-test-change-scope.mjs "$RUNNER_TEMP/mobile-test-changes"; then
echo 'should_run=true' >> "$GITHUB_OUTPUT"
fi
- name: Setup Bun for tests
if: steps.test-scope.outputs.should_run != 'false'
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version-file: config/.bun-version
- name: Test
run: pnpm test
if: steps.test-scope.outputs.should_run != 'false'
env:
ORCA_BACKGROUND_LAUNCH: '1'
run: node ../config/scripts/run-mobile-ci-tests.mjs
# All checks read the completed install; compilers need not hold up test workers.
- wait: [production-types, test-types, ruby-tools]
# Reports, never gates: the goldens are content-addressed JSON, so their raw diff is hashes.
# This decodes which recorded behaviour the pull request moves into the job summary, and runs
# after a red Test too, when a reviewer most wants it.
- name: Summarize RPC recording changes
if: ${{ !cancelled() }}
continue-on-error: true
run: pnpm run rpc:diff HEAD^1 --summary "$GITHUB_STEP_SUMMARY"
- name: Test iOS release version resolution
if: steps.ruby-scope.outputs.should_run != 'false'
run: ruby fastlane/ios_release_version_test.rb
- name: Test TestFlight lane arguments
if: steps.ruby-scope.outputs.should_run != 'false'
run: ruby fastlane/fastfile_testflight_arguments_test.rb
# Why: nothing else in CI loads the Fastfile, so a syntax error, a broken
@@ -88,13 +234,74 @@ jobs:
# loads and lists, so it needs no App Store Connect credentials and makes
# no network calls to Apple.
- name: Smoke-check the Fastfile
if: steps.ruby-scope.outputs.should_run != 'false'
env:
FASTLANE_SKIP_UPDATE_CHECK: '1'
FASTLANE_OPT_OUT_USAGE: '1'
run: bundle exec fastlane lanes
- name: Smoke-check CocoaPods
if: steps.ruby-scope.outputs.should_run != 'false'
run: bundle exec pod --version
- name: Lint
run: pnpm lint
- name: Check formatting
run: pnpm format:check
main-tests:
name: Mobile tests on main
env:
ORCA_MOBILE_TYPECHECK_COMPILER: 'native'
if: github.event_name == 'push'
# Why ARM: the same pure-Node Vitest run `verify` makes on a pull request.
runs-on: ubuntu-24.04-arm
defaults:
run:
working-directory: mobile
steps:
- name: Checkout
uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Typecheck
id: production-types
background: true
env:
GOMAXPROCS: '2'
run: node scripts/typecheck-mobile.mjs
- name: Typecheck tests (ratchet)
id: test-types
background: true
env:
GOMAXPROCS: '2'
run: node scripts/check-tests-typecheck-ratchet.mjs
# The whole suite, not only the recordings: the mutant and page-bridge suites read the goldens
# too. A red run here names each golden and the `rpc:record` command; the author of the merge
# that turned it red re-records in a follow-up.
- name: Setup Bun for tests
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version-file: config/.bun-version
- name: Test
env:
ORCA_BACKGROUND_LAUNCH: '1'
run: node ../config/scripts/run-mobile-ci-tests.mjs
- wait: [production-types, test-types]
+45
View File
@@ -0,0 +1,45 @@
name: Scheduled x86 unit compatibility
on:
schedule:
# Keep current and future Node coverage on x86 while PR unit shards use ARM.
- cron: '0 10 * * *'
workflow_dispatch:
concurrency:
group: node-next-compat
cancel-in-progress: true
permissions:
contents: read
jobs:
# Prime each Node ABI before its shards restore native modules.
test_native_cache:
name: prepare test native cache node ${{ matrix.node }}
strategy:
fail-fast: false
matrix:
node: ['24', '26']
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
node-version: ${{ matrix.node }}
unit_plan:
uses: ./.github/workflows/unit-plan.yml
test:
needs: [test_native_cache, unit_plan]
uses: ./.github/workflows/unit-tests.yml
with:
node_versions: '["24", "26"]'
shards: ${{ needs.unit_plan.outputs.shards }}
+587
View File
@@ -0,0 +1,587 @@
name: Headless Node server
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- 'src/**'
- 'config/**'
- 'native/**'
- 'tests/**'
- 'resources/**'
- 'package.json'
- 'pnpm-lock.yaml'
- 'pnpm-workspace.yaml'
- 'tsconfig.json'
- '.npmrc'
- '.pnpmfile.cjs'
- '.github/actions/install-node-dependencies/**'
- '.github/actions/restore-pnpm-verification/**'
- '.github/actions/prepare-headless-compiler/**'
- '.github/actions/prepare-native-runtime/**'
- '.github/actions/prepare-orcad-prebuilds/**'
- '.github/workflows/node-server-tests.yml'
workflow_dispatch:
inputs:
build_template:
description: Also merge every lane's slot into the desktop orcad template artifact
type: boolean
default: false
# Release and dev builds qualify every platform for the template they ship.
workflow_call:
inputs:
ref:
description: Git ref every lane checks out, e.g. the release tag
type: string
default: ''
build_template:
description: Upload each lane's release slot and merge them into the orcad-template artifact
type: boolean
default: false
# Requalify fresh builds daily; publish missing immutable Windows cache keys.
schedule:
- cron: '30 11 * * *'
permissions:
contents: read
# Keep template callers independent while replacing superseded PR qualification.
concurrency:
group: node-server-${{ (inputs.build_template || github.event_name == 'push') && format('run-{0}', github.run_id) || github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ !inputs.build_template && github.event_name != 'push' }}
jobs:
changes:
if: >-
github.event_name == 'push' ||
(github.event_name == 'pull_request' && github.event.pull_request.draft != true)
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
should_run: ${{ steps.scope.outputs.should_run || steps.graph.outputs.should_run }}
qualification: ${{ steps.scope.outputs.qualification || steps.graph.outputs.qualification }}
native_compatibility: ${{ steps.scope.outputs.native_compatibility || steps.graph.outputs.native_compatibility }}
runners: ${{ steps.scope.outputs.runners || steps.graph.outputs.runners }}
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 2
persist-credentials: false
- uses: actions/setup-node@v6
with:
node-version-file: package.json
package-manager-cache: false
- name: Detect headless-server build and test inputs
id: scope
shell: bash
env:
PUSH_BASE: ${{ github.event.before }}
EVENT_NAME: ${{ github.event_name }}
ALLOW_TEST_ONLY_NATIVE_SKIP: ${{ github.event_name == 'pull_request' && !inputs.build_template && inputs.ref == '' }}
run: |
if [ "$EVENT_NAME" = push ]; then
# Compare the entire push, including multi-commit pushes and removed files.
if git fetch --no-tags --depth=1 origin "$PUSH_BASE" &&
git diff --name-only --no-renames -z "$PUSH_BASE" HEAD > "$RUNNER_TEMP/node-server-changes" &&
git diff --name-only --no-renames --diff-filter=M -z "$PUSH_BASE" HEAD > "$RUNNER_TEMP/node-server-modified"; then
node config/scripts/node-server-change-scope.mjs "$RUNNER_TEMP/node-server-changes" --defer-graph --full-qualification --modified-paths "$RUNNER_TEMP/node-server-modified"
else
echo 'should_run=true' >> "$GITHUB_OUTPUT"
fi
exit 0
fi
NATIVE_SCOPE_ARGS=()
if [ "$ALLOW_TEST_ONLY_NATIVE_SKIP" = true ]; then
NATIVE_SCOPE_ARGS=(--pull-request --diff-paths "$RUNNER_TEMP/node-server-native-diff")
fi
# Compare the tested merge with its base, retaining both sides of renames.
if git diff --name-only --no-renames -z HEAD^1 HEAD > "$RUNNER_TEMP/node-server-changes" &&
git diff --name-only --no-renames --diff-filter=M -z HEAD^1 HEAD > "$RUNNER_TEMP/node-server-modified" &&
git diff --raw --no-renames --no-abbrev -z HEAD^1 HEAD > "$RUNNER_TEMP/node-server-native-diff"; then
node config/scripts/node-server-change-scope.mjs "$RUNNER_TEMP/node-server-changes" --defer-graph --modified-paths "$RUNNER_TEMP/node-server-modified" "${NATIVE_SCOPE_ARGS[@]}"
else
echo 'should_run=true' >> "$GITHUB_OUTPUT"
fi
- uses: ./.github/actions/prepare-headless-compiler
id: compiler
if: steps.scope.outputs.graph_required == 'true'
continue-on-error: true
- uses: ./.github/actions/install-node-dependencies
if: steps.scope.outputs.graph_required == 'true' && steps.compiler.outputs.available != 'true'
- name: Check the headless import graph
id: graph
if: steps.scope.outputs.graph_required == 'true'
shell: bash
env:
EVENT_NAME: ${{ github.event_name }}
ALLOW_TEST_ONLY_NATIVE_SKIP: ${{ github.event_name == 'pull_request' && !inputs.build_template && inputs.ref == '' }}
run: |
if [ "$EVENT_NAME" = push ]; then
node config/scripts/node-server-change-scope.mjs "$RUNNER_TEMP/node-server-changes" --full-qualification --modified-paths "$RUNNER_TEMP/node-server-modified"
else
NATIVE_SCOPE_ARGS=()
if [ "$ALLOW_TEST_ONLY_NATIVE_SKIP" = true ]; then
NATIVE_SCOPE_ARGS=(--pull-request --diff-paths "$RUNNER_TEMP/node-server-native-diff")
fi
node config/scripts/node-server-change-scope.mjs "$RUNNER_TEMP/node-server-changes" --modified-paths "$RUNNER_TEMP/node-server-modified" "${NATIVE_SCOPE_ARGS[@]}"
fi
persistence:
needs: changes
concurrency:
group: node-server-persistence-${{ matrix.os }}-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }}
# Missing/failed detection runs the full matrix; scheduled and manual runs stay unconditional.
# Drafts re-trigger at readiness; inherited caller events must not depend on PR fields.
if: >-
${{ !cancelled() && needs.changes.outputs.should_run != 'false' &&
(github.event_name != 'pull_request' || github.event.pull_request.draft != true) }}
strategy:
fail-fast: false
matrix:
os: ${{ fromJSON(needs.changes.outputs.runners || '["ubuntu-22.04","ubuntu-24.04-arm","macos-15","macos-15-intel","windows-2022","windows-11-arm"]') }}
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
id: dependencies
with:
native-runtime: ${{ runner.os == 'Windows' && 'node' || 'none' }}
cache-pnpm-store: ${{ runner.os != 'Windows' }}
cache-pnpm-store-lookup-only: 'true'
# Design D7 upgrade and rollback: the last Bun orcad, built from a main commit that shipped
# it, using the current Bun pin beside this checkout's Node slot; hand-over skips once PROTOCOL_VERSION
# moves past the Bun daemon's. Install its pinned dependencies independently of this checkout.
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
if: runner.os == 'Linux'
with:
bun-version-file: config/.bun-version
- name: Build the last Bun orcad for the cross-runtime tests
id: bun-orcad
background: true
shell: bash
env:
BUN_ORCAD_COMMIT: f4092c06d639ee13ad446261dcabc78b27a21fbc
run: |
if [ "$RUNNER_OS" != Linux ]; then exit 0; fi
git fetch --no-tags --depth=1 origin "$BUN_ORCAD_COMMIT"
git worktree add --detach "$RUNNER_TEMP/bun-orcad-source" "$BUN_ORCAD_COMMIT"
pnpm --dir "$RUNNER_TEMP/bun-orcad-source" install --frozen-lockfile --ignore-scripts
node "$RUNNER_TEMP/bun-orcad-source/config/scripts/build-orcad-bun.mjs" --out-dir "$RUNNER_TEMP/bun-orcad"
echo "slot=$RUNNER_TEMP/bun-orcad" >> "$GITHUB_OUTPUT"
echo "executable=$(command -v bun)" >> "$GITHUB_OUTPUT"
# Linux release slots come from the floor and Alpine lanes; this slot serves local tests.
- uses: ./.github/actions/prepare-orcad-prebuilds
id: orcad-prebuild
with:
resolve-windows-cache: >-
${{ !inputs.build_template && inputs.ref == '' &&
(github.event_name == 'pull_request' ||
(github.ref == 'refs/heads/main' && contains(fromJSON('["push","schedule","workflow_dispatch"]'), github.event_name))) }}
restore-windows-cache: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}
# The template lane cross-compiles arm64 too, as it does for the process-table addon.
- uses: ./.github/actions/prepare-windows-cli-launcher
if: runner.os == 'Windows'
with:
arches: ${{ (inputs.build_template && matrix.os == 'windows-2022') && 'x64 arm64' || (runner.arch == 'ARM64' && 'arm64' || 'x64') }}
- uses: ./.github/actions/build-mobile-web-bundle
- run: pnpm build:orcad
env:
ORCA_REUSE_PREPARED_WINDOWS_SECURITY: ${{ github.event_name == 'pull_request' && runner.os == 'Windows' && steps.dependencies.outcome == 'success' }}
- wait: bun-orcad
- run: pnpm test:node-server --artifact ${{ runner.os == 'Linux' && '--cross-runtime' || '' }}
env:
ORCA_BUN_ORCAD_SLOT: ${{ steps.bun-orcad.outputs.slot }}
BUN_EXECUTABLE: ${{ steps.bun-orcad.outputs.executable }}
- name: Build the host-target desktop server template
if: runner.os == 'Linux'
run: node config/scripts/build-orcad-template.mjs --targets linux-${{ runner.arch == 'ARM64' && 'arm64' || 'x64' }}-glibc
- name: Verify a structured chat from the desktop server template
if: runner.os == 'Linux'
run: pnpm test:node-server --artifact src/main/orcad/orcad-packaged-structured-chat.integration.test.ts --maxWorkers=2
# Only a Windows runner compiles it; arm64 cross-compiles here.
# Before the Node 18 check below: the build script imports TypeScript, which Node 18 cannot load.
# orcad's instance lock and stop proof read a PID's creation time through it; the x64 binary this
# runner loads must answer it, and answer nothing for a missing PID rather than a stale time.
- name: Build the Windows process-table addons for the desktop template
if: inputs.build_template && matrix.os == 'windows-2022'
shell: bash
run: |
node config/scripts/build-windows-process-tree-addon.mjs --arch=x64
node config/scripts/build-windows-process-tree-addon.mjs --arch=arm64
node -e "
const { assertWindowsProcessTreeCreationTime } = require('./config/scripts/windows-process-tree-creation-time.cjs')
const addon = require('./.build/windows-process-tree/x64/windows-process-tree.node')
assertWindowsProcessTreeCreationTime({ module: addon })
const own = addon.getProcessCreationTime(process.pid)
const started = Date.now() - process.uptime() * 1000
if (typeof own !== 'number' || Math.abs(own - started) > 5000) throw new Error('creation time ' + own + ' vs ' + started)
if (addon.getProcessCreationTime(0x7ffffff0) !== undefined) throw new Error('a missing PID must read undefined')
console.log('creation time ok:', own)
"
- name: Keep the Windows process-table addons for the desktop template
if: inputs.build_template && matrix.os == 'windows-2022'
uses: actions/upload-artifact@v7
with:
name: orcad-windows-process-tree
path: .build/windows-process-tree/
include-hidden-files: true
if-no-files-found: error
retention-days: 7
overwrite: true
- name: Keep the Windows CLI launchers for the desktop template
if: inputs.build_template && matrix.os == 'windows-2022'
uses: actions/upload-artifact@v7
with:
name: orcad-windows-cli-launcher
path: .build/windows-cli-launcher/
if-no-files-found: error
retention-days: 7
overwrite: true
# orcad's data-root privacy check fails closed without it, so every Windows slot needs one.
# Older requested refs use the previous security implementation and have no addon builder.
- name: Build the Windows security addons for the desktop template
if: inputs.build_template && matrix.os == 'windows-2022' && hashFiles('config/scripts/build-windows-security-addon.mjs') != ''
shell: bash
run: |
node config/scripts/build-windows-security-addon.mjs --arch=x64
node config/scripts/build-windows-security-addon.mjs --arch=arm64
node -e "require('./.build/windows-security/x64/windows-security.node').currentUserSid()"
- name: Keep the Windows security addons for the desktop template
if: inputs.build_template && matrix.os == 'windows-2022' && hashFiles('config/scripts/build-windows-security-addon.mjs') != ''
uses: actions/upload-artifact@v7
with:
name: orcad-windows-security
path: .build/windows-security/*/windows-security.node
if-no-files-found: error
retention-days: 7
overwrite: true
- uses: actions/setup-node@v6
if: runner.arch == 'X64'
with:
node-version: '18'
- name: Verify Node 18 loads and hands off to the pinned Node
if: runner.arch == 'X64'
shell: bash
run: |
node out/orcad/orcad.js --orcad-smoke-load-check
node out/orcad/orcad.js --orcad-profile-state-preflight 00000000-0000-4000-8000-000000000018 | tee "$RUNNER_TEMP/preflight.json"
node -e "const r=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'));if(r.runtime!=='node'||!/^24\./.test(r.runtimeVersion))process.exit(1)" "$RUNNER_TEMP/preflight.json"
- name: Save the freshly qualified Windows server prebuild on main
if: >-
success() && runner.os == 'Windows' && (runner.arch == 'X64' || runner.arch == 'ARM64') &&
!inputs.build_template && inputs.ref == '' && github.ref == 'refs/heads/main' &&
contains(fromJSON('["push","schedule","workflow_dispatch"]'), github.event_name) &&
steps.orcad-prebuild.outputs.cache-identity-outcome == 'success' &&
steps.orcad-prebuild.outputs.cache-key != ''
continue-on-error: true
uses: actions/cache/save@v5
with:
path: ${{ steps.orcad-prebuild.outputs.cache-path }}
key: ${{ steps.orcad-prebuild.outputs.cache-key }}
# Linux release slots come from the floor and Alpine lanes; these runners own the rest.
- name: Keep this runner's qualified slot for the desktop template
if: inputs.build_template && runner.os != 'Linux'
uses: actions/upload-artifact@v7
with:
name: orcad-prebuild-${{ matrix.os }}
path: out/orcad-prebuilds/
if-no-files-found: error
retention-days: 7
# A rerun attempt re-uploads under the same name, which v4 otherwise refuses.
overwrite: true
# The container lanes take the page from here: it is platform-independent, and building it needs
# mobile/node_modules, which those images never install.
mobile_web_bundle:
needs: changes
if: >-
${{ !cancelled() && needs.changes.outputs.should_run != 'false' &&
needs.changes.outputs.qualification != 'false' &&
(github.event_name != 'pull_request' || github.event.pull_request.draft != true) }}
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- uses: ./.github/actions/build-mobile-web-bundle
- uses: actions/upload-artifact@v7
with:
name: orcad-mobile-web-bundle
path: out/mobile-web/
if-no-files-found: error
retention-days: 1
overwrite: true
linux_glibc_floor:
needs: [changes, persistence, mobile_web_bundle]
concurrency:
group: node-server-linux_glibc_floor-${{ matrix.os }}-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }}
# A failed smoke already blocks qualification; missing scope still selects every platform.
if: >-
${{ !cancelled() && needs.persistence.result == 'success' &&
needs.mobile_web_bundle.result == 'success' &&
needs.changes.outputs.should_run != 'false' &&
needs.changes.outputs.qualification != 'false' }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-22.04, ubuntu-24.04-arm]
# Design D6: server glibc slots are built and gated on glibc 2.28, where the pinned Node's
# own floor is; the desktop keeps its Ubuntu 20.04 gate. gcc-toolset-14 supplies C++20 and
# links newer libstdc++ symbols statically, so the slot needs only RHEL 8's GLIBCXX_3.4.25.
include:
- os: ubuntu-22.04
image: quay.io/pypa/manylinux_2_28_x86_64@sha256:407f771c51a2c3e83ebe5a7970b4289ead3a6db21d9b9c089168775cad11d328
- os: ubuntu-24.04-arm
image: quay.io/pypa/manylinux_2_28_aarch64@sha256:c22ffd129ac99a8a42d1f2c2f4e88a9089288dd9ee987a7092da1c7dc48f27a9
runs-on: ${{ matrix.os }}
container: ${{ matrix.image }}
timeout-minutes: 25
env:
ORCA_BACKGROUND_LAUNCH: '1'
# The image puts /opt/clang first on PATH; the slot is built with its gcc-toolset.
CC: gcc
CXX: g++
PYTHON: /opt/python/cp312-cp312/bin/python3
steps:
- name: Install glibc 2.28 prerequisites
run: |
missing_tool=false
for tool in git ps unzip which xz; do
if ! command -v "$tool" >/dev/null 2>&1; then
missing_tool=true
fi
done
if [ "$missing_tool" = true ]; then
# The image's source-built Git needs no RPM; missing tools come from AlmaLinux.
dnf --disablerepo='epel*' install -y git procps-ng unzip which xz
fi
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
persist-credentials: false
- name: Trust the checked-out workspace
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
- uses: ./.github/actions/install-node-dependencies
- name: Build and smoke this runner's node-pty prebuild slot under the pinned Node
run: |
pnpm build:orcad-prebuilds
pnpm build:orcad-prebuilds --require-slots "$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)"
pnpm build:orcad-prebuilds --smoke
- uses: actions/download-artifact@v8
with:
name: orcad-mobile-web-bundle
path: out/mobile-web
- run: pnpm build:orcad
- run: pnpm test:node-server --artifact
- name: Keep this runner's glibc 2.28 slot for the desktop template
if: inputs.build_template
uses: actions/upload-artifact@v7
with:
name: orcad-prebuild-glibc-${{ matrix.os }}
path: out/orcad-prebuilds/
if-no-files-found: error
retention-days: 7
overwrite: true
linux_glibc217_compat:
needs: [changes, persistence]
concurrency:
group: node-server-linux_glibc217_compat-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }}
# A failed smoke already blocks qualification; missing scope still selects every platform.
if: >-
${{ !cancelled() && needs.persistence.result == 'success' &&
needs.changes.outputs.should_run != 'false' &&
needs.changes.outputs.qualification != 'false' &&
(github.event_name != 'pull_request' || inputs.build_template || inputs.ref != '' ||
needs.changes.outputs.native_compatibility != 'false') }}
runs-on: ubuntu-22.04
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
# Design D6 rung B: the opt-in linux-x64-glibc217 slot beside the unofficial glibc-217 Node.
# Why docker run and not container: the runner's own Node for JS actions needs glibc 2.28,
# so the host installs and fetches the pinned Node, and the glibc 2.17 image only builds and
# smokes with that Node. devtoolset-10 supplies C++20; the slot links libstdc++ statically.
- name: Build and smoke the glibc 2.17 compat slot under the glibc-217 Node
run: |
compat_node="$(node config/scripts/build-orcad-prebuilds.mjs --slot=linux-x64-glibc217 --print-runtime)"
case "$compat_node" in
"$GITHUB_WORKSPACE"/*) ;;
*) echo "glibc-217 Node cached outside the workspace: $compat_node" >&2; exit 1 ;;
esac
docker run --rm --init -i \
-e ORCA_BACKGROUND_LAUNCH=1 \
-e COMPAT_NODE="/work/${compat_node#"$GITHUB_WORKSPACE"/}" \
-e CC=gcc -e CXX=g++ \
-e PYTHON=/opt/python/cp312-cp312/bin/python3 \
-v "$GITHUB_WORKSPACE:/work" -w /work \
quay.io/pypa/manylinux2014_x86_64@sha256:6f74cabeac2432570aa4bfdb29f7c1f30313d4d6654d764c44e574b6ffdd4ed5 bash -s <<'GLIBC217_COMPAT_SLOT'
set -eu
export PATH="$(dirname "$COMPAT_NODE"):$PATH"
node config/scripts/build-orcad-prebuilds.mjs --slot=linux-x64-glibc217
node config/scripts/build-orcad-prebuilds.mjs --require-slots linux-x64-glibc217
# The image's devtoolset LD_LIBRARY_PATH must not stand in for a host C++ runtime.
env -u LD_LIBRARY_PATH node config/scripts/build-orcad-prebuilds.mjs --slot=linux-x64-glibc217 --smoke
GLIBC217_COMPAT_SLOT
- name: Keep the glibc 2.17 compat slot for the desktop template
if: inputs.build_template
uses: actions/upload-artifact@v7
with:
name: orcad-prebuild-glibc217
path: out/orcad-prebuilds/
if-no-files-found: error
retention-days: 7
overwrite: true
linux_musl:
needs: [changes, persistence, mobile_web_bundle]
concurrency:
group: node-server-linux_musl-${{ matrix.os }}-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }}
# A failed smoke already blocks qualification; missing scope still selects every platform.
if: >-
${{ !cancelled() && needs.persistence.result == 'success' &&
needs.mobile_web_bundle.result == 'success' &&
needs.changes.outputs.should_run != 'false' &&
needs.changes.outputs.qualification != 'false' }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-22.04, ubuntu-24.04-arm]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
persist-credentials: false
- uses: ./.github/actions/restore-pnpm-verification
id: pnpm-verification
with:
container-toolchain: 'true'
- uses: actions/download-artifact@v8
with:
name: orcad-mobile-web-bundle
path: out/mobile-web
- name: Verify native Alpine artifact and persistence
env:
VERIFICATION_CACHE_PATH: ${{ steps.pnpm-verification.outputs.path }}
run: |
touch "$VERIFICATION_CACHE_PATH"
# Multi-arch index digest of the tag; re-resolve it whenever NODE_RUNTIME_PIN moves.
docker run --rm --init -i \
-e ORCA_BACKGROUND_LAUNCH=1 \
-v "$VERIFICATION_CACHE_PATH:/root/.cache/pnpm/lockfile-verified.jsonl" \
-v "$GITHUB_WORKSPACE:/work" -w /work \
node:24.21.0-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1 sh -s <<'NODE_SERVER_QUALIFICATION'
set -eu
apk add --no-cache bash git libstdc++ python3 make g++
git config --global --add safe.directory /work
npm install -g "$(node -p "require('./package.json').packageManager.split('+')[0]")"
pnpm install --frozen-lockfile --ignore-scripts
pnpm build:orcad-prebuilds
pnpm build:orcad-prebuilds --require-slots "$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)"
pnpm build:orcad-prebuilds --smoke
pnpm build:orcad
pnpm test:node-server --artifact
NODE_SERVER_QUALIFICATION
- name: Keep this runner's musl slot for the desktop template
if: inputs.build_template
uses: actions/upload-artifact@v7
with:
name: orcad-prebuild-musl-${{ matrix.os }}
path: out/orcad-prebuilds/
if-no-files-found: error
retention-days: 7
overwrite: true
# Design D2: the desktop ships every target's addons, merged from the lanes that qualified them.
desktop_template:
needs: [persistence, mobile_web_bundle, linux_glibc_floor, linux_glibc217_compat, linux_musl]
if: >-
${{ !cancelled() && inputs.build_template &&
needs.persistence.result == 'success' && needs.mobile_web_bundle.result == 'success' &&
needs.linux_glibc_floor.result == 'success' &&
needs.linux_glibc217_compat.result == 'success' && needs.linux_musl.result == 'success' }}
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Collect every lane's slot
uses: actions/download-artifact@v8
with:
pattern: orcad-prebuild-*
path: ${{ runner.temp }}/orcad-prebuild-lanes
- name: Collect the Windows process-table addons
uses: actions/download-artifact@v8
with:
name: orcad-windows-process-tree
path: .build/windows-process-tree
- name: Collect the Windows CLI launchers
uses: actions/download-artifact@v8
with:
name: orcad-windows-cli-launcher
path: .build/windows-cli-launcher
- name: Collect the Windows security addons
if: hashFiles('config/scripts/build-windows-security-addon.mjs') != ''
uses: actions/download-artifact@v8
with:
name: orcad-windows-security
path: .build/windows-security
- name: Merge the lanes and gate the full slot matrix
shell: bash
run: |
node config/scripts/merge-orcad-prebuilds.mjs "$RUNNER_TEMP"/orcad-prebuild-lanes/*
pnpm build:orcad-prebuilds --require-slots
pnpm build:orcad-prebuilds --require-slots linux-x64-glibc217
# The page the qualification lanes tested, not a rebuild.
- uses: actions/download-artifact@v8
with:
name: orcad-mobile-web-bundle
path: out/mobile-web
- run: pnpm build:orcad-template
- name: Report the template size
shell: bash
run: |
{
echo '### orcad template'
echo '```'
du -sh out/orcad-template
du -sh out/orcad-template/targets/*
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@v7
with:
name: orcad-template
path: out/orcad-template/
# The per-target .server-target and .runtime-node markers are dotfiles.
include-hidden-files: true
if-no-files-found: error
retention-days: 7
overwrite: true
+118
View File
@@ -0,0 +1,118 @@
name: Packaged browser compatibility
# Both jobs pair the current build with a pinned released one. `layout-compat` holds all the layout
# rework baselines (P3), compared with the shapes recorded on main in
# tests/e2e/layout-compat-baselines.json; its artifact carries the observed shapes for re-recording.
on:
workflow_dispatch:
inputs:
ref:
description: Commit SHA or ref to validate (defaults to the selected revision)
type: string
required: false
schedule:
- cron: '20 8 * * 1'
workflow_call:
inputs:
ref:
type: string
required: false
permissions:
contents: read
jobs:
compatibility:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.sha }}
persist-credentials: false
- name: Install headless tools
run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client python3 ripgrep xvfb zsh openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- name: Download pinned old release
uses: ./.github/actions/download-pinned-orca-release
with:
version: 1.4.188
sha512: uGONFUDfinYggxcT9ac72wnnlofLQaqasDDeP0HWOSqarBwTi1Ax3khmzKUY3vUnvuYOpSCEmsH4InzLZ2vg6g==
github-token: ${{ github.token }}
- name: Build current Electron app
env:
VITE_EXPOSE_STORE: 'true'
run: |
pnpm run build:wsl-guest
pnpm exec electron-vite build --mode e2e
pnpm run build:web-from-renderer
- name: Run both mixed-version directions
env:
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/packaged-browser-results.json
run: >-
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1
pnpm exec playwright test --config tests/playwright.config.ts
tests/e2e/packaged-mixed-version-browser-placement.spec.ts
--project=electron-headless --workers=1 --retries=0 --repeat-each=3 --reporter=list,json
- name: Require all six compatibility executions
if: always()
run: node config/scripts/verify-packaged-browser-participation.mjs test-results/packaged-browser-results.json
- uses: actions/upload-artifact@v7
if: always()
with:
name: packaged-mixed-version-audit
path: test-results/
retention-days: 3
layout-compat:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.sha }}
persist-credentials: false
- name: Install headless tools
run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client python3 ripgrep xvfb zsh openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- name: Download pinned old release
uses: ./.github/actions/download-pinned-orca-release
with:
version: 1.4.224
sha512: +/VyaYSgqSAuSSWe1ySrCKRjHHJ6REXqmz58g69bxvoUdV41iH9vlobM18xdxWTkgJF2kbu/tSKFvH0jMnZ0+Q==
github-token: ${{ github.token }}
- name: Build current Electron app
env:
VITE_EXPOSE_STORE: 'true'
run: |
pnpm run build:wsl-guest
pnpm exec electron-vite build --mode e2e
pnpm run build:cli
pnpm run build:web-from-renderer
# The two specs without a released build ride along, so one dispatch reruns every baseline.
- name: Run the layout baselines
env:
ORCA_LAYOUT_COMPAT_REPORT_DIR: test-results/layout-compat
run: >-
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1
pnpm exec playwright test --config tests/playwright.config.ts
tests/e2e/layout-compat-old-desktop-on-new-server.spec.ts
tests/e2e/layout-compat-new-desktop-on-old-server.spec.ts
tests/e2e/layout-compat-old-phone-on-new-server.spec.ts
tests/e2e/layout-compat-serve-restart-keeps-at-at-terminals.spec.ts
--project=electron-headless --workers=1 --retries=0 --reporter=list
- name: Require every layout scenario to have run
if: always()
# A missing env skips the specs and passes; each run writes its observation first.
run: |
for scenario in old-desktop-on-new-server old-cli-terminal-create new-desktop-on-old-server old-phone-on-new-server serve-restart-keeps-@@-terminals; do
test -f "test-results/layout-compat/$scenario.linux.json" || { echo "::error::$scenario did not run"; exit 1; }
done
- uses: actions/upload-artifact@v7
if: always()
with:
name: packaged-layout-compat
path: test-results/
retention-days: 7
@@ -0,0 +1,74 @@
name: Performance contracts
on:
schedule:
- cron: '15 9 * * *'
workflow_dispatch:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- '.github/workflows/performance-contracts.yml'
- 'config/vitest.performance.config.ts'
- 'config/oxlint-performance-audit.json'
- 'config/oxlint-plugins/*performance.mjs'
- 'config/oxlint-plugins/quadratic-buffer-concat.mjs'
- 'config/scripts/*-plugin.test.mjs'
- '!config/scripts/renderer-scrollbar-style-plugin.test.mjs'
- '!config/scripts/mobile-pairing-qrcode-import-plugin.test.mjs'
# Keep in sync with the contract list in config/vitest.performance.config.ts;
# without these a rename lands green and only breaks the next nightly.
- 'src/main/sqlite/sync-database.test.ts'
- 'src/main/runtime/orchestration/db/row-column-lists.test.ts'
- 'src/shared/native-chat-tool-attribution-allocation.test.ts'
- 'src/shared/native-chat-tool-pairing.performance.test.ts'
- 'src/main/codex-usage/codex-rollout-file-lifetime.test.ts'
- 'src/main/codex-usage/scanner-incremental-append.test.ts'
- 'src/main/usage/jsonl-file-reader.test.ts'
- 'src/main/usage/jsonl-file-snapshot.test.ts'
- 'src/renderer/src/components/editor/rich-markdown-list-tokenizers.test.ts'
- 'src/renderer/src/components/editor/rich-markdown-lowlight-cache.test.ts'
- 'src/renderer/src/components/terminal-pane/agent-completion-coordinator-queued-inspection-disposal.test.ts'
- 'src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler-queue-retention.test.ts'
- 'src/main/claude-usage/scanner-incremental-append.test.ts'
- 'src/main/claude-usage/store-checkpoint-persistence.test.ts'
permissions:
contents: read
concurrency:
group: performance-contracts-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
contracts:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Run operation-count and retention contracts
run: pnpm test:perf:contracts --reporter=default --reporter=json --outputFile=performance-contracts.json
# Source-only scan: identical on every OS, so run it once.
- name: Audit production performance patterns
if: always() && matrix.os == 'ubuntu-latest'
shell: bash
run: pnpm --silent audit:perf > performance-audit.json
- uses: actions/upload-artifact@v7
if: always()
with:
name: performance-contracts-${{ matrix.os }}
path: performance-contracts.json
if-no-files-found: error
- uses: actions/upload-artifact@v7
if: always() && matrix.os == 'ubuntu-latest'
with:
name: performance-audit
path: performance-audit.json
if-no-files-found: error
+38
View File
@@ -0,0 +1,38 @@
name: Pi owner runtime verification
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- 'src/main/pi/**'
- '!src/main/pi/**/*.test.ts'
- 'tests/tools/pi-owner-runtime-smoke.mjs'
- 'tests/tools/omp-completion-runtime-smoke.mjs'
- '.github/workflows/pi-owner-runtime.yml'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: pi-owner-runtime-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
runtime:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned extension loader
run: npm install --prefix .cache/pi-owner --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.83.0
- name: Verify real owner exit and hook delivery
run: node tests/tools/pi-owner-runtime-smoke.mjs .cache/pi-owner/node_modules/@earendil-works/pi-coding-agent
- name: Verify OMP completion over native HTTP
run: node tests/tools/omp-completion-runtime-smoke.mjs
+33
View File
@@ -0,0 +1,33 @@
name: Pi extension provider verification
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- 'src/shared/commit-message-agent-specs-primary.ts'
- 'tests/tools/pi-provider-runtime-smoke.mjs'
- '.github/workflows/pi-provider-runtime.yml'
permissions:
contents: read
concurrency:
group: pi-provider-runtime-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
runtime:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned Pi runtime
run: npm install --prefix .cache/pi-provider --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.84.2
- name: Verify extension model generation before and after
run: node tests/tools/pi-provider-runtime-smoke.mjs .cache/pi-provider/node_modules/@earendil-works/pi-coding-agent/dist/cli.js
+2 -2
View File
@@ -6,7 +6,6 @@ on:
- opened
- synchronize
- reopened
- ready_for_review
concurrency:
group: pr-test-loc-${{ github.event.pull_request.number }}
@@ -19,7 +18,8 @@ permissions:
jobs:
loc:
name: test vs non-test LoC
runs-on: ubuntu-latest
# API calls and a small Node script fit the free single-CPU container runner.
runs-on: ubuntu-slim
timeout-minutes: 2
steps:
# Why no checkout: the Files API already has per-file additions/deletions.
+1192 -341
View File
File diff suppressed because it is too large Load Diff
+110
View File
@@ -0,0 +1,110 @@
name: Preflight comparison
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- .github/workflows/preflight-benchmark.yml
- config/scripts/benchmark-preflight-*.mjs
- config/scripts/preflight-benchmark-command.mjs
workflow_dispatch:
permissions:
contents: read
concurrency:
group: preflight-comparison-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
ORCA_BACKGROUND_LAUNCH: '1'
jobs:
compilers:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: ubuntu-24.04-arm
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: none
node-version: '24'
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version-file: config/.bun-version
- name: Compare native and Bun typechecking
run: node config/scripts/benchmark-preflight-compilers.mjs
- uses: actions/upload-artifact@v7
if: always()
with:
name: preflight-compilers
path: .build/preflight-benchmark/
retention-days: 30
static-analysis:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: ubuntu-24.04-arm
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: none
node-version: '24'
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version-file: config/.bun-version
- name: Compare anti-slop patch and script runtimes
run: node config/scripts/benchmark-preflight-static-analysis.mjs
- uses: actions/upload-artifact@v7
if: always()
with:
name: preflight-static-analysis
path: .build/preflight-benchmark/
retention-days: 30
compatibility-imports:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: ubuntu-24.04-arm
timeout-minutes: 10
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
fetch-depth: 2
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: none
node-version: '24'
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version-file: config/.bun-version
- name: Compare compatibility-import scan and runtimes
run: node config/scripts/benchmark-preflight-compat-boundary.mjs
- uses: actions/upload-artifact@v7
if: always()
with:
name: preflight-compat-boundary
path: .build/preflight-benchmark/
retention-days: 30
localization-extraction:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: ubuntu-24.04-arm
timeout-minutes: 12
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: none
node-version: '24'
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version-file: config/.bun-version
- name: Compare localization extraction runtimes and complete catalogs
run: node config/scripts/benchmark-preflight-localization-extraction.mjs
- uses: actions/upload-artifact@v7
if: always()
with:
name: preflight-localization-extraction
path: .build/preflight-benchmark/
retention-days: 30
@@ -0,0 +1,60 @@
name: Preflight CLI platforms
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- 'src/main/ipc/preflight*'
- 'src/main/ipc/agent-detection-shell-path.ts'
- 'src/main/ipc/command-path-resolver*'
- 'src/main/preflight/**'
- 'src/main/startup/*shell*'
- 'src/packages/process-host/**'
- '.github/actions/install-node-dependencies/**'
- '.github/workflows/preflight-cli-platforms.yml'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: preflight-cli-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
preflight:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
name: Preflight CLI (${{ matrix.os }})
runs-on: ${{ matrix.os }}
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
os: [ubuntu-22.04, macos-15, windows-2022]
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
cache-electron-package: 'true'
- name: Check native CLI execution and shell PATH discovery
run: >-
pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=2
src/main/ipc/preflight-command-exec.test.ts
src/main/ipc/preflight-host-cli-status.test.ts
src/main/ipc/preflight-runnable-local-cli.test.ts
src/main/ipc/preflight-agent-detection.test.ts
src/main/ipc/preflight-agent-refresh.test.ts
src/main/ipc/preflight-wsl-agent-detection.test.ts
src/main/ipc/preflight-windows-path-refresh.repro.test.ts
src/main/startup/hydrate-shell-path.test.ts
src/main/startup/hydrate-shell-path.windows.test.ts
src/main/startup/login-shell-environment.test.ts
src/main/runtime/rpc/methods/preflight.test.ts
src/packages/process-host/src/process-tree-termination.test.ts
src/packages/process-host/src/run-process.test.ts
src/packages/process-host/src/run-process-termination-failure.test.ts

Some files were not shown because too many files have changed in this diff Show More