mirror of
https://github.com/stablyai/orca.git
synced 2026-10-05 00:02:33 +00:00
Jinwoo Hong
07dad6739a
* refactor(relay): sample fleet health inside the same-cap roll instead of a separate monitor run A same-cap wave no longer consumes a 15-minute monitor dry-run and its sealed, single-use, five-minute-fresh evidence. Each apply wave now samples fleet health itself right before isolation, with the monitor's evaluator, thresholds, and tolerances, for a window sized to the cell's host count (3/5/8 min), plus three lookback rules: no cell container exit in 10 min, no minute over 500 director 503s in 10 min, and director concurrency p99 within the monitor bar over 4 min. Removes the monitor-run inputs, the gate's consume/authorize steps, the break-glass override, and the same-cap-only authorization shapes in relay-monitor-evidence.mjs. The monitor workflow and the rehome enable path are unchanged. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * fix(relay): bound the pre-drain sample overrun and keep the drain token fresh Review follow-ups: alternating tolerated readings could hold the sample open until its step timeout, so cap the overrun at three samples past the window; record why a read failed; mint a fresh admin ID token for the drain after the sample; raise the job timeout to 90 min so a long sample cannot cancel the job past the failsafe. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * feat(relay): exempt the rolled cell and existing-only cells from the pre-drain crash rule The exit rule counted every relay container exit fleet-wide, so a cell that crashes every few hours (c25, 12 a week) blocked the very roll that fixes it, and existing-only legacy cells (c5, 15 a week) blocked rolls they take no part in. Exits are now grouped by instance, each instance is named by its own newest runtime-metrics log line, and only exits on general or migration-only cells other than the target count. An exit no configured cell can be named for trips the rule; a failed lookup is a failed read. relay-observability.tf joins the evidence-code set because the rule depends on its filter. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * test(relay): cover re-asking for an unnamed exiting instance; note the boot-exit risk Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010
400 lines
17 KiB
YAML
400 lines
17 KiB
YAML
name: Deploy Relay Production Same-Cap
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
mode:
|
|
description: Verify, roll one canary, roll a bounded batch, or roll back
|
|
required: true
|
|
default: verify
|
|
type: choice
|
|
options: [verify, canary-apply, batch-apply, rollback]
|
|
cell-ids:
|
|
description: Ordered comma-separated serving cells; one canary or two to ten batch cells
|
|
required: true
|
|
type: string
|
|
target-image-digest:
|
|
description: Exact immutable compatibility image digest
|
|
required: true
|
|
type: string
|
|
rollback-image-digest:
|
|
description: Exact immutable currently serving rollback digest
|
|
required: true
|
|
type: string
|
|
target-rehome-protocol:
|
|
description: Exact target regional-rehome protocol
|
|
required: true
|
|
default: '1'
|
|
type: choice
|
|
options: ['0', '1', '3']
|
|
rollback-rehome-protocol:
|
|
description: Exact rollback regional-rehome protocol
|
|
required: true
|
|
default: '0'
|
|
type: choice
|
|
options: ['0', '1', '3']
|
|
expected-selector-generation:
|
|
description: Exact selector generation before the first cell
|
|
required: true
|
|
type: string
|
|
expected-existing-only-cells:
|
|
description: Exact existing-only membership, or none
|
|
required: true
|
|
type: string
|
|
expected-migration-only-cells:
|
|
description: Exact migration-only membership, or none
|
|
required: true
|
|
type: string
|
|
expected-general-cells:
|
|
description: Exact general membership, or none
|
|
required: true
|
|
type: string
|
|
expected-rehome-generation:
|
|
description: Exact durable regional-rehome control generation; it must be disabled
|
|
required: true
|
|
type: string
|
|
canary-run-id:
|
|
description: Successful same-code canary in this rehome control generation; reusable across batches
|
|
required: false
|
|
type: string
|
|
confirmation:
|
|
description: Exact digest-and-cell-bound mutation confirmation
|
|
required: false
|
|
type: string
|
|
|
|
permissions:
|
|
actions: read
|
|
contents: read
|
|
id-token: write
|
|
|
|
concurrency:
|
|
group: production-cloud-sql-rollout
|
|
cancel-in-progress: false
|
|
|
|
defaults:
|
|
run:
|
|
working-directory: cloud
|
|
|
|
jobs:
|
|
gate:
|
|
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
|
runs-on: blacksmith-2vcpu-ubuntu-2204
|
|
# Headroom for the full-history checkout the canary provenance check needs.
|
|
timeout-minutes: 15
|
|
environment: production
|
|
outputs:
|
|
cells: ${{ steps.wave.outputs.cells }}
|
|
job-mode: ${{ steps.wave.outputs.job-mode }}
|
|
steps:
|
|
# Full history: the canary authority a batch verifies is sealed at an ancestor commit, and
|
|
# the provenance check fails closed on a commit a shallow clone left out.
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: 24 }
|
|
|
|
- id: wave
|
|
env:
|
|
MODE: ${{ inputs.mode }}
|
|
CELL_IDS: ${{ inputs.cell-ids }}
|
|
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
|
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
|
CONFIRMATION: ${{ inputs.confirmation }}
|
|
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
|
run: |
|
|
CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \
|
|
--mode "${MODE}" --cell-ids "${CELL_IDS}" \
|
|
--target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \
|
|
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}")"
|
|
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
|
|
if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then
|
|
echo 'job-mode=apply' >> "${GITHUB_OUTPUT}"
|
|
else
|
|
echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}"
|
|
fi
|
|
|
|
- name: Download exact prior canary authority
|
|
if: ${{ inputs.mode == 'batch-apply' }}
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: relay-same-cap-canary-${{ inputs.canary-run-id }}
|
|
path: ${{ runner.temp }}/relay-same-cap-canary
|
|
github-token: ${{ github.token }}
|
|
run-id: ${{ inputs.canary-run-id }}
|
|
|
|
- name: Verify canary authority against this batch
|
|
if: ${{ inputs.mode == 'batch-apply' }}
|
|
env:
|
|
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
|
CELL_IDS: ${{ inputs.cell-ids }}
|
|
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
|
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
|
SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
|
REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
|
run: |
|
|
node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
|
|
--file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
|
|
--commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
|
|
--cell-ids "${CELL_IDS}" \
|
|
--target-digest "${TARGET_DIGEST}" \
|
|
--rollback-digest "${ROLLBACK_DIGEST}" \
|
|
--selector-generation "${SELECTOR_GENERATION}" \
|
|
--rehome-generation "${REHOME_GENERATION}"
|
|
|
|
cell_1:
|
|
needs: gate
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[0] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
wave-index: '0'
|
|
secrets: inherit
|
|
|
|
cell_2:
|
|
if: ${{ needs.cell_1.result == 'success' && fromJSON(needs.gate.outputs.cells)[1] != null }}
|
|
needs: [gate, cell_1]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[1] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
wave-index: '1'
|
|
secrets: inherit
|
|
|
|
cell_3:
|
|
if: ${{ needs.cell_2.result == 'success' && fromJSON(needs.gate.outputs.cells)[2] != null }}
|
|
needs: [gate, cell_2]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[2] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
wave-index: '2'
|
|
secrets: inherit
|
|
|
|
cell_4:
|
|
if: ${{ needs.cell_3.result == 'success' && fromJSON(needs.gate.outputs.cells)[3] != null }}
|
|
needs: [gate, cell_3]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[3] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
wave-index: '3'
|
|
secrets: inherit
|
|
|
|
cell_5:
|
|
if: ${{ needs.cell_4.result == 'success' && fromJSON(needs.gate.outputs.cells)[4] != null }}
|
|
needs: [gate, cell_4]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[4] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
wave-index: '4'
|
|
secrets: inherit
|
|
|
|
cell_6:
|
|
if: ${{ needs.cell_5.result == 'success' && fromJSON(needs.gate.outputs.cells)[5] != null }}
|
|
needs: [gate, cell_5]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[5] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
wave-index: '5'
|
|
secrets: inherit
|
|
|
|
cell_7:
|
|
if: ${{ needs.cell_6.result == 'success' && fromJSON(needs.gate.outputs.cells)[6] != null }}
|
|
needs: [gate, cell_6]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[6] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
wave-index: '6'
|
|
secrets: inherit
|
|
|
|
cell_8:
|
|
if: ${{ needs.cell_7.result == 'success' && fromJSON(needs.gate.outputs.cells)[7] != null }}
|
|
needs: [gate, cell_7]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[7] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
wave-index: '7'
|
|
secrets: inherit
|
|
|
|
cell_9:
|
|
if: ${{ needs.cell_8.result == 'success' && fromJSON(needs.gate.outputs.cells)[8] != null }}
|
|
needs: [gate, cell_8]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[8] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
wave-index: '8'
|
|
secrets: inherit
|
|
|
|
cell_10:
|
|
if: ${{ needs.cell_9.result == 'success' && fromJSON(needs.gate.outputs.cells)[9] != null }}
|
|
needs: [gate, cell_9]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[9] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
wave-index: '9'
|
|
secrets: inherit
|
|
|
|
seal_canary:
|
|
if: ${{ inputs.mode == 'canary-apply' }}
|
|
needs: [gate, cell_1]
|
|
runs-on: blacksmith-2vcpu-ubuntu-2204
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: 24 }
|
|
|
|
- name: Seal exact successful canary authority
|
|
run: |
|
|
mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary"
|
|
node dev/scripts/relay-production-same-cap-wave.mjs create-canary \
|
|
--cell-id "${{ inputs.cell-ids }}" \
|
|
--target-digest "${{ inputs.target-image-digest }}" \
|
|
--rollback-digest "${{ inputs.rollback-image-digest }}" \
|
|
--confirmation "${{ inputs.confirmation }}" \
|
|
--commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \
|
|
--selector-generation "${{ inputs.expected-selector-generation }}" \
|
|
--rehome-generation "${{ inputs.expected-rehome-generation }}" \
|
|
> "${RUNNER_TEMP}/relay-same-cap-canary/authority.json"
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
with:
|
|
name: relay-same-cap-canary-${{ github.run_id }}
|
|
path: ${{ runner.temp }}/relay-same-cap-canary/authority.json
|
|
retention-days: 30
|
|
if-no-files-found: error
|
|
|
|
# Every cell job re-enters the run's lease with release: 'false'; only this job frees it.
|
|
release_lease:
|
|
if: always()
|
|
needs:
|
|
- gate
|
|
- cell_1
|
|
- cell_2
|
|
- cell_3
|
|
- cell_4
|
|
- cell_5
|
|
- cell_6
|
|
- cell_7
|
|
- cell_8
|
|
- cell_9
|
|
- cell_10
|
|
- seal_canary
|
|
runs-on: blacksmith-2vcpu-ubuntu-2204
|
|
timeout-minutes: 10
|
|
environment: production
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: google-github-actions/auth@v2
|
|
with:
|
|
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
|
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
|
|
|
- uses: google-github-actions/setup-gcloud@v2
|
|
|
|
- uses: ./.github/actions/cloud-sql-rollout-lease
|
|
with:
|
|
bucket: onorca-cloud-terraform-state
|
|
object: terraform/state/cloud-sql-rollout/production.lock
|
|
release: 'true'
|