mirror of
https://github.com/stablyai/orca.git
synced 2026-09-23 08:02:31 +00:00
Jinwoo Hong
ff8f7085cc
A batch-apply wave verified only that the sealed canary named some approved same-cap cell, so a canary rolled on the migration-only, zero-host, 600-cap c17 or c18 was accepted as authority for a general 1000/3000-cap batch. The verify step now hands the batch's own cells to the check, which requires the sealed cell's entry admission to equal the batch's class.
382 lines
16 KiB
YAML
382 lines
16 KiB
YAML
name: Deploy Relay Production Same-Cap
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
mode:
|
|
description: Verify, roll one canary, roll a bounded batch, or roll back
|
|
required: true
|
|
default: verify
|
|
type: choice
|
|
options: [verify, canary-apply, batch-apply, rollback]
|
|
cell-ids:
|
|
description: Ordered comma-separated serving cells; one canary or two to four batch cells
|
|
required: true
|
|
type: string
|
|
target-image-digest:
|
|
description: Exact immutable compatibility image digest
|
|
required: true
|
|
type: string
|
|
rollback-image-digest:
|
|
description: Exact immutable currently serving rollback digest
|
|
required: true
|
|
type: string
|
|
target-rehome-protocol:
|
|
description: Exact target regional-rehome protocol
|
|
required: true
|
|
default: '1'
|
|
type: choice
|
|
options: ['0', '1', '3']
|
|
rollback-rehome-protocol:
|
|
description: Exact rollback regional-rehome protocol
|
|
required: true
|
|
default: '0'
|
|
type: choice
|
|
options: ['0', '1', '3']
|
|
expected-selector-generation:
|
|
description: Exact selector generation before the first cell
|
|
required: true
|
|
type: string
|
|
expected-existing-only-cells:
|
|
description: Exact existing-only membership, or none
|
|
required: true
|
|
type: string
|
|
expected-migration-only-cells:
|
|
description: Exact migration-only membership, or none
|
|
required: true
|
|
type: string
|
|
expected-general-cells:
|
|
description: Exact general membership, or none
|
|
required: true
|
|
type: string
|
|
expected-rehome-generation:
|
|
description: Exact durable regional-rehome control generation; it must be disabled
|
|
required: true
|
|
type: string
|
|
monitor-run-id:
|
|
description: Fresh successful aggregate dry-run monitor workflow run
|
|
required: false
|
|
type: string
|
|
monitor-run-attempt:
|
|
description: Exact monitor attempt
|
|
required: false
|
|
type: string
|
|
canary-run-id:
|
|
description: Successful same-code canary in this rehome control generation; reusable across batches
|
|
required: false
|
|
type: string
|
|
confirmation:
|
|
description: Exact digest-and-cell-bound mutation confirmation
|
|
required: false
|
|
type: string
|
|
gate-override-reason:
|
|
description: >-
|
|
Break-glass only: why this wave may skip the aggregate 15-minute monitor
|
|
dry-run gate. The live per-wave preflight still runs.
|
|
required: false
|
|
type: string
|
|
gate-override-confirmation:
|
|
description: >-
|
|
Break-glass only: exactly "SKIP_RELAY_MONITOR_GATE <target-image-digest>"
|
|
required: false
|
|
type: string
|
|
|
|
permissions:
|
|
actions: read
|
|
contents: read
|
|
id-token: write
|
|
|
|
concurrency:
|
|
group: production-cloud-sql-rollout
|
|
cancel-in-progress: false
|
|
|
|
defaults:
|
|
run:
|
|
working-directory: cloud
|
|
|
|
jobs:
|
|
gate:
|
|
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
|
runs-on: blacksmith-2vcpu-ubuntu-2204
|
|
# Headroom for the full-history checkout the canary provenance check needs.
|
|
timeout-minutes: 15
|
|
environment: production
|
|
outputs:
|
|
cells: ${{ steps.wave.outputs.cells }}
|
|
job-mode: ${{ steps.wave.outputs.job-mode }}
|
|
steps:
|
|
# Full history: the canary authority a batch verifies is sealed at an ancestor commit, and
|
|
# the provenance check fails closed on a commit a shallow clone left out.
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: 24 }
|
|
|
|
- id: wave
|
|
env:
|
|
MODE: ${{ inputs.mode }}
|
|
CELL_IDS: ${{ inputs.cell-ids }}
|
|
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
|
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
|
CONFIRMATION: ${{ inputs.confirmation }}
|
|
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
|
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
|
|
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
|
|
run: |
|
|
# Fails closed on a partial or mismatched override, before any mutation.
|
|
CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \
|
|
--mode "${MODE}" --cell-ids "${CELL_IDS}" \
|
|
--target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \
|
|
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}" \
|
|
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
|
|
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}")"
|
|
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
|
|
if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then
|
|
echo 'job-mode=apply' >> "${GITHUB_OUTPUT}"
|
|
else
|
|
echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}"
|
|
fi
|
|
|
|
- name: Record the monitor gate override in the run summary
|
|
if: ${{ inputs.gate-override-confirmation != '' }}
|
|
env:
|
|
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
|
|
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
|
|
ACTOR: ${{ github.actor }}
|
|
MODE: ${{ inputs.mode }}
|
|
CELL_IDS: ${{ inputs.cell-ids }}
|
|
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
|
run: |
|
|
{
|
|
echo '## Aggregate monitor gate overridden (break-glass)'
|
|
echo
|
|
echo '| field | value |'
|
|
echo '| --- | --- |'
|
|
echo "| actor | ${ACTOR} |"
|
|
echo "| mode | ${MODE} |"
|
|
echo "| cells | ${CELL_IDS} |"
|
|
echo "| target digest | \`${TARGET_DIGEST}\` |"
|
|
echo "| reason | ${GATE_OVERRIDE_REASON} |"
|
|
echo "| confirmation | \`${GATE_OVERRIDE_CONFIRMATION}\` |"
|
|
echo
|
|
echo 'Skipped: the 15-minute aggregate monitor dry-run and its sealed evidence.'
|
|
echo 'Still enforced: the live per-wave preflight against the same thresholds,'
|
|
echo 'durable rehome disabled, the exact selector generation and membership, the'
|
|
echo 'reviewed Terraform plan, one cell at a time behind the rollout lease, and'
|
|
echo 'single-dispatch mutation.'
|
|
} >> "${GITHUB_STEP_SUMMARY}"
|
|
|
|
- name: Download exact prior canary authority
|
|
if: ${{ inputs.mode == 'batch-apply' }}
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: relay-same-cap-canary-${{ inputs.canary-run-id }}
|
|
path: ${{ runner.temp }}/relay-same-cap-canary
|
|
github-token: ${{ github.token }}
|
|
run-id: ${{ inputs.canary-run-id }}
|
|
|
|
- name: Verify canary authority against this batch
|
|
if: ${{ inputs.mode == 'batch-apply' }}
|
|
env:
|
|
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
|
CELL_IDS: ${{ inputs.cell-ids }}
|
|
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
|
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
|
SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
|
REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
|
run: |
|
|
node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
|
|
--file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
|
|
--commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
|
|
--cell-ids "${CELL_IDS}" \
|
|
--target-digest "${TARGET_DIGEST}" \
|
|
--rollback-digest "${ROLLBACK_DIGEST}" \
|
|
--selector-generation "${SELECTOR_GENERATION}" \
|
|
--rehome-generation "${REHOME_GENERATION}"
|
|
|
|
- name: Reject previously consumed aggregate safety evidence
|
|
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
|
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
|
run: |
|
|
[[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
|
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
|
MARKER_NAME="relay-same-cap-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
|
COUNT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
|
--jq '.total_count')"
|
|
test "${COUNT}" = 0
|
|
mkdir -p "${RUNNER_TEMP}/relay-same-cap-monitor-authority"
|
|
printf '%s\n' "${GITHUB_RUN_ID}" \
|
|
> "${RUNNER_TEMP}/relay-same-cap-monitor-authority/${MARKER_NAME}"
|
|
|
|
- name: Consume aggregate safety evidence for this exact wave
|
|
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
|
path: ${{ runner.temp }}/relay-same-cap-monitor-authority/relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
|
retention-days: 90
|
|
if-no-files-found: error
|
|
|
|
cell_1:
|
|
needs: gate
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[0] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '0'
|
|
secrets: inherit
|
|
|
|
cell_2:
|
|
if: ${{ needs.cell_1.result == 'success' && fromJSON(needs.gate.outputs.cells)[1] != null }}
|
|
needs: [gate, cell_1]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[1] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '1'
|
|
secrets: inherit
|
|
|
|
cell_3:
|
|
if: ${{ needs.cell_2.result == 'success' && fromJSON(needs.gate.outputs.cells)[2] != null }}
|
|
needs: [gate, cell_2]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[2] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '2'
|
|
secrets: inherit
|
|
|
|
cell_4:
|
|
if: ${{ needs.cell_3.result == 'success' && fromJSON(needs.gate.outputs.cells)[3] != null }}
|
|
needs: [gate, cell_3]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[3] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '3'
|
|
secrets: inherit
|
|
|
|
seal_canary:
|
|
if: ${{ inputs.mode == 'canary-apply' }}
|
|
needs: [gate, cell_1]
|
|
runs-on: blacksmith-2vcpu-ubuntu-2204
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: 24 }
|
|
|
|
- name: Seal exact successful canary authority
|
|
env:
|
|
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
|
|
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
|
|
ACTOR: ${{ github.actor }}
|
|
run: |
|
|
mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary"
|
|
node dev/scripts/relay-production-same-cap-wave.mjs create-canary \
|
|
--cell-id "${{ inputs.cell-ids }}" \
|
|
--target-digest "${{ inputs.target-image-digest }}" \
|
|
--rollback-digest "${{ inputs.rollback-image-digest }}" \
|
|
--confirmation "${{ inputs.confirmation }}" \
|
|
--commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \
|
|
--selector-generation "${{ inputs.expected-selector-generation }}" \
|
|
--rehome-generation "${{ inputs.expected-rehome-generation }}" \
|
|
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
|
|
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}" \
|
|
--actor "${ACTOR}" \
|
|
> "${RUNNER_TEMP}/relay-same-cap-canary/authority.json"
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
with:
|
|
name: relay-same-cap-canary-${{ github.run_id }}
|
|
path: ${{ runner.temp }}/relay-same-cap-canary/authority.json
|
|
retention-days: 30
|
|
if-no-files-found: error
|
|
|
|
# Every cell job re-enters the run's lease with release: 'false'; only this job frees it.
|
|
release_lease:
|
|
if: always()
|
|
needs:
|
|
- gate
|
|
- cell_1
|
|
- cell_2
|
|
- cell_3
|
|
- cell_4
|
|
- seal_canary
|
|
runs-on: blacksmith-2vcpu-ubuntu-2204
|
|
timeout-minutes: 10
|
|
environment: production
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: google-github-actions/auth@v2
|
|
with:
|
|
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
|
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
|
|
|
- uses: google-github-actions/setup-gcloud@v2
|
|
|
|
- uses: ./.github/actions/cloud-sql-rollout-lease
|
|
with:
|
|
bucket: onorca-cloud-terraform-state
|
|
object: terraform/state/cloud-sql-rollout/production.lock
|
|
release: 'true'
|