mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
Jinwoo Hong
4f839cc8c9
* perf(relay): cut the cell LB drain to 60s and widen the same-cap batch to ten cells Two independent sources of relay roll wall clock, neither of which protects a host: 1. `connection_draining_timeout_sec` on the per-cell backend services was 300s. The same-cap job drains every host off the cell to a restart-safe condition before Terraform runs, so the LB drain only ever covers a host still mid-handshake. Measured 2026-09-16 over ten same-cap cell jobs, it sat as ~5m55s of dead time between `Apply complete` and the old VM powering off, inside an 8.5-minute `wait-until --stable` step. Now 60s, and pinned in the topology `check` block beside the other fixed-one invariants. 2. The same-cap wave capped a batch at four cells, so a 22-cell roll needed six batches, six single-use monitor gates, and a human handoff per batch. The wave workflow now declares cell_1..cell_10 with the identical serial shape and chaining, and the validator accepts two to ten. The shared wave-index rule (`relay-monitor-evidence.mjs` and the relay-ops preflight CLI) widens from 0-3 to 0-9 so the later cells can present the same evidence; each job workflow keeps its own narrower range, so the capacity wave stays at four. Cells remain strictly serial, one at a time behind the rollout lease, each with its own live preflight. Claude-Session: https://claude.ai/session/relay-roll-drain-timeout-and-batch-cap * fix(relay): align the Asia topology plan validator with the 60s cell drain `validate-relay-asia-topology-plan.mjs` rejected any Asia backend whose `connection_draining_timeout_sec` was not 300, and `cloud-deploy-relay-asia-topology.yml` targets `google_compute_backend_service.relay_gce_cell["<cell>"]` per cell. With the Terraform local at 60 that workflow would have failed its own plan review. The validator's two restated topology values are now named exports, and a new census test reads `relay-gce-cells.tf` and equates three statements of each: the `relay_gce_topology` local, the topology `check` assert that pins it, and the validator constant. Terraform cannot export a local to JS, so reading the source is the only way to stop them drifting; the test was confirmed to fail when the local alone is moved back to 300. Repo-wide grep finds no other pin of the drain value. Claude-Session: https://claude.ai/session/relay-roll-drain-timeout-and-batch-cap
526 lines
24 KiB
YAML
526 lines
24 KiB
YAML
name: Deploy Relay Production Same-Cap
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
mode:
|
|
description: Verify, roll one canary, roll a bounded batch, or roll back
|
|
required: true
|
|
default: verify
|
|
type: choice
|
|
options: [verify, canary-apply, batch-apply, rollback]
|
|
cell-ids:
|
|
description: Ordered comma-separated serving cells; one canary or two to ten batch cells
|
|
required: true
|
|
type: string
|
|
target-image-digest:
|
|
description: Exact immutable compatibility image digest
|
|
required: true
|
|
type: string
|
|
rollback-image-digest:
|
|
description: Exact immutable currently serving rollback digest
|
|
required: true
|
|
type: string
|
|
target-rehome-protocol:
|
|
description: Exact target regional-rehome protocol
|
|
required: true
|
|
default: '1'
|
|
type: choice
|
|
options: ['0', '1', '3']
|
|
rollback-rehome-protocol:
|
|
description: Exact rollback regional-rehome protocol
|
|
required: true
|
|
default: '0'
|
|
type: choice
|
|
options: ['0', '1', '3']
|
|
expected-selector-generation:
|
|
description: Exact selector generation before the first cell
|
|
required: true
|
|
type: string
|
|
expected-existing-only-cells:
|
|
description: Exact existing-only membership, or none
|
|
required: true
|
|
type: string
|
|
expected-migration-only-cells:
|
|
description: Exact migration-only membership, or none
|
|
required: true
|
|
type: string
|
|
expected-general-cells:
|
|
description: Exact general membership, or none
|
|
required: true
|
|
type: string
|
|
expected-rehome-generation:
|
|
description: Exact durable regional-rehome control generation; it must be disabled
|
|
required: true
|
|
type: string
|
|
monitor-run-id:
|
|
description: Fresh successful aggregate dry-run monitor workflow run
|
|
required: false
|
|
type: string
|
|
monitor-run-attempt:
|
|
description: Exact monitor attempt
|
|
required: false
|
|
type: string
|
|
canary-run-id:
|
|
description: Successful same-code canary in this rehome control generation; reusable across batches
|
|
required: false
|
|
type: string
|
|
confirmation:
|
|
description: Exact digest-and-cell-bound mutation confirmation
|
|
required: false
|
|
type: string
|
|
gate-override-reason:
|
|
description: >-
|
|
Break-glass only: why this wave may skip the aggregate 15-minute monitor
|
|
dry-run gate. The live per-wave preflight still runs.
|
|
required: false
|
|
type: string
|
|
gate-override-confirmation:
|
|
description: >-
|
|
Break-glass only: exactly "SKIP_RELAY_MONITOR_GATE <target-image-digest>"
|
|
required: false
|
|
type: string
|
|
|
|
permissions:
|
|
actions: read
|
|
contents: read
|
|
id-token: write
|
|
|
|
concurrency:
|
|
group: production-cloud-sql-rollout
|
|
cancel-in-progress: false
|
|
|
|
defaults:
|
|
run:
|
|
working-directory: cloud
|
|
|
|
jobs:
|
|
gate:
|
|
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
|
runs-on: blacksmith-2vcpu-ubuntu-2204
|
|
# Headroom for the full-history checkout the canary provenance check needs.
|
|
timeout-minutes: 15
|
|
environment: production
|
|
outputs:
|
|
cells: ${{ steps.wave.outputs.cells }}
|
|
job-mode: ${{ steps.wave.outputs.job-mode }}
|
|
steps:
|
|
# Full history: the canary authority a batch verifies is sealed at an ancestor commit, and
|
|
# the provenance check fails closed on a commit a shallow clone left out.
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: 24 }
|
|
|
|
- id: wave
|
|
env:
|
|
MODE: ${{ inputs.mode }}
|
|
CELL_IDS: ${{ inputs.cell-ids }}
|
|
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
|
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
|
CONFIRMATION: ${{ inputs.confirmation }}
|
|
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
|
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
|
|
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
|
|
run: |
|
|
# Fails closed on a partial or mismatched override, before any mutation.
|
|
CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \
|
|
--mode "${MODE}" --cell-ids "${CELL_IDS}" \
|
|
--target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \
|
|
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}" \
|
|
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
|
|
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}")"
|
|
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
|
|
if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then
|
|
echo 'job-mode=apply' >> "${GITHUB_OUTPUT}"
|
|
else
|
|
echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}"
|
|
fi
|
|
|
|
- name: Record the monitor gate override in the run summary
|
|
if: ${{ inputs.gate-override-confirmation != '' }}
|
|
env:
|
|
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
|
|
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
|
|
ACTOR: ${{ github.actor }}
|
|
MODE: ${{ inputs.mode }}
|
|
CELL_IDS: ${{ inputs.cell-ids }}
|
|
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
|
run: |
|
|
{
|
|
echo '## Aggregate monitor gate overridden (break-glass)'
|
|
echo
|
|
echo '| field | value |'
|
|
echo '| --- | --- |'
|
|
echo "| actor | ${ACTOR} |"
|
|
echo "| mode | ${MODE} |"
|
|
echo "| cells | ${CELL_IDS} |"
|
|
echo "| target digest | \`${TARGET_DIGEST}\` |"
|
|
echo "| reason | ${GATE_OVERRIDE_REASON} |"
|
|
echo "| confirmation | \`${GATE_OVERRIDE_CONFIRMATION}\` |"
|
|
echo
|
|
echo 'Skipped: the 15-minute aggregate monitor dry-run and its sealed evidence.'
|
|
echo 'Still enforced: the live per-wave preflight against the same thresholds,'
|
|
echo 'durable rehome disabled, the exact selector generation and membership, the'
|
|
echo 'reviewed Terraform plan, one cell at a time behind the rollout lease, and'
|
|
echo 'single-dispatch mutation.'
|
|
} >> "${GITHUB_STEP_SUMMARY}"
|
|
|
|
- name: Download exact prior canary authority
|
|
if: ${{ inputs.mode == 'batch-apply' }}
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: relay-same-cap-canary-${{ inputs.canary-run-id }}
|
|
path: ${{ runner.temp }}/relay-same-cap-canary
|
|
github-token: ${{ github.token }}
|
|
run-id: ${{ inputs.canary-run-id }}
|
|
|
|
- name: Verify canary authority against this batch
|
|
if: ${{ inputs.mode == 'batch-apply' }}
|
|
env:
|
|
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
|
CELL_IDS: ${{ inputs.cell-ids }}
|
|
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
|
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
|
SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
|
REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
|
run: |
|
|
node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
|
|
--file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
|
|
--commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
|
|
--cell-ids "${CELL_IDS}" \
|
|
--target-digest "${TARGET_DIGEST}" \
|
|
--rollback-digest "${ROLLBACK_DIGEST}" \
|
|
--selector-generation "${SELECTOR_GENERATION}" \
|
|
--rehome-generation "${REHOME_GENERATION}"
|
|
|
|
- name: Reject previously consumed aggregate safety evidence
|
|
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
|
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
|
run: |
|
|
[[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
|
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
|
MARKER_NAME="relay-same-cap-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
|
COUNT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
|
--jq '.total_count')"
|
|
test "${COUNT}" = 0
|
|
mkdir -p "${RUNNER_TEMP}/relay-same-cap-monitor-authority"
|
|
printf '%s\n' "${GITHUB_RUN_ID}" \
|
|
> "${RUNNER_TEMP}/relay-same-cap-monitor-authority/${MARKER_NAME}"
|
|
|
|
- name: Consume aggregate safety evidence for this exact wave
|
|
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
|
path: ${{ runner.temp }}/relay-same-cap-monitor-authority/relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
|
retention-days: 90
|
|
if-no-files-found: error
|
|
|
|
cell_1:
|
|
needs: gate
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[0] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '0'
|
|
secrets: inherit
|
|
|
|
cell_2:
|
|
if: ${{ needs.cell_1.result == 'success' && fromJSON(needs.gate.outputs.cells)[1] != null }}
|
|
needs: [gate, cell_1]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[1] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '1'
|
|
secrets: inherit
|
|
|
|
cell_3:
|
|
if: ${{ needs.cell_2.result == 'success' && fromJSON(needs.gate.outputs.cells)[2] != null }}
|
|
needs: [gate, cell_2]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[2] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '2'
|
|
secrets: inherit
|
|
|
|
cell_4:
|
|
if: ${{ needs.cell_3.result == 'success' && fromJSON(needs.gate.outputs.cells)[3] != null }}
|
|
needs: [gate, cell_3]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[3] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '3'
|
|
secrets: inherit
|
|
|
|
cell_5:
|
|
if: ${{ needs.cell_4.result == 'success' && fromJSON(needs.gate.outputs.cells)[4] != null }}
|
|
needs: [gate, cell_4]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[4] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '4'
|
|
secrets: inherit
|
|
|
|
cell_6:
|
|
if: ${{ needs.cell_5.result == 'success' && fromJSON(needs.gate.outputs.cells)[5] != null }}
|
|
needs: [gate, cell_5]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[5] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '5'
|
|
secrets: inherit
|
|
|
|
cell_7:
|
|
if: ${{ needs.cell_6.result == 'success' && fromJSON(needs.gate.outputs.cells)[6] != null }}
|
|
needs: [gate, cell_6]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[6] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '6'
|
|
secrets: inherit
|
|
|
|
cell_8:
|
|
if: ${{ needs.cell_7.result == 'success' && fromJSON(needs.gate.outputs.cells)[7] != null }}
|
|
needs: [gate, cell_7]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[7] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '7'
|
|
secrets: inherit
|
|
|
|
cell_9:
|
|
if: ${{ needs.cell_8.result == 'success' && fromJSON(needs.gate.outputs.cells)[8] != null }}
|
|
needs: [gate, cell_8]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[8] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '8'
|
|
secrets: inherit
|
|
|
|
cell_10:
|
|
if: ${{ needs.cell_9.result == 'success' && fromJSON(needs.gate.outputs.cells)[9] != null }}
|
|
needs: [gate, cell_9]
|
|
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
|
with:
|
|
mode: ${{ needs.gate.outputs.job-mode }}
|
|
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[9] }}
|
|
target-image-digest: ${{ inputs.target-image-digest }}
|
|
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
|
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
|
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
|
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
|
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
|
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
|
expected-general-cells: ${{ inputs.expected-general-cells }}
|
|
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
|
monitor-run-id: ${{ inputs.monitor-run-id }}
|
|
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
|
gate-override-reason: ${{ inputs.gate-override-reason }}
|
|
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
|
wave-index: '9'
|
|
secrets: inherit
|
|
|
|
seal_canary:
|
|
if: ${{ inputs.mode == 'canary-apply' }}
|
|
needs: [gate, cell_1]
|
|
runs-on: blacksmith-2vcpu-ubuntu-2204
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: 24 }
|
|
|
|
- name: Seal exact successful canary authority
|
|
env:
|
|
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
|
|
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
|
|
ACTOR: ${{ github.actor }}
|
|
run: |
|
|
mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary"
|
|
node dev/scripts/relay-production-same-cap-wave.mjs create-canary \
|
|
--cell-id "${{ inputs.cell-ids }}" \
|
|
--target-digest "${{ inputs.target-image-digest }}" \
|
|
--rollback-digest "${{ inputs.rollback-image-digest }}" \
|
|
--confirmation "${{ inputs.confirmation }}" \
|
|
--commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \
|
|
--selector-generation "${{ inputs.expected-selector-generation }}" \
|
|
--rehome-generation "${{ inputs.expected-rehome-generation }}" \
|
|
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
|
|
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}" \
|
|
--actor "${ACTOR}" \
|
|
> "${RUNNER_TEMP}/relay-same-cap-canary/authority.json"
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
with:
|
|
name: relay-same-cap-canary-${{ github.run_id }}
|
|
path: ${{ runner.temp }}/relay-same-cap-canary/authority.json
|
|
retention-days: 30
|
|
if-no-files-found: error
|
|
|
|
# Every cell job re-enters the run's lease with release: 'false'; only this job frees it.
|
|
release_lease:
|
|
if: always()
|
|
needs:
|
|
- gate
|
|
- cell_1
|
|
- cell_2
|
|
- cell_3
|
|
- cell_4
|
|
- cell_5
|
|
- cell_6
|
|
- cell_7
|
|
- cell_8
|
|
- cell_9
|
|
- cell_10
|
|
- seal_canary
|
|
runs-on: blacksmith-2vcpu-ubuntu-2204
|
|
timeout-minutes: 10
|
|
environment: production
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: google-github-actions/auth@v2
|
|
with:
|
|
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
|
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
|
|
|
- uses: google-github-actions/setup-gcloud@v2
|
|
|
|
- uses: ./.github/actions/cloud-sql-rollout-lease
|
|
with:
|
|
bucket: onorca-cloud-terraform-state
|
|
object: terraform/state/cloud-sql-rollout/production.lock
|
|
release: 'true'
|