mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
Move Relay deployment configuration into private cloud operations (#31792)
* refactor(cloud): move deployment configuration into private operations repo * docs(cloud): clarify private operator ownership * fix(ci): install pnpm through npm-compatible bootstrap * fix(ci): avoid native downloader metadata timeout * test(push): move deployment preflight contract into private repo * fix(ci): bootstrap Node before installing pnpm in minimal containers * fix(ci): pass Windows boundary test paths through the Node runner * fix(ci): run Windows boundary Vitest with installed Node
This commit is contained in:
@@ -1,678 +0,0 @@
|
||||
name: Bootstrap Relay Staging Capacity
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
confirmation:
|
||||
description: Enter BOOTSTRAP_STAGING_CAPACITY
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
bootstrap:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
GCP_REGION: us-central1
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
LEGACY_C3_IMAGE_DIGEST: sha256:2d0f6e6db2b0eb9d6aba188698de8330f8c30b4e76badfcf0fac3f3eb9508a87
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: deploy-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: capacity-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: access_token
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Require explicit bootstrap confirmation
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: test "${CONFIRMATION}" = "BOOTSTRAP_STAGING_CAPACITY"
|
||||
|
||||
- name: Read and verify reviewed 600/60 topology
|
||||
shell: bash
|
||||
run: |
|
||||
node dev/scripts/infra.mjs init --env staging
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
|
||||
> "${RUNNER_TEMP}/relay-gce-state.json"
|
||||
DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'local.relay_director_cells_json' | jq -r '.')"
|
||||
DESIRED_IMAGES_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.image })' \
|
||||
| jq -r '.')"
|
||||
DESIRED_ZONES_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.zone })' \
|
||||
| jq -r '.')"
|
||||
DESIRED_TARGET_SIZES_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode(local.relay_gce_cell_target_sizes)' | jq -r '.')"
|
||||
jq -e \
|
||||
--argjson images "${DESIRED_IMAGES_JSON}" \
|
||||
--argjson target_sizes "${DESIRED_TARGET_SIZES_JSON}" \
|
||||
'([.[] | select(.id == "staging-gce-c2")] | length == 1) and
|
||||
([.[] | select(.id == "staging-gce-c3")] | length == 1) and
|
||||
(any(.[]; .id == "staging-gce-c2" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and
|
||||
(any(.[]; .id == "staging-gce-c3" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and
|
||||
($images["staging-gce-c2"] == $images["staging-gce-c3"]) and
|
||||
($target_sizes["staging-gce-c2"] == 1) and
|
||||
($target_sizes["staging-gce-c3"] == 1)' \
|
||||
<<< "${DESIRED_CELLS_JSON}" >/dev/null
|
||||
jq \
|
||||
--argjson desired "${DESIRED_CELLS_JSON}" \
|
||||
--argjson images "${DESIRED_IMAGES_JSON}" \
|
||||
--argjson zones "${DESIRED_ZONES_JSON}" \
|
||||
'($desired | map({key: .id, value: .}) | from_entries) as $cells |
|
||||
to_entries | map(. as $entry | {
|
||||
key: $entry.key,
|
||||
value: ($entry.value + {
|
||||
origin: $cells[$entry.key].url,
|
||||
image: $images[$entry.key],
|
||||
zone: $zones[$entry.key],
|
||||
connection_hard_cap: $cells[$entry.key].connectionHardCap,
|
||||
connection_unobserved_bound: $cells[$entry.key].connectionUnobservedBound
|
||||
})
|
||||
}) | from_entries' \
|
||||
"${RUNNER_TEMP}/relay-gce-state.json" \
|
||||
> "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
ACTIVE_REVISION="$(gcloud run services describe orca-cloud-relay-staging \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region us-central1 \
|
||||
--format=json \
|
||||
| jq -r '
|
||||
[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test -n "${ACTIVE_REVISION}"
|
||||
DESIRED_IMAGE="$(jq -r '.["staging-gce-c2"]' <<< "${DESIRED_IMAGES_JSON}")"
|
||||
gcloud run revisions describe "${ACTIVE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region us-central1 \
|
||||
--format=json \
|
||||
| jq -e \
|
||||
--arg image "${DESIRED_IMAGE}" \
|
||||
--arg capacity_service_account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
'(.spec.containers[0].image == $image) and
|
||||
any(.spec.containers[0].env[]?;
|
||||
.name == "ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT" and
|
||||
.value == $capacity_service_account)' >/dev/null
|
||||
CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format=json \
|
||||
| jq -cer '
|
||||
[.spec.containers[0].env[]? |
|
||||
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
|
||||
if length == 1 then .[0] | fromjson else error("missing director topology") end')"
|
||||
jq -e --argjson desired "${DESIRED_CELLS_JSON}" '
|
||||
def without_bootstrap_capacity:
|
||||
map(if .id == "staging-gce-c2" or .id == "staging-gce-c3"
|
||||
then del(.connectionHardCap, .connectionUnobservedBound)
|
||||
else . end);
|
||||
without_bootstrap_capacity == ($desired | without_bootstrap_capacity)
|
||||
' <<< "${CURRENT_CELLS_JSON}" >/dev/null
|
||||
|
||||
- name: Bootstrap C2 then C3
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
topology="${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
|
||||
fixed_one_instance_name() {
|
||||
local cell_id="$1"
|
||||
local mig_name zone
|
||||
mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")"
|
||||
zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
|
||||
gcloud compute instance-groups managed list-instances \
|
||||
"${mig_name}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${zone}" \
|
||||
--format=json \
|
||||
| jq -er '
|
||||
if length == 1 and .[0].instanceStatus == "RUNNING" and
|
||||
.[0].currentAction == "NONE"
|
||||
then .[0].instance | split("/") | last
|
||||
else error("legacy cell does not have one stable running instance") end'
|
||||
}
|
||||
|
||||
fixed_one_instance_id() {
|
||||
local cell_id="$1"
|
||||
local instance_name="$2"
|
||||
local zone
|
||||
zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
|
||||
gcloud compute instances describe "${instance_name}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${zone}" \
|
||||
--format='value(id)'
|
||||
}
|
||||
|
||||
write_legacy_metrics() {
|
||||
local cell_id="$1"
|
||||
local instance_id="$2"
|
||||
local after="$3"
|
||||
local output="$4"
|
||||
gcloud logging read \
|
||||
"resource.type=\"gce_instance\" AND
|
||||
resource.labels.instance_id=\"${instance_id}\" AND
|
||||
jsonPayload.event=\"orca_relay_runtime_metrics\" AND
|
||||
jsonPayload.cellId=\"${cell_id}\" AND
|
||||
timestamp>=\"${after}\"" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--limit 10 \
|
||||
--order desc \
|
||||
--format json \
|
||||
| jq '[.[] | {
|
||||
timestamp,
|
||||
cellId: .jsonPayload.cellId,
|
||||
metricVersion: .jsonPayload.metricVersion,
|
||||
totalConnections: .jsonPayload.totalConnections,
|
||||
preAuthConnections: .jsonPayload.preAuthConnections,
|
||||
controls: .jsonPayload.controls,
|
||||
splices: .jsonPayload.splices,
|
||||
pendingSplices: .jsonPayload.pendingSplices,
|
||||
queuedBytes: .jsonPayload.queuedBytes
|
||||
}]' > "${output}"
|
||||
}
|
||||
|
||||
verify_legacy_cell() {
|
||||
local cell_id="$1"
|
||||
local admission="$2"
|
||||
local after="$3"
|
||||
local expected_instance_id="$4"
|
||||
local runtime_started_after="${5:-}"
|
||||
local previous_incarnation_digest="${6:-}"
|
||||
local hard_cap="${7:-}"
|
||||
local unobserved_bound="${8:-}"
|
||||
local capacity_state="${9:-}"
|
||||
local current_instance current_instance_id metrics origin result
|
||||
local runtime_start_args=() incarnation_args=() capacity_args=()
|
||||
current_instance="$(fixed_one_instance_name "${cell_id}")"
|
||||
current_instance_id="$(fixed_one_instance_id "${cell_id}" "${current_instance}")"
|
||||
test "${current_instance_id}" = "${expected_instance_id}"
|
||||
metrics="${RUNNER_TEMP}/${cell_id}-legacy-runtime-metrics.json"
|
||||
origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
|
||||
if test -n "${runtime_started_after}"; then
|
||||
runtime_start_args=(--runtime-started-after "${runtime_started_after}")
|
||||
fi
|
||||
if test -n "${previous_incarnation_digest}"; then
|
||||
incarnation_args=(--previous-incarnation-digest "${previous_incarnation_digest}")
|
||||
fi
|
||||
if test -n "${hard_cap}"; then
|
||||
capacity_args=(--hard-cap "${hard_cap}" --unobserved-bound "${unobserved_bound}")
|
||||
fi
|
||||
if test -n "${capacity_state}"; then
|
||||
capacity_args+=(--capacity-state "${capacity_state}")
|
||||
fi
|
||||
for _attempt in $(seq 1 18); do
|
||||
write_legacy_metrics \
|
||||
"${cell_id}" "${current_instance_id}" "${after}" "${metrics}"
|
||||
if result="$(node dev/scripts/verify-relay-legacy-bootstrap.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--admission "${admission}" \
|
||||
--expected-image-digest "${LEGACY_C3_IMAGE_DIGEST}" \
|
||||
--metrics-after "${after}" \
|
||||
--metrics-file "${metrics}" \
|
||||
"${runtime_start_args[@]}" \
|
||||
"${incarnation_args[@]}" \
|
||||
"${capacity_args[@]}")"; then
|
||||
echo "${result}"
|
||||
return 0
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
post_admin() {
|
||||
local origin="$1"
|
||||
local path="$2"
|
||||
local body="$3"
|
||||
curl --fail --silent --show-error \
|
||||
--request POST \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "${body}" \
|
||||
"${origin}${path}"
|
||||
}
|
||||
|
||||
runtime_kind() {
|
||||
local cell_id="$1"
|
||||
local origin desired_digest digest
|
||||
origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
|
||||
desired_digest="$(jq -r --arg cell "${cell_id}" \
|
||||
'.[$cell].image | split("@") | last' "${topology}")"
|
||||
digest="$(post_admin "${origin}" /v1/admin/runtime-status '{"v":1}' \
|
||||
| jq -er '.imageDigest')"
|
||||
if test "${digest}" = "${LEGACY_C3_IMAGE_DIGEST}"; then
|
||||
echo legacy
|
||||
elif test "${digest}" = "${desired_digest}"; then
|
||||
echo modern
|
||||
else
|
||||
echo 'bootstrap cell image is neither legacy nor reviewed' >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
cell_admission() {
|
||||
local cell_id="$1"
|
||||
post_admin "${DIRECTOR_ORIGIN}" /v1/admin/cell-status \
|
||||
"$(jq -cn --arg cell "${cell_id}" '{v:1, cellId:$cell}')" \
|
||||
| jq -er '.status.admissionState |
|
||||
if . == "general" or . == "migration-only" then .
|
||||
else error("bootstrap admission is not recoverable") end'
|
||||
}
|
||||
|
||||
verify_modern_cell() {
|
||||
local cell_id="$1"
|
||||
local admission="$2"
|
||||
local origin
|
||||
origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission "${admission}" \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
}
|
||||
|
||||
ensure_modern_general() {
|
||||
local cell_id="$1"
|
||||
local admission="$2"
|
||||
verify_modern_cell "${cell_id}" "${admission}"
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--mode restore \
|
||||
--general-cell-ids "${cell_id}"
|
||||
verify_modern_cell "${cell_id}" general
|
||||
}
|
||||
|
||||
prepare_legacy_c3_fallback() {
|
||||
legacy_c3_metrics_boundary="$(node -e \
|
||||
'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')"
|
||||
legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
|
||||
legacy_c3_instance_id="$(fixed_one_instance_id \
|
||||
staging-gce-c3 "${legacy_c3_instance}")"
|
||||
verify_legacy_cell \
|
||||
staging-gce-c3 general \
|
||||
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}"
|
||||
node dev/scripts/probe-relay-legacy-admission.mjs \
|
||||
--cell-origin https://c3.relay-staging.onorca.dev
|
||||
legacy_c3_restart_started_after=
|
||||
legacy_c3_old_incarnation=
|
||||
}
|
||||
|
||||
normalize_legacy_c3() {
|
||||
legacy_pre_boundary="$(node -e \
|
||||
'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')"
|
||||
legacy_c2_instance="$(fixed_one_instance_name staging-gce-c2)"
|
||||
legacy_c2_instance_id="$(fixed_one_instance_id \
|
||||
staging-gce-c2 "${legacy_c2_instance}")"
|
||||
verify_legacy_cell \
|
||||
staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}"
|
||||
node dev/scripts/probe-relay-legacy-admission.mjs \
|
||||
--cell-origin https://c2.relay-staging.onorca.dev
|
||||
|
||||
legacy_c3_isolated=false
|
||||
restore_legacy_c3_fallback() {
|
||||
if test "${legacy_c3_isolated}" = true; then
|
||||
verify_legacy_cell \
|
||||
staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}"
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id staging-gce-c3 \
|
||||
--mode restore-fallback \
|
||||
--general-cell-ids staging-gce-c2
|
||||
fi
|
||||
}
|
||||
|
||||
trap restore_legacy_c3_fallback EXIT
|
||||
legacy_c3_isolated=true
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin https://c3.relay-staging.onorca.dev \
|
||||
--cell-id staging-gce-c3 \
|
||||
--mode isolate
|
||||
legacy_c3_drain_boundary="$(node -e \
|
||||
'process.stdout.write(new Date().toISOString())')"
|
||||
legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
|
||||
legacy_c3_instance_id="$(fixed_one_instance_id \
|
||||
staging-gce-c3 "${legacy_c3_instance}")"
|
||||
legacy_c3_drained="$(verify_legacy_cell \
|
||||
staging-gce-c3 migration-only \
|
||||
"${legacy_c3_drain_boundary}" "${legacy_c3_instance_id}")"
|
||||
echo "${legacy_c3_drained}"
|
||||
legacy_c3_old_incarnation="$(jq -er '.incarnationDigest' \
|
||||
<<< "${legacy_c3_drained}")"
|
||||
legacy_c3_restart_started_after="$(node -e \
|
||||
'process.stdout.write(new Date().toISOString())')"
|
||||
gcloud compute instance-groups managed recreate-instances \
|
||||
orca-cloud-staging-relay-gce-c3 \
|
||||
--instances "${legacy_c3_instance}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone us-central1-a \
|
||||
--quiet
|
||||
gcloud compute instance-groups managed wait-until \
|
||||
orca-cloud-staging-relay-gce-c3 \
|
||||
--stable \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone us-central1-a \
|
||||
--timeout 900
|
||||
legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
|
||||
legacy_c3_instance_id="$(fixed_one_instance_id \
|
||||
staging-gce-c3 "${legacy_c3_instance}")"
|
||||
legacy_c3_metrics_boundary="$(node -e \
|
||||
'process.stdout.write(new Date().toISOString())')"
|
||||
verify_legacy_cell \
|
||||
staging-gce-c3 migration-only \
|
||||
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
|
||||
"${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id staging-gce-c3 \
|
||||
--mode restore \
|
||||
--general-cell-ids staging-gce-c2,staging-gce-c3
|
||||
verify_legacy_cell \
|
||||
staging-gce-c3 general \
|
||||
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
|
||||
"${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
|
||||
legacy_c3_isolated=false
|
||||
trap - EXIT
|
||||
}
|
||||
|
||||
roll_cell() (
|
||||
local cell_id="$1"
|
||||
local fallback_cell_id="$2"
|
||||
local target_kind="$3"
|
||||
local fallback_kind="$4"
|
||||
local active_revision cell_origin current_cells_json desired_bound desired_cap
|
||||
local desired_cells_json director_result image mig_name plan
|
||||
local fallback_origin plan_changes plan_result restored target_drain_boundary
|
||||
local target_instance target_instance_id zone
|
||||
cell_origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
|
||||
fallback_origin="$(jq -r --arg cell "${fallback_cell_id}" \
|
||||
'.[$cell].origin' "${topology}")"
|
||||
zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
|
||||
mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")"
|
||||
image="$(jq -r --arg cell "${cell_id}" '.[$cell].image' "${topology}")"
|
||||
desired_cap="$(jq -r --arg cell "${cell_id}" \
|
||||
'.[$cell].connection_hard_cap' "${topology}")"
|
||||
desired_bound="$(jq -r --arg cell "${cell_id}" \
|
||||
'.[$cell].connection_unobserved_bound' "${topology}")"
|
||||
plan="${RUNNER_TEMP}/${cell_id}-capacity-bootstrap.tfplan"
|
||||
restored=false
|
||||
|
||||
restore_fallback() {
|
||||
if test "${restored}" = false; then
|
||||
verify_fallback
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--mode restore-fallback \
|
||||
--general-cell-ids "${fallback_cell_id}"
|
||||
fi
|
||||
}
|
||||
|
||||
verify_fallback() {
|
||||
if test "${fallback_kind}" = legacy; then
|
||||
verify_legacy_cell \
|
||||
"${fallback_cell_id}" general \
|
||||
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
|
||||
"${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
|
||||
return
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${fallback_origin}" \
|
||||
--cell-id "${fallback_cell_id}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
}
|
||||
|
||||
verify_fallback
|
||||
trap restore_fallback EXIT
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${cell_origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--mode isolate
|
||||
target_drain_boundary="$(node -e \
|
||||
'process.stdout.write(new Date().toISOString())')"
|
||||
if test "${target_kind}" = legacy; then
|
||||
target_instance="$(fixed_one_instance_name "${cell_id}")"
|
||||
target_instance_id="$(fixed_one_instance_id "${cell_id}" "${target_instance}")"
|
||||
verify_legacy_cell \
|
||||
"${cell_id}" migration-only \
|
||||
"${target_drain_boundary}" "${target_instance_id}" \
|
||||
'' '' "${desired_cap}" "${desired_bound}" absent-or-stale
|
||||
else
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${cell_origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity quiescent
|
||||
fi
|
||||
|
||||
active_revision="$(gcloud run services describe orca-cloud-relay-staging \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format=json \
|
||||
| jq -r '
|
||||
[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test -n "${active_revision}"
|
||||
current_cells_json="$(gcloud run revisions describe "${active_revision}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format=json \
|
||||
| jq -cer '
|
||||
[.spec.containers[0].env[]? |
|
||||
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
|
||||
if length == 1 then .[0] | fromjson else error("missing director topology") end')"
|
||||
desired_cells_json="$(jq -ce \
|
||||
--arg cell "${cell_id}" \
|
||||
--argjson cap "${desired_cap}" \
|
||||
--argjson bound "${desired_bound}" \
|
||||
'map(if .id == $cell then . + {
|
||||
connectionHardCap: $cap,
|
||||
connectionUnobservedBound: $bound
|
||||
} else . end)' <<< "${current_cells_json}")"
|
||||
director_result="$(node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service orca-cloud-relay-staging \
|
||||
--image "${image}" \
|
||||
--role director \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--capacity-cell-id "${cell_id}" \
|
||||
--director-cells-json "${desired_cells_json}" \
|
||||
--prune-revisions true \
|
||||
--release-id "bootstrap-${cell_id}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}")"
|
||||
echo "${director_result}"
|
||||
if test "${target_kind}" = legacy; then
|
||||
verify_legacy_cell \
|
||||
"${cell_id}" migration-only \
|
||||
"${target_drain_boundary}" "${target_instance_id}" \
|
||||
'' '' "${desired_cap}" "${desired_bound}"
|
||||
else
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${cell_origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--hard-cap "${desired_cap}" \
|
||||
--unobserved-bound "${desired_bound}" \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity quiescent
|
||||
fi
|
||||
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
"-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \
|
||||
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \
|
||||
-out="${plan}"
|
||||
plan_result="$(terraform -chdir=infra/terraform show -json "${plan}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode bootstrap-cell \
|
||||
--cell-id "${cell_id}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--image "${image}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
|
||||
echo "${plan_result}"
|
||||
plan_changes="$(jq -r '.changes' <<< "${plan_result}")"
|
||||
[[ "${plan_changes}" =~ ^(0|2)$ ]]
|
||||
if test "${plan_changes}" = 2; then
|
||||
terraform -chdir=infra/terraform apply -auto-approve "${plan}"
|
||||
else
|
||||
target_instance="$(fixed_one_instance_name "${cell_id}")"
|
||||
gcloud compute instance-groups managed recreate-instances "${mig_name}" \
|
||||
--instances "${target_instance}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${zone}" \
|
||||
--quiet
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until "${mig_name}" \
|
||||
--stable \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${zone}" \
|
||||
--timeout 900
|
||||
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${cell_origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--mode restore \
|
||||
--general-cell-ids staging-gce-c2,staging-gce-c3
|
||||
restored=true
|
||||
trap - EXIT
|
||||
)
|
||||
|
||||
c2_kind="$(runtime_kind staging-gce-c2)"
|
||||
c3_kind="$(runtime_kind staging-gce-c3)"
|
||||
c2_admission="$(cell_admission staging-gce-c2)"
|
||||
c3_admission="$(cell_admission staging-gce-c3)"
|
||||
bootstrap_phase="$(node dev/scripts/classify-relay-staging-bootstrap.mjs \
|
||||
--c2-kind "${c2_kind}" \
|
||||
--c2-admission "${c2_admission}" \
|
||||
--c3-kind "${c3_kind}" \
|
||||
--c3-admission "${c3_admission}")"
|
||||
jq -cn --arg phase "${bootstrap_phase}" \
|
||||
'{event:"relay_staging_bootstrap_phase", phase:$phase}'
|
||||
|
||||
case "${bootstrap_phase}" in
|
||||
normalize-and-roll-both)
|
||||
normalize_legacy_c3
|
||||
roll_cell staging-gce-c2 staging-gce-c3 legacy legacy
|
||||
roll_cell staging-gce-c3 staging-gce-c2 legacy modern
|
||||
;;
|
||||
resume-c2-then-c3)
|
||||
prepare_legacy_c3_fallback
|
||||
roll_cell staging-gce-c2 staging-gce-c3 legacy legacy
|
||||
roll_cell staging-gce-c3 staging-gce-c2 legacy modern
|
||||
;;
|
||||
roll-c2)
|
||||
ensure_modern_general staging-gce-c3 "${c3_admission}"
|
||||
roll_cell staging-gce-c2 staging-gce-c3 legacy modern
|
||||
;;
|
||||
roll-c3)
|
||||
ensure_modern_general staging-gce-c2 "${c2_admission}"
|
||||
roll_cell staging-gce-c3 staging-gce-c2 legacy modern
|
||||
;;
|
||||
complete)
|
||||
ensure_modern_general staging-gce-c2 "${c2_admission}"
|
||||
ensure_modern_general staging-gce-c3 "${c3_admission}"
|
||||
;;
|
||||
*)
|
||||
echo 'unsupported staging bootstrap phase' >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Verify both bootstrapped cells
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
for number in 2 3; do
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "https://c${number}.relay-staging.onorca.dev" \
|
||||
--cell-id "staging-gce-c${number}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
done
|
||||
@@ -1,288 +0,0 @@
|
||||
name: Deploy Relay Asia Topology
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
environment:
|
||||
description: Target Relay environment
|
||||
required: true
|
||||
type: choice
|
||||
options: [staging, production]
|
||||
mode:
|
||||
description: Validate a saved plan or apply that exact plan
|
||||
required: true
|
||||
default: plan
|
||||
type: choice
|
||||
options: [plan, apply]
|
||||
cell-ids:
|
||||
description: Exact reviewed comma-separated cell set (Asia, or the US cells at the Asia shape)
|
||||
required: true
|
||||
type: string
|
||||
image:
|
||||
description: Full environment Relay image pinned by sha256 digest
|
||||
required: true
|
||||
type: string
|
||||
confirmation:
|
||||
description: Enter APPLY_RELAY_ASIA_TOPOLOGY for apply mode
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
topology:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 30
|
||||
environment: ${{ inputs.environment }}
|
||||
env:
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
TARGET_ENVIRONMENT: ${{ inputs.environment }}
|
||||
TARGET_CELL_IDS: ${{ inputs.cell-ids }}
|
||||
TARGET_IMAGE: ${{ inputs.image }}
|
||||
GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }}
|
||||
CLOUD_SQL_INSTANCE: ${{ inputs.environment == 'production' && 'orca-cloud-auth-db' || 'orca-cloud-staging-auth-db' }}
|
||||
VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360
|
||||
VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17
|
||||
# SHOW max_connections on the live instance, 2026-09-16; no flag is set.
|
||||
VERIFIED_DEFAULT_MAX_CONNECTIONS: '500'
|
||||
TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }}
|
||||
TF_VARS: ${{ inputs.environment == 'production' && 'environments/production.tfvars' || 'environments/staging.tfvars' }}
|
||||
TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
TOPOLOGY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Validate the reviewed request before authentication
|
||||
shell: bash
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -n "${TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${TOPOLOGY_SERVICE_ACCOUNT}"
|
||||
target_region=asia-east2
|
||||
case "${TARGET_ENVIRONMENT}:${TARGET_CELL_IDS}" in
|
||||
staging:staging-gce-c4) ;;
|
||||
production:production-gce-c27,production-gce-c28,production-gce-c29) ;;
|
||||
production:production-gce-c30) ;;
|
||||
production:production-gce-c31) ;;
|
||||
production:production-gce-c32,production-gce-c33) target_region=us-central1 ;;
|
||||
production:production-gce-c34) ;;
|
||||
production:production-gce-c35) ;;
|
||||
*) echo "cell-ids do not match the reviewed environment topology" >&2; exit 1 ;;
|
||||
esac
|
||||
echo "TARGET_REGION=${target_region}" >> "${GITHUB_ENV}"
|
||||
[[ "${TARGET_IMAGE}" =~ ^us-central1-docker\.pkg\.dev/${GCP_PROJECT_ID}/orca-cloud/relay@sha256:[0-9a-f]{64}$ ]]
|
||||
if test "${DEPLOY_MODE}" = apply; then
|
||||
test "${CONFIRMATION}" = APPLY_RELAY_ASIA_TOPOLOGY
|
||||
else
|
||||
test "${DEPLOY_MODE}" = plan
|
||||
test -z "${CONFIRMATION}"
|
||||
fi
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ env.TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ env.TOPOLOGY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }}
|
||||
object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }}
|
||||
|
||||
- name: Require the checked Cloud SQL connection budget
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
budget="$(node dev/scripts/relay-cloud-sql-connection-budget.mjs)"
|
||||
checked_max="$(jq -er '.maxConnections' <<< "${budget}")"
|
||||
jq -e '.withinBudget == true' <<< "${budget}" >/dev/null
|
||||
if test "${TARGET_ENVIRONMENT}" = production; then
|
||||
instance="$(gcloud sql instances describe "${CLOUD_SQL_INSTANCE}" \
|
||||
--project "${GCP_PROJECT_ID}" --format=json)"
|
||||
live_flag="$(jq -er '[.settings.databaseFlags[]? |
|
||||
select(.name == "max_connections") | .value] |
|
||||
if length <= 1 then (.[0] // "") else error("duplicate max_connections flags") end' \
|
||||
<<< "${instance}")"
|
||||
if test -n "${live_flag}"; then
|
||||
live_max="${live_flag}"
|
||||
live_source=explicit-flag
|
||||
else
|
||||
# No flag: the ceiling is the tier default measured for this exact
|
||||
# shape; fail closed if the shape changes.
|
||||
test "$(jq -er '.settings.tier' <<< "${instance}")" = \
|
||||
"${VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER}"
|
||||
test "$(jq -er '.databaseVersion' <<< "${instance}")" = \
|
||||
"${VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION}"
|
||||
live_max="${VERIFIED_DEFAULT_MAX_CONNECTIONS}"
|
||||
live_source=verified-shape-default
|
||||
fi
|
||||
test "${live_max}" = "${checked_max}"
|
||||
else
|
||||
live_max="not-read-for-staging"
|
||||
live_source=not-read-for-staging
|
||||
fi
|
||||
{
|
||||
echo "### Relay Cloud SQL connection budget"
|
||||
echo "- Checked maximum: ${checked_max}"
|
||||
echo "- Configured maximum: $(jq -er '.configuredMaximum' <<< "${budget}")"
|
||||
echo "- Rollout operating maximum: $(jq -er '.operatingMaximum' <<< "${budget}")"
|
||||
echo "- Explicit reserve: $(jq -er '.explicitReserve' <<< "${budget}")"
|
||||
echo "- Production live max_connections: ${live_max}"
|
||||
echo "- Production live maximum source: ${live_source}"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Initialize the exact environment state
|
||||
run: terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}"
|
||||
|
||||
- id: targets
|
||||
name: Build the exact additive target set
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
file="${RUNNER_TEMP}/relay-asia-targets"
|
||||
: > "${file}"
|
||||
# The root region's network predates every cell; only asia-east2's is additive.
|
||||
if test "${TARGET_REGION}" = asia-east2; then
|
||||
printf '%s\n' \
|
||||
'-target=google_compute_subnetwork.relay_gce_additional["asia-east2"]' \
|
||||
'-target=google_compute_router.relay_gce_additional["asia-east2"]' \
|
||||
'-target=google_compute_router_nat.relay_gce_additional["asia-east2"]' >> "${file}"
|
||||
fi
|
||||
printf '%s\n' '-target=google_compute_url_map.relay_gce[0]' >> "${file}"
|
||||
IFS=, read -ra cells <<< "${TARGET_CELL_IDS}"
|
||||
for cell_id in "${cells[@]}"; do
|
||||
printf '%s\n' \
|
||||
"-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \
|
||||
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \
|
||||
"-target=google_compute_backend_service.relay_gce_cell[\"${cell_id}\"]" >> "${file}"
|
||||
done
|
||||
echo "file=${file}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- id: live-images
|
||||
name: Plan every non-target cell at the image it serves
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# The URL map target pulls every cell's template into the plan, and same-cap rolls
|
||||
# leave committed images behind the served ones; read only templates out of state.
|
||||
cells="${RUNNER_TEMP}/relay-committed-cells.json"
|
||||
live="${RUNNER_TEMP}/relay-live-cell-templates.json"
|
||||
overlay="${RUNNER_TEMP}/relay-asia-live-images.tfvars.json"
|
||||
committed_plan="${RUNNER_TEMP}/relay-committed-cells.tfplan"
|
||||
mapfile -t targets < "${{ steps.targets.outputs.file }}"
|
||||
# Not console: it evaluates every output against state, where a new cell has no MIG yet.
|
||||
terraform -chdir=infra/terraform plan -input=false -refresh=false -lock=false \
|
||||
-var-file="${TF_VARS}" "${targets[@]}" -out="${committed_plan}" > /dev/null
|
||||
terraform -chdir=infra/terraform show -json "${committed_plan}" \
|
||||
| jq -ce '.variables.relay_gce_cells.value | objects' > "${cells}"
|
||||
terraform -chdir=infra/terraform show -json | jq -ce '[
|
||||
.values.root_module.resources[]?
|
||||
| select(.mode == "managed" and
|
||||
.type == "google_compute_instance_template" and .name == "relay_gce_cell")
|
||||
| { index, metadata_startup_script: .values.metadata_startup_script }
|
||||
]' > "${live}"
|
||||
summary="$(node dev/scripts/relay-live-cell-image-overlay.mjs \
|
||||
--cells-json "${cells}" --live-templates-json "${live}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" --output "${overlay}")"
|
||||
echo "file=${overlay}" >> "${GITHUB_OUTPUT}"
|
||||
{
|
||||
echo "### Live images held for non-target cells"
|
||||
echo "- Cells whose committed image differs from the served one: $(jq -r '.drifted | join(", ")' <<< "${summary}")"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Create and validate the saved topology plan
|
||||
id: plan
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
plan="${RUNNER_TEMP}/relay-asia-topology.tfplan"
|
||||
plan_json="${RUNNER_TEMP}/relay-asia-topology.json"
|
||||
mapfile -t targets < "${{ steps.targets.outputs.file }}"
|
||||
terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \
|
||||
-var-file="${TF_VARS}" -var-file="${{ steps.live-images.outputs.file }}" \
|
||||
"${targets[@]}" -out="${plan}"
|
||||
terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}"
|
||||
committed="${RUNNER_TEMP}/relay-committed-asia-topology.json"
|
||||
jq -e '{
|
||||
relay_gce_cells: .variables.relay_gce_cells.value,
|
||||
relay_gce_additional_region_subnetwork_cidrs:
|
||||
.variables.relay_gce_additional_region_subnetwork_cidrs.value
|
||||
}' "${plan_json}" > "${committed}"
|
||||
node dev/scripts/prepare-relay-asia-topology-input.mjs \
|
||||
--existing-json "${committed}" \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--image "${TARGET_IMAGE}"
|
||||
result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \
|
||||
--plan-json "${plan_json}" \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--region "${TARGET_REGION}" \
|
||||
--image "${TARGET_IMAGE}")"
|
||||
changes="$(jq -er '.changes' <<< "${result}")"
|
||||
digest="$(sha256sum "${plan}" | awk '{print $1}')"
|
||||
echo "plan=${plan}" >> "${GITHUB_OUTPUT}"
|
||||
echo "changes=${changes}" >> "${GITHUB_OUTPUT}"
|
||||
{
|
||||
echo "### Relay Asia topology saved plan"
|
||||
echo "- Environment: ${TARGET_ENVIRONMENT}"
|
||||
echo "- Cells: ${TARGET_CELL_IDS}"
|
||||
echo "- Region: ${TARGET_REGION}"
|
||||
echo "- Mutating resources: ${changes}"
|
||||
echo "- Saved-plan SHA-256: ${digest}"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Apply only the validated saved plan
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
run: terraform -chdir=infra/terraform apply -input=false -auto-approve "${{ steps.plan.outputs.plan }}"
|
||||
|
||||
- name: Prove the exact topology targets converged
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mapfile -t targets < "${{ steps.targets.outputs.file }}"
|
||||
plan="${RUNNER_TEMP}/relay-asia-topology-readback.tfplan"
|
||||
plan_json="${RUNNER_TEMP}/relay-asia-topology-readback.json"
|
||||
terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \
|
||||
-var-file="${TF_VARS}" -var-file="${{ steps.live-images.outputs.file }}" \
|
||||
"${targets[@]}" -out="${plan}"
|
||||
terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}"
|
||||
result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \
|
||||
--plan-json "${plan_json}" \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--region "${TARGET_REGION}" \
|
||||
--image "${TARGET_IMAGE}")"
|
||||
test "$(jq -er '.changes' <<< "${result}")" = 0
|
||||
|
||||
- name: Record the required selector-safe next step
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
run: |
|
||||
{
|
||||
echo "### Required next step"
|
||||
echo "The VMs are not eligible for ordinary placement yet."
|
||||
echo "Register the exact new cells atomically as migration-only before any director configuration lists them."
|
||||
echo "Rollback is migration-only admission; do not destroy the Asia network on rollout day."
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,92 +0,0 @@
|
||||
name: Deploy Relay Fence Broker
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
image-digest:
|
||||
description: Immutable broker image digest built from this main commit
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: >-
|
||||
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
|
||||
github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
SERVICE_NAME: orca-cloud-relay-fence
|
||||
IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay-fence-broker
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
|
||||
- name: Resolve exact-commit broker image
|
||||
run: |
|
||||
[[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
|
||||
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \
|
||||
--format='value(image_summary.digest)')"
|
||||
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
|
||||
TAGS="$(gcloud artifacts docker tags list "${IMAGE_REPOSITORY}" \
|
||||
--filter="version:${IMAGE_DIGEST}" \
|
||||
--format=json)"
|
||||
jq -e --arg tag "/tags/sha-${GITHUB_SHA}" \
|
||||
'any(.[]; .tag | endswith($tag))' <<< "${TAGS}"
|
||||
echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Deploy broker image only
|
||||
run: |
|
||||
gcloud run services update "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--image "${IMAGE}" \
|
||||
--quiet
|
||||
|
||||
- name: Verify ready singleton revision
|
||||
run: |
|
||||
SERVICE="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format=json)"
|
||||
jq -e '.status.conditions[] | select(.type == "Ready" and .status == "True")' \
|
||||
<<< "${SERVICE}"
|
||||
REVISION="$(jq -r \
|
||||
'[.status.traffic[] | select((.percent // 0) == 100)] |
|
||||
if length == 1 then .[0].revisionName // empty else empty end' \
|
||||
<<< "${SERVICE}")"
|
||||
test -n "${REVISION}"
|
||||
SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format='value(spec.containers[0].image)')"
|
||||
test "${SERVED_IMAGE}" = "${IMAGE}"
|
||||
@@ -1,825 +0,0 @@
|
||||
name: Deploy Relay Production Capacity Job
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
mode:
|
||||
required: true
|
||||
type: string
|
||||
target-cell-id:
|
||||
required: true
|
||||
type: string
|
||||
confirmation:
|
||||
required: true
|
||||
type: string
|
||||
monitor-run-id:
|
||||
required: true
|
||||
type: string
|
||||
monitor-run-attempt:
|
||||
required: true
|
||||
type: string
|
||||
evidence-mode:
|
||||
required: true
|
||||
type: string
|
||||
wave-cell-ids:
|
||||
required: true
|
||||
type: string
|
||||
wave-index:
|
||||
required: true
|
||||
type: string
|
||||
source-wave-run-id:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
capacity:
|
||||
if: ${{ github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 75
|
||||
environment: production
|
||||
env:
|
||||
# These drains are unpaced, but restart-safe still waits out the production pace window
|
||||
# from the drain, the same one the same-cap drain uses.
|
||||
RESTART_SAFE_PACE_WINDOW_MS: '300000'
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
DIRECTOR_SERVICE_NAME: orca-cloud-relay
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
|
||||
CAPACITY_CELL_IDS: production-gce-c7,production-gce-c8,production-gce-c9,production-gce-c10,production-gce-c13,production-gce-c14,production-gce-c15,production-gce-c16,production-gce-c19,production-gce-c20,production-gce-c21,production-gce-c22,production-gce-c23,production-gce-c24,production-gce-c25,production-gce-c26
|
||||
PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d
|
||||
COMPATIBLE_DIRECTOR_IMAGE_DIGEST: sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73
|
||||
COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
EVIDENCE_MODE: ${{ inputs.evidence-mode }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
||||
WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }}
|
||||
WAVE_INDEX: ${{ inputs.wave-index }}
|
||||
SOURCE_WAVE_RUN_ID: ${{ inputs.source-wave-run-id }}
|
||||
steps:
|
||||
- name: Require exact reusable-workflow invocation
|
||||
working-directory: .
|
||||
run: |
|
||||
[[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]]
|
||||
if test "${EVIDENCE_MODE}" = continuation; then
|
||||
test "${DEPLOY_MODE}" = apply
|
||||
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
|
||||
test "${WAVE_CELL_IDS}" != none
|
||||
test "${SOURCE_WAVE_RUN_ID}" = none
|
||||
elif test "${EVIDENCE_MODE}" = resume; then
|
||||
test "${DEPLOY_MODE}" = apply
|
||||
test "${WAVE_INDEX}" = resume
|
||||
test "${WAVE_CELL_IDS}" != none
|
||||
[[ "${SOURCE_WAVE_RUN_ID}" =~ ^[0-9]+$ ]]
|
||||
else
|
||||
test "${EVIDENCE_MODE}" = single
|
||||
test "${WAVE_CELL_IDS}" = none
|
||||
test "${WAVE_INDEX}" = 0
|
||||
test "${SOURCE_WAVE_RUN_ID}" = none
|
||||
fi
|
||||
|
||||
- name: Require production workflow configuration
|
||||
working-directory: .
|
||||
env:
|
||||
DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
CAPACITY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
run: |
|
||||
test -n "${GCP_REGION}"
|
||||
test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${DEPLOY_SERVICE_ACCOUNT}"
|
||||
test -n "${CAPACITY_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${CAPACITY_SERVICE_ACCOUNT}"
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: resume-provenance
|
||||
if: ${{ inputs.evidence-mode == 'resume' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
test "${MONITOR_RUN_ATTEMPT}" = 1
|
||||
case "${MONITOR_RUN_ID}:${SOURCE_WAVE_RUN_ID}:${WAVE_CELL_IDS}:${TARGET_CELL_ID}" in
|
||||
31554591366:31555510376:production-gce-c16,production-gce-c15,production-gce-c14,production-gce-c13:production-gce-c13)
|
||||
EXPECTED_SHA=a917e8e1fc1a2654e8cb81ba39b57733ec56be9c
|
||||
EXPECTED_SOURCE_ATTEMPT=1
|
||||
;;
|
||||
31562760783:31563664692:production-gce-c10,production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c10)
|
||||
EXPECTED_SHA=6082e9ca89a918ca51f0c87db003f5e8805b64b7
|
||||
EXPECTED_SOURCE_ATTEMPT=1
|
||||
;;
|
||||
31571019947:31572080665:production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c8)
|
||||
EXPECTED_SHA=e59958130c9d9b7a6cd805df2678d08997842c7c
|
||||
EXPECTED_SOURCE_ATTEMPT=2
|
||||
;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
MONITOR_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${MONITOR_RUN_ID}" \
|
||||
--jq 'select(.name == "Monitor Relay Production" and
|
||||
.path == ".github/workflows/cloud-monitor-relay-production.yml" and
|
||||
.head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and
|
||||
.event == "workflow_dispatch" and .conclusion == "success" and .run_attempt == 1) |
|
||||
.head_sha')"
|
||||
SOURCE_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \
|
||||
--jq 'select(.name == "Deploy Relay Production Capacity" and
|
||||
.path == ".github/workflows/cloud-deploy-relay-production-capacity.yml" and
|
||||
.head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and
|
||||
.event == "workflow_dispatch" and .conclusion == "failure") |
|
||||
.head_sha')"
|
||||
SOURCE_ATTEMPT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \
|
||||
--jq '.run_attempt')"
|
||||
[[ "${MONITOR_SHA}" =~ ^[0-9a-f]{40}$ ]]
|
||||
test "${MONITOR_SHA}" = "${EXPECTED_SHA}"
|
||||
test "${SOURCE_SHA}" = "${MONITOR_SHA}"
|
||||
test "${SOURCE_ATTEMPT}" = "${EXPECTED_SOURCE_ATTEMPT}"
|
||||
echo "commit-sha=${MONITOR_SHA}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Require fresh dry-run evidence reference
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
run: |
|
||||
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
|
||||
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
|
||||
- name: Download private dry-run evidence
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Verify dry-run artifact before cloud authentication
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
run: |
|
||||
EVIDENCE_COMMIT_SHA="${GITHUB_SHA}"
|
||||
if test "${EVIDENCE_MODE:-single}" = resume; then
|
||||
EVIDENCE_COMMIT_SHA="${{ steps.resume-provenance.outputs.commit-sha }}"
|
||||
fi
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${EVIDENCE_COMMIT_SHA}" \
|
||||
--mode dry-run
|
||||
|
||||
- name: Reject previously consumed dry-run evidence
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
COUNT="$(gh api \
|
||||
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
||||
--jq '.total_count')"
|
||||
test "${COUNT}" = "0"
|
||||
|
||||
- name: Download this workflow's wave authority
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-wave-authority
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ github.run_id }}
|
||||
|
||||
- name: Download the failed wave authority for resume
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-wave-authority
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.source-wave-run-id }}
|
||||
|
||||
- name: Require wave evidence consumed by this workflow
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${GITHUB_RUN_ID}"
|
||||
|
||||
- name: Require wave evidence consumed by the failed source workflow
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${SOURCE_WAVE_RUN_ID}"
|
||||
|
||||
- id: deploy-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
release: 'false'
|
||||
|
||||
- name: Require exact mutation confirmation
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if test "${EVIDENCE_MODE:-single}" = resume; then
|
||||
test "${CONFIRMATION}" = "RESUME_SELECTED_CELL_TO_1000 ${TARGET_CELL_ID}"
|
||||
elif test "${DEPLOY_MODE}" = apply; then
|
||||
test "${CONFIRMATION}" = "RAISE_SELECTED_CELL_TO_1000"
|
||||
else
|
||||
test "${CONFIRMATION}" = "ROLL_BACK_SELECTED_CELL_TO_600 ${TARGET_CELL_ID}"
|
||||
fi
|
||||
|
||||
- name: Initialize the exact production backend
|
||||
run: node dev/scripts/infra.mjs init --env production
|
||||
|
||||
- name: Build the exact selected-cell configuration
|
||||
shell: bash
|
||||
run: |
|
||||
if test "${DEPLOY_MODE}" = rollback; then
|
||||
TARGET_HARD_CAP=600
|
||||
else
|
||||
TARGET_HARD_CAP=1000
|
||||
fi
|
||||
TARGET_UNOBSERVED_BOUND=60
|
||||
TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"
|
||||
[[ "${TARGET_HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]]
|
||||
CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev"
|
||||
CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
|
||||
OVERRIDE_CELLS_JSON="$(jq -ce \
|
||||
--arg cell "${TARGET_CELL_ID}" \
|
||||
--argjson cap "${TARGET_HARD_CAP}" \
|
||||
--argjson bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
'.[$cell].connection_hard_cap = $cap |
|
||||
.[$cell].connection_unobserved_bound = $bound' \
|
||||
<<< "${CELLS_JSON}")"
|
||||
jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \
|
||||
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-capacity.tfvars.json"
|
||||
BASE_CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
<<< 'local.relay_director_cells_json' | jq -er '.')"
|
||||
IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image"
|
||||
ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone"
|
||||
DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
|
||||
<<< "${IMAGE_EXPRESSION}" | jq -r '.')"
|
||||
TARGET_ZONE="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
|
||||
<<< "${ZONE_EXPRESSION}" | jq -r '.')"
|
||||
MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
|
||||
| jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
|
||||
jq -e --arg cell "${TARGET_CELL_ID}" \
|
||||
'any(.[]; .id == $cell and .connectionHardCap == 1000 and
|
||||
.connectionUnobservedBound == 60)' \
|
||||
<<< "${BASE_CELLS_JSON}" >/dev/null
|
||||
[[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]]
|
||||
DESIRED_IMAGE_DIGEST="${DESIRED_IMAGE##*@}"
|
||||
[[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]]
|
||||
test "${MIG_NAME}" = "orca-cloud-relay-gce-${TARGET_HOSTNAME}"
|
||||
{
|
||||
echo "CELL_ORIGIN=${CELL_ORIGIN}"
|
||||
echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}"
|
||||
echo "TARGET_HARD_CAP=${TARGET_HARD_CAP}"
|
||||
echo "TARGET_UNOBSERVED_BOUND=${TARGET_UNOBSERVED_BOUND}"
|
||||
echo "DESIRED_IMAGE=${DESIRED_IMAGE}"
|
||||
echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}"
|
||||
echo "TARGET_ZONE=${TARGET_ZONE}"
|
||||
echo "MIG_NAME=${MIG_NAME}"
|
||||
echo "BASE_CELLS_JSON=${BASE_CELLS_JSON}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Require the exact compatible production image and topology
|
||||
shell: bash
|
||||
run: |
|
||||
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
ACTIVE_REVISION="$(jq -r \
|
||||
'[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \
|
||||
<<< "${SERVICE_JSON}")"
|
||||
test -n "${ACTIVE_REVISION}"
|
||||
ACTIVE_REVISION_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
ACTIVE_IMAGE="$(jq -er '.spec.containers[0].image' <<< "${ACTIVE_REVISION_JSON}")"
|
||||
ACTIVE_IMAGE_DIGEST="${ACTIVE_IMAGE##*@}"
|
||||
if test "${ACTIVE_IMAGE}" != "${DESIRED_IMAGE}"; then
|
||||
test "${ACTIVE_IMAGE_DIGEST}" = "${COMPATIBLE_DIRECTOR_IMAGE_DIGEST}"
|
||||
test "${DESIRED_IMAGE_DIGEST}" = "${COMPATIBLE_CELL_IMAGE_DIGEST}"
|
||||
fi
|
||||
CURRENT_CELLS_JSON="$(jq -cer '[.spec.containers[0].env[]? |
|
||||
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
|
||||
if length == 1 then .[0] | fromjson else error("missing director topology") end' \
|
||||
<<< "${ACTIVE_REVISION_JSON}")"
|
||||
CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON="$(
|
||||
node dev/scripts/read-relay-production-capacity-identity.mjs \
|
||||
<<< "${ACTIVE_REVISION_JSON}"
|
||||
)"
|
||||
CLASSIFICATION="$(jq -nc \
|
||||
--argjson baseCells "${BASE_CELLS_JSON}" \
|
||||
--argjson currentCells "${CURRENT_CELLS_JSON}" \
|
||||
--arg capacityCellIds "${CAPACITY_CELL_IDS}" \
|
||||
--arg targetCellId "${TARGET_CELL_ID}" \
|
||||
--argjson targetHardCap "${TARGET_HARD_CAP}" \
|
||||
--argjson currentCapacityServiceAccount \
|
||||
"${CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON}" \
|
||||
'{baseCells:$baseCells, currentCells:$currentCells,
|
||||
capacityCellIds:($capacityCellIds | split(",")),
|
||||
targetCellId:$targetCellId, targetHardCap:$targetHardCap,
|
||||
currentCapacityServiceAccount:$currentCapacityServiceAccount}' \
|
||||
| node dev/scripts/classify-relay-production-capacity-director.mjs \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
|
||||
TOPOLOGY_PHASE="$(jq -er '.topologyPhase' <<< "${CLASSIFICATION}")"
|
||||
DESIRED_CELLS_JSON="$(jq -cer '.desiredCells' <<< "${CLASSIFICATION}")"
|
||||
DIRECTOR_READY="$(jq -er \
|
||||
'if (.directorReady | type) == "boolean" then
|
||||
(.directorReady | tostring)
|
||||
else error("invalid directorReady classification") end' \
|
||||
<<< "${CLASSIFICATION}")"
|
||||
{
|
||||
echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}"
|
||||
echo "TOPOLOGY_PHASE=${TOPOLOGY_PHASE}"
|
||||
echo "DIRECTOR_READY=${DIRECTOR_READY}"
|
||||
echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Require the exact wave predecessor topology
|
||||
if: ${{ inputs.mode == 'apply' && (inputs.evidence-mode == 'continuation' || inputs.evidence-mode == 'resume') }}
|
||||
run: test "${TOPOLOGY_PHASE}" = predecessor
|
||||
|
||||
- name: Verify fresh dry-run evidence against the live selector
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run \
|
||||
--mutation-mode capacity-transition \
|
||||
--source-cell-id "${TARGET_CELL_ID}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}"
|
||||
|
||||
- name: Recheck every live safety signal
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
|
||||
|
||||
- name: Recheck exact wave state and every live safety signal
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/relay-production-capacity-wave.mjs build-preflight \
|
||||
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
|
||||
--wave-cell-ids "${WAVE_CELL_IDS}" \
|
||||
--wave-index "${WAVE_INDEX}" \
|
||||
--target-cell-id "${TARGET_CELL_ID}" \
|
||||
--output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json"
|
||||
RETRY_ARGS=()
|
||||
if test "${WAVE_INDEX}" != 0; then RETRY_ARGS=(--retry-freshness); fi
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \
|
||||
"${RETRY_ARGS[@]}"
|
||||
|
||||
- name: Recheck exact isolated resume state and every live safety signal
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/relay-production-capacity-wave.mjs build-resume-preflight \
|
||||
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
|
||||
--wave-cell-ids "${WAVE_CELL_IDS}" \
|
||||
--target-cell-id "${TARGET_CELL_ID}" \
|
||||
--output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json"
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \
|
||||
--retry-freshness
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity allowed \
|
||||
--runtime required \
|
||||
--expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}"
|
||||
|
||||
- name: Consume the single-use dry-run evidence
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
|
||||
printf '%s\n' "${GITHUB_RUN_ID}" \
|
||||
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
|
||||
|
||||
- name: Publish the consumed-evidence marker
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Verify current selected-cell capacity
|
||||
if: ${{ inputs.mode == 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
CURRENT_CAP="${TARGET_HARD_CAP}"
|
||||
CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
|
||||
if test "${TOPOLOGY_PHASE}" = predecessor; then
|
||||
CURRENT_CAP=600
|
||||
CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${CURRENT_CAP}" \
|
||||
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed \
|
||||
--expected-image-digests "${CURRENT_IMAGE_DIGEST}"
|
||||
|
||||
- name: Arm fail-closed mutation cleanup
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
run: echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Reversibly isolate only the selected cell
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
test "${MUTATION_STARTED:-false}" = true || exit 0
|
||||
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode isolate
|
||||
|
||||
- name: Drain the selected cell or prove an offline rollback
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
if node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode drain; then
|
||||
echo "OFFLINE_ROLLBACK=false" >> "${GITHUB_ENV}"
|
||||
elif test "${DEPLOY_MODE}" = rollback; then
|
||||
echo "OFFLINE_ROLLBACK=true" >> "${GITHUB_ENV}"
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- id: restart-auth-one
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: restart-gate-one
|
||||
name: Require restart-safe selected-cell activity
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-one.outputs.id_token }}
|
||||
run: |
|
||||
if test "${OFFLINE_ROLLBACK:-false}" = true; then
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--heartbeat stale \
|
||||
--admission migration-only \
|
||||
--draining either \
|
||||
--activity restart-safe \
|
||||
--runtime unavailable
|
||||
echo "settled=true" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
CURRENT_CAP=600
|
||||
if test "${TOPOLOGY_PHASE}" = desired; then
|
||||
CURRENT_CAP="${TARGET_HARD_CAP}"
|
||||
elif test "${TARGET_HARD_CAP}" = 600; then
|
||||
CURRENT_CAP=1000
|
||||
fi
|
||||
GATE_LOG="${RUNNER_TEMP}/relay-capacity-restart-gate-one.log"
|
||||
set +e
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${CURRENT_CAP}" \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--runtime required \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
|
||||
--timeout-ms 450000 \
|
||||
--expected-image-digests \
|
||||
"${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
2> "${GATE_LOG}"
|
||||
GATE_EXIT=$?
|
||||
set -e
|
||||
cat "${GATE_LOG}" >&2
|
||||
if test "${GATE_EXIT}" = 0; then
|
||||
echo "settled=true" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
if test "$(wc -l < "${GATE_LOG}" | tr -d ' ')" = 1 &&
|
||||
grep -Eq '^capacity transition verification timed out: \{.*\}$' "${GATE_LOG}"; then
|
||||
echo "settled=false" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
exit "${GATE_EXIT}"
|
||||
|
||||
- id: restart-auth-two
|
||||
if: ${{ steps.restart-gate-one.outputs.settled == 'false' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Require extended restart-safe selected-cell activity
|
||||
if: ${{ steps.restart-gate-one.outputs.settled == 'false' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-two.outputs.id_token }}
|
||||
run: |
|
||||
CURRENT_CAP=600
|
||||
if test "${TOPOLOGY_PHASE}" = desired; then
|
||||
CURRENT_CAP="${TARGET_HARD_CAP}"
|
||||
elif test "${TARGET_HARD_CAP}" = 600; then
|
||||
CURRENT_CAP=1000
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${CURRENT_CAP}" \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--runtime required \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
|
||||
--timeout-ms 450000 \
|
||||
--expected-image-digests \
|
||||
"${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
|
||||
|
||||
- name: Deploy only the reviewed director topology
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
run: |
|
||||
if test "${DIRECTOR_READY}" = true; then exit 0; fi
|
||||
RELEASE_ID="capacity-${TARGET_HOSTNAME}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
|
||||
node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service "${DIRECTOR_SERVICE_NAME}" \
|
||||
--image "${ACTIVE_IMAGE}" \
|
||||
--role director \
|
||||
--max-instances 5 \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--capacity-cell-id "${TARGET_CELL_ID}" \
|
||||
--director-cells-json "${DESIRED_CELLS_JSON}" \
|
||||
--min-instances 5 \
|
||||
--prune-revisions false \
|
||||
--release-id "${RELEASE_ID}"
|
||||
|
||||
- id: director-transition-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Require fail-closed director transition
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.director-transition-auth.outputs.id_token }}
|
||||
run: |
|
||||
if test "${OFFLINE_ROLLBACK:-false}" = true; then
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--heartbeat stale \
|
||||
--admission migration-only \
|
||||
--draining either \
|
||||
--activity restart-safe \
|
||||
--runtime unavailable
|
||||
exit 0
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${TARGET_HARD_CAP}" \
|
||||
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--runtime required \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
|
||||
--timeout-ms 480000 \
|
||||
--expected-image-digests \
|
||||
"${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
|
||||
|
||||
- id: capacity-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Plan and apply only the empty selected cell
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
shell: bash
|
||||
run: |
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/production.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
|
||||
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
-out="${RUNNER_TEMP}/relay-capacity-cell.tfplan"
|
||||
PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-capacity-cell.tfplan" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode bootstrap-cell \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${TARGET_HARD_CAP}" \
|
||||
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
--image "${DESIRED_IMAGE}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
|
||||
echo "${PLAN_RESULT}"
|
||||
PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")"
|
||||
[[ "${PLAN_CHANGES}" =~ ^(0|1|2)$ ]]
|
||||
if test "${PLAN_CHANGES}" != 0; then
|
||||
terraform -chdir=infra/terraform apply \
|
||||
-auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan"
|
||||
else
|
||||
INSTANCE="$(gcloud compute instance-groups managed list-instances \
|
||||
"${MIG_NAME}" --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
|
||||
--format=json | jq -er 'if length == 1 and
|
||||
.[0].instanceStatus == "RUNNING" and .[0].currentAction == "NONE"
|
||||
then .[0].instance | split("/") | last
|
||||
else error("selected cell is not one stable running instance") end')"
|
||||
gcloud compute instance-groups managed recreate-instances \
|
||||
"${MIG_NAME}" --instances "${INSTANCE}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --quiet
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until \
|
||||
"${MIG_NAME}" --stable --project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" --timeout 900
|
||||
|
||||
- id: capacity-transition-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Verify fresh exact selected-cell heartbeat before admission
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${TARGET_HARD_CAP}" \
|
||||
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining forbidden \
|
||||
--activity allowed \
|
||||
--expected-image-digests "${DESIRED_IMAGE_DIGEST}"
|
||||
|
||||
- name: Restore only the selected cell to general admission
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode activate
|
||||
|
||||
- name: Verify the live general selected cell
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${TARGET_HARD_CAP}" \
|
||||
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed \
|
||||
--expected-image-digests "${DESIRED_IMAGE_DIGEST}"
|
||||
|
||||
- id: cleanup-auth
|
||||
if: ${{ failure() && inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Keep the selected cell isolated after a failed mutation
|
||||
if: ${{ failure() && inputs.mode != 'verify' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }}
|
||||
run: |
|
||||
test "${MUTATION_STARTED:-false}" = true || exit 0
|
||||
CLEANUP_STATUS=0
|
||||
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode isolate || CLEANUP_STATUS=$?
|
||||
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode drain || CLEANUP_STATUS=$?
|
||||
exit "${CLEANUP_STATUS}"
|
||||
@@ -1,356 +0,0 @@
|
||||
name: Deploy Relay Production Capacity
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Verify, change one cell, or raise a sequential wave
|
||||
required: true
|
||||
default: verify
|
||||
type: choice
|
||||
options:
|
||||
- verify
|
||||
- apply
|
||||
- rollback
|
||||
- wave-apply
|
||||
- wave-resume
|
||||
target-cell-id:
|
||||
description: Exact serving cell for verify, apply, or rollback
|
||||
required: true
|
||||
default: production-gce-c26
|
||||
type: choice
|
||||
options:
|
||||
- production-gce-c7
|
||||
- production-gce-c8
|
||||
- production-gce-c9
|
||||
- production-gce-c10
|
||||
- production-gce-c13
|
||||
- production-gce-c14
|
||||
- production-gce-c15
|
||||
- production-gce-c16
|
||||
- production-gce-c19
|
||||
- production-gce-c20
|
||||
- production-gce-c21
|
||||
- production-gce-c22
|
||||
- production-gce-c23
|
||||
- production-gce-c24
|
||||
- production-gce-c25
|
||||
- production-gce-c26
|
||||
wave-cell-ids:
|
||||
description: Ordered comma-separated wave of two to four serving cells
|
||||
required: false
|
||||
default: none
|
||||
type: string
|
||||
confirmation:
|
||||
description: Enter the exact single-cell or wave confirmation
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-id:
|
||||
description: Successful fresh dry-run monitor workflow run ID for apply
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-attempt:
|
||||
description: Exact dry-run monitor workflow attempt for apply
|
||||
required: false
|
||||
type: string
|
||||
source-wave-run-id:
|
||||
description: Failed wave run that isolated the resume target
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
single_cell:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode != 'wave-apply' && inputs.mode != 'wave-resume') }}
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: ${{ inputs.mode }}
|
||||
target-cell-id: ${{ inputs.target-cell-id }}
|
||||
confirmation: ${{ inputs.confirmation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: single
|
||||
wave-cell-ids: none
|
||||
wave-index: '0'
|
||||
source-wave-run-id: none
|
||||
secrets: inherit
|
||||
|
||||
resume_cell:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-resume' && github.ref == 'refs/heads/main') }}
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: apply
|
||||
target-cell-id: ${{ inputs.target-cell-id }}
|
||||
confirmation: ${{ inputs.confirmation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: resume
|
||||
wave-cell-ids: ${{ inputs.wave-cell-ids }}
|
||||
wave-index: resume
|
||||
source-wave-run-id: ${{ inputs.source-wave-run-id }}
|
||||
secrets: inherit
|
||||
|
||||
wave_gate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-apply' && github.ref == 'refs/heads/main') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 30
|
||||
environment: production
|
||||
outputs:
|
||||
cells: ${{ steps.wave.outputs.cells }}
|
||||
env:
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
||||
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
|
||||
PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d
|
||||
COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f
|
||||
WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }}
|
||||
steps:
|
||||
- name: Require production workflow configuration
|
||||
working-directory: .
|
||||
env:
|
||||
DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${DEPLOY_SERVICE_ACCOUNT}"
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- id: wave
|
||||
name: Validate the exact wave request
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
CELLS="$(node dev/scripts/relay-production-capacity-wave.mjs validate \
|
||||
--wave-cell-ids "${WAVE_CELL_IDS}" \
|
||||
--confirmation "${CONFIRMATION}")"
|
||||
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Require fresh dry-run evidence reference
|
||||
run: |
|
||||
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
|
||||
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
|
||||
- name: Download private dry-run evidence
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ github.workspace }}/relay-monitor-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
- name: Verify dry-run artifact before cloud authentication
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
|
||||
--directory "${OUTPUT_DIRECTORY}" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run
|
||||
|
||||
- name: Reject previously consumed dry-run evidence
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
COUNT="$(gh api \
|
||||
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
||||
--jq '.total_count')"
|
||||
test "${COUNT}" = "0"
|
||||
|
||||
- id: deploy-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
release: 'false'
|
||||
|
||||
- name: Verify wave evidence against the live selector
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
FIRST_CELL="$(jq -er '.[0]' <<< '${{ steps.wave.outputs.cells }}')"
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
|
||||
--directory "${OUTPUT_DIRECTORY}" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run \
|
||||
--mutation-mode capacity-transition \
|
||||
--source-cell-id "${FIRST_CELL}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}"
|
||||
|
||||
- name: Recheck every live safety signal
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json"
|
||||
|
||||
- name: Require exact 600/60 predecessor wave cells
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
while read -r CELL_ID; do
|
||||
HOSTNAME="${CELL_ID#production-gce-}"
|
||||
[[ "${HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]]
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "https://${HOSTNAME}.relay.onorca.dev" \
|
||||
--cell-id "${CELL_ID}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed \
|
||||
--expected-image-digests \
|
||||
"${PREDECESSOR_IMAGE_DIGEST},${COMPATIBLE_CELL_IMAGE_DIGEST}"
|
||||
done < <(jq -r '.[]' <<< '${{ steps.wave.outputs.cells }}')
|
||||
|
||||
- name: Consume the single-use dry-run evidence
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
|
||||
printf '%s\n' "${GITHUB_RUN_ID}" \
|
||||
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
|
||||
|
||||
- name: Publish the consumed-evidence marker
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
wave_cell_1:
|
||||
needs: wave_gate
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: apply
|
||||
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[0] }}
|
||||
confirmation: RAISE_SELECTED_CELL_TO_1000
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: continuation
|
||||
wave-cell-ids: ${{ inputs.wave-cell-ids }}
|
||||
wave-index: '0'
|
||||
source-wave-run-id: none
|
||||
secrets: inherit
|
||||
|
||||
wave_cell_2:
|
||||
needs: [wave_gate, wave_cell_1]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: apply
|
||||
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[1] }}
|
||||
confirmation: RAISE_SELECTED_CELL_TO_1000
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: continuation
|
||||
wave-cell-ids: ${{ inputs.wave-cell-ids }}
|
||||
wave-index: '1'
|
||||
source-wave-run-id: none
|
||||
secrets: inherit
|
||||
|
||||
wave_cell_3:
|
||||
if: ${{ needs.wave_cell_2.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[2] != null }}
|
||||
needs: [wave_gate, wave_cell_2]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: apply
|
||||
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[2] }}
|
||||
confirmation: RAISE_SELECTED_CELL_TO_1000
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: continuation
|
||||
wave-cell-ids: ${{ inputs.wave-cell-ids }}
|
||||
wave-index: '2'
|
||||
source-wave-run-id: none
|
||||
secrets: inherit
|
||||
|
||||
wave_cell_4:
|
||||
if: ${{ needs.wave_cell_3.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[3] != null }}
|
||||
needs: [wave_gate, wave_cell_3]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: apply
|
||||
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[3] }}
|
||||
confirmation: RAISE_SELECTED_CELL_TO_1000
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: continuation
|
||||
wave-cell-ids: ${{ inputs.wave-cell-ids }}
|
||||
wave-index: '3'
|
||||
source-wave-run-id: none
|
||||
secrets: inherit
|
||||
|
||||
# Every wave job re-enters the run's lease with release: 'false'; only this job frees it.
|
||||
release_lease:
|
||||
if: always()
|
||||
needs:
|
||||
- single_cell
|
||||
- resume_cell
|
||||
- wave_gate
|
||||
- wave_cell_1
|
||||
- wave_cell_2
|
||||
- wave_cell_3
|
||||
- wave_cell_4
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 10
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
release: 'true'
|
||||
@@ -1,311 +0,0 @@
|
||||
name: Deploy Relay Production Director
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
image-digest:
|
||||
description: 'Immutable relay image digest (sha256: plus 64 lowercase hex characters)'
|
||||
required: true
|
||||
type: string
|
||||
regional-placement-mode:
|
||||
description: Preserve the live switch, explicitly enable Asia preference, or force US-first
|
||||
required: true
|
||||
default: preserve
|
||||
type: choice
|
||||
options: [preserve, enable, disable]
|
||||
region-correction-cohort-percent:
|
||||
description: 'Preserve the measured-correction cohort, or set an integer 0–100; durable rehome stays disabled'
|
||||
required: true
|
||||
default: preserve
|
||||
type: string
|
||||
reserve-placement:
|
||||
description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve
|
||||
required: true
|
||||
default: preserve
|
||||
type: choice
|
||||
options: [preserve, 'off', dry-run, 'on']
|
||||
shadow-seat-feed-cells:
|
||||
description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused
|
||||
required: true
|
||||
default: preserve
|
||||
type: string
|
||||
prune-incompatible-revisions:
|
||||
description: Retain only the newly verified serving and rollback revisions
|
||||
required: true
|
||||
default: false
|
||||
type: boolean
|
||||
confirmation:
|
||||
description: Enter the exact confirmation required by a destructive option
|
||||
required: false
|
||||
type: string
|
||||
expected-rehome-generation:
|
||||
description: Exact durable regional-rehome generation; it must remain disabled
|
||||
required: true
|
||||
type: string
|
||||
bootstrap-runtime-identity:
|
||||
description: One-time move from the stamped-cell identity to the director identity
|
||||
required: true
|
||||
default: false
|
||||
type: boolean
|
||||
predecessor-image-digest:
|
||||
description: Exact immutable serving predecessor digest for the one-time identity bootstrap
|
||||
required: true
|
||||
type: string
|
||||
shape-baseline-revision:
|
||||
description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# Director updates and candidate operations both mutate production relay control state.
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
DIRECTOR_SERVICE_NAME: orca-cloud-relay
|
||||
IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay
|
||||
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }}
|
||||
REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }}
|
||||
RESERVE_PLACEMENT: ${{ inputs.reserve-placement }}
|
||||
SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }}
|
||||
PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }}
|
||||
# Floor the served revision must keep, matching relay_min_instances in
|
||||
# environments/production.tfvars. This gate only fails a bad deploy; Terraform
|
||||
# still owns the value, and the candidate inherits it from the serving revision.
|
||||
DIRECTOR_MIN_INSTANCES: 5
|
||||
DIRECTOR_MAX_INSTANCES: 5
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
PREDECESSOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_RUNTIME_SERVICE_ACCOUNT }}
|
||||
REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain
|
||||
EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
||||
BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }}
|
||||
PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }}
|
||||
SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Resolve immutable production image
|
||||
shell: bash
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if [[ ! "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
|
||||
echo "image-digest must be an immutable lowercase sha256 digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
if test "${REGION_CORRECTION_COHORT_PERCENT}" != preserve; then
|
||||
[[ "${REGION_CORRECTION_COHORT_PERCENT}" =~ ^([0-9]|[1-9][0-9]|100)$ ]]
|
||||
fi
|
||||
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
|
||||
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
|
||||
[[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]]
|
||||
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]]
|
||||
[[ "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]]
|
||||
[[ "${BOOTSTRAP_RUNTIME_IDENTITY}" =~ ^(true|false)$ ]]
|
||||
if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then
|
||||
[[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false
|
||||
test "${REGIONAL_PLACEMENT_MODE}" = preserve
|
||||
test -z "${SHAPE_BASELINE_REVISION}"
|
||||
test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY
|
||||
elif test -n "${SHAPE_BASELINE_REVISION}"; then
|
||||
[[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-[0-9]{5}-[a-z0-9]{3}$ ]]
|
||||
test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false
|
||||
test "${REGIONAL_PLACEMENT_MODE}" = preserve
|
||||
test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE
|
||||
elif test "${PRUNE_INCOMPATIBLE_REVISIONS}" = true; then
|
||||
test "${REGIONAL_PLACEMENT_MODE}" = preserve
|
||||
test "${CONFIRMATION}" = PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS
|
||||
elif test "${REGIONAL_PLACEMENT_MODE}" = disable; then
|
||||
test "${CONFIRMATION}" = FORCE_RELAY_US_FIRST
|
||||
else
|
||||
test -z "${CONFIRMATION}"
|
||||
fi
|
||||
echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
|
||||
|
||||
# Why: the deploy INHERITS the serving revision's floor, so when that revision has
|
||||
# already lost it the candidate inherits zero, the in-script gate compares zero against
|
||||
# zero and passes, and the post-deploy check below only notices after traffic moved.
|
||||
# The documented rollback target is created at minimum instances zero, so promoting it
|
||||
# arms exactly that. Refuse to inherit a degraded floor rather than latch it.
|
||||
- name: Require a healthy serving floor before deploying
|
||||
shell: bash
|
||||
run: |
|
||||
SERVING="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
|
||||
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test -n "${SERVING}"
|
||||
FLOOR="$(gcloud run revisions describe "${SERVING}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
|
||||
if [[ "${FLOOR:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then
|
||||
echo "serving revision ${SERVING} holds ${FLOOR:-0} minimum instances," \
|
||||
"below ${DIRECTOR_MIN_INSTANCES}; deploying would inherit and latch it." >&2
|
||||
echo "Restore the floor first: gcloud run services update ${DIRECTOR_SERVICE_NAME}" \
|
||||
"--min-instances=${DIRECTOR_MIN_INSTANCES}" >&2
|
||||
exit 1
|
||||
fi
|
||||
CEILING="$(gcloud run revisions describe "${SERVING}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
|
||||
test "${CEILING}" = "${DIRECTOR_MAX_INSTANCES}"
|
||||
echo "serving revision ${SERVING} holds ${FLOOR} minimum instances"
|
||||
echo "SERVING_REVISION=${SERVING}" >> "${GITHUB_ENV}"
|
||||
|
||||
# Why: no --min-instances here. The candidate inherits the Terraform-owned
|
||||
# scaling, and this step ends with 100% traffic on it. Pinning 1 rebuilt the
|
||||
# per-instance admission shortage that took placement failures to ~70%.
|
||||
- name: Deploy director blue/green
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
served_version="$(gcloud run revisions describe "${SERVING_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '[.spec.containers[0].env[]? |
|
||||
select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
|
||||
(.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
|
||||
(.version // .key // empty)] |
|
||||
if length == 1 then .[0] else empty end')"
|
||||
if [[ "${served_version}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
current_version="${served_version}"
|
||||
else
|
||||
test "${REGIONAL_PLACEMENT_MODE}" = preserve
|
||||
current_version="$(gcloud secrets versions describe latest \
|
||||
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \
|
||||
--format='value(name)' | awk -F/ '{print $NF}')"
|
||||
[[ "${current_version}" =~ ^[1-9][0-9]*$ ]]
|
||||
fi
|
||||
current="$(gcloud secrets versions access "${current_version}" \
|
||||
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")"
|
||||
[[ "${current}" =~ ^(true|false)$ ]]
|
||||
case "${REGIONAL_PLACEMENT_MODE}" in
|
||||
preserve) desired="${current}" ;;
|
||||
enable) desired=true ;;
|
||||
disable) desired=false ;;
|
||||
*) echo "regional-placement-mode is invalid" >&2; exit 1 ;;
|
||||
esac
|
||||
if test "${current}" != "${desired}"; then
|
||||
target_version="$(printf '%s' "${desired}" | gcloud secrets versions add \
|
||||
"${REGIONAL_PLACEMENT_SECRET}" --project "${GCP_PROJECT_ID}" --data-file=- \
|
||||
--format='value(name)' --quiet | awk -F/ '{print $NF}')"
|
||||
else
|
||||
target_version="${current_version}"
|
||||
fi
|
||||
[[ "${target_version}" =~ ^[1-9][0-9]*$ ]]
|
||||
RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
|
||||
shape_args=()
|
||||
if test -n "${SHAPE_BASELINE_REVISION}"; then
|
||||
shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}")
|
||||
fi
|
||||
node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service "${DIRECTOR_SERVICE_NAME}" \
|
||||
--image "${IMAGE}" \
|
||||
--role director \
|
||||
"${shape_args[@]}" \
|
||||
--runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--bootstrap-runtime-identity "${BOOTSTRAP_RUNTIME_IDENTITY}" \
|
||||
--predecessor-image-digest "${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience "${REHOME_AUDIENCE}" \
|
||||
--rehome-control-origin https://relay.onorca.dev \
|
||||
--admin-audience https://relay.onorca.dev/v1/admin/drain \
|
||||
--expected-rehome-generation "${EXPECTED_REHOME_GENERATION}" \
|
||||
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
|
||||
--prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \
|
||||
--release-id "${RELEASE_ID}" \
|
||||
--regional-placement-secret-version "${target_version}" \
|
||||
--region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}" \
|
||||
--reserve-placement "${RESERVE_PLACEMENT}" \
|
||||
--shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}"
|
||||
echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}"
|
||||
echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify served revision and native health
|
||||
shell: bash
|
||||
run: |
|
||||
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format=json)"
|
||||
REVISION="$(jq -r '[.status.traffic[] | select((.percent // 0) > 0)] | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' <<< "${SERVICE_JSON}")"
|
||||
test -n "${REVISION}"
|
||||
SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format='value(spec.containers[0].image)')"
|
||||
test "${SERVED_IMAGE}" = "${IMAGE}"
|
||||
SERVED_REGIONAL_PLACEMENT_SECRET="$(gcloud run revisions describe "${REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -cer '[.spec.containers[0].env[] |
|
||||
select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
|
||||
(.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
|
||||
{secret: (.secret // .name), version: (.version // .key)}] |
|
||||
if length == 1 then .[0] else error("regional placement secret missing") end')"
|
||||
test "$(jq -r '.secret' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \
|
||||
"${REGIONAL_PLACEMENT_SECRET}"
|
||||
test "$(jq -r '.version' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \
|
||||
"${REGIONAL_PLACEMENT_VERSION}"
|
||||
test "$(gcloud secrets versions access "${REGIONAL_PLACEMENT_VERSION}" --project "${GCP_PROJECT_ID}" \
|
||||
--secret "${REGIONAL_PLACEMENT_SECRET}")" = "${REGIONAL_PLACEMENT_ENABLED}"
|
||||
# Why: a served revision with no warm-instance floor still passes health and digest
|
||||
# checks while quietly shrinking per-instance admission capacity.
|
||||
SERVED_MIN_INSTANCES="$(gcloud run revisions describe "${REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
|
||||
if [[ "${SERVED_MIN_INSTANCES:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then
|
||||
echo "served revision ${REVISION} holds ${SERVED_MIN_INSTANCES:-0} minimum instances, expected at least ${DIRECTOR_MIN_INSTANCES}" >&2
|
||||
exit 1
|
||||
fi
|
||||
SERVED_MAX_INSTANCES="$(gcloud run revisions describe "${REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
|
||||
test "${SERVED_MAX_INSTANCES}" = "${DIRECTOR_MAX_INSTANCES}"
|
||||
SERVICE_URL="$(jq -r '.status.url' <<< "${SERVICE_JSON}")"
|
||||
node dev/scripts/smoke-relay.mjs "${SERVICE_URL}"
|
||||
@@ -1,500 +0,0 @@
|
||||
name: Deploy Relay Production Multi-Target
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source-cell-id:
|
||||
description: Existing Terraform source cell ID
|
||||
required: true
|
||||
type: string
|
||||
target-cell-ids:
|
||||
description: Comma-separated distinct Terraform target cell IDs
|
||||
required: true
|
||||
type: string
|
||||
general-cell-ids:
|
||||
description: Comma-separated proven cells that remain eligible for ordinary placement
|
||||
required: false
|
||||
type: string
|
||||
unobserved-connection-bound:
|
||||
description: Exact worst-case unobserved connection bound proven by the passing load gate
|
||||
required: false
|
||||
type: string
|
||||
failed-target-cell-id:
|
||||
description: Registered failed target to fence and supersede
|
||||
required: false
|
||||
type: string
|
||||
replacement-target-cell-id:
|
||||
description: Healthy replacement for registered failed target
|
||||
required: false
|
||||
type: string
|
||||
mode:
|
||||
description: Preflight/audit are read-only; other modes mutate production
|
||||
required: true
|
||||
default: preflight
|
||||
type: choice
|
||||
options:
|
||||
- audit
|
||||
- preflight
|
||||
- cutover-admission
|
||||
- add-migration-cells
|
||||
- promote-general-cell
|
||||
- retire-migration-cell
|
||||
- execute
|
||||
- recover-forward
|
||||
- fence-source
|
||||
- supersede-target
|
||||
confirmation:
|
||||
description: Enter CUTOVER_SELECTOR, ADD_MIGRATION_CELLS, PROMOTE_GENERAL_CELL, RETIRE_MIGRATION_CELL, EVACUATE_MULTI, RECOVER_FORWARD, or FENCE_SOURCE
|
||||
required: false
|
||||
type: string
|
||||
selector-attempt-id:
|
||||
description: Exact durable selector attempt ID for admission mutations
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-id:
|
||||
description: Successful fresh dry-run monitor workflow run ID
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-attempt:
|
||||
description: Exact dry-run monitor workflow attempt
|
||||
required: false
|
||||
type: string
|
||||
broker-operation-id:
|
||||
description: Stable durable broker operation ID for target supersession
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-attempt-id:
|
||||
description: Exact older completed fence attempt to recover without replay
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-commit:
|
||||
description: Exact older fence commit bound to the completed attempt
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-operation:
|
||||
description: Exact DONE Compute resize operation to adopt
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-state-serial:
|
||||
description: Exact Terraform serial before the completed fence
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-plan-generation:
|
||||
description: Exact saved-plan object generation
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-state-generation:
|
||||
description: Exact current Terraform state object generation
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-state-sha256:
|
||||
description: Exact current Terraform state object SHA-256
|
||||
required: false
|
||||
type: string
|
||||
expected-lease-generation:
|
||||
description: Exact live lease generation authorized for conditional takeover
|
||||
required: false
|
||||
type: string
|
||||
expected-lease-operation-id:
|
||||
description: Exact live lease operation ID authorized for takeover
|
||||
required: false
|
||||
type: string
|
||||
expected-lease-request-digest:
|
||||
description: Exact live lease request digest authorized for takeover
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: >-
|
||||
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
|
||||
github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain
|
||||
SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
|
||||
TARGET_CELL_IDS: ${{ inputs.target-cell-ids }}
|
||||
GENERAL_CELL_IDS: ${{ inputs.general-cell-ids }}
|
||||
UNOBSERVED_CONNECTION_BOUND: ${{ inputs.unobserved-connection-bound }}
|
||||
FAILED_TARGET_CELL_ID: ${{ inputs.failed-target-cell-id }}
|
||||
REPLACEMENT_TARGET_CELL_ID: ${{ inputs.replacement-target-cell-id }}
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
||||
SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }}
|
||||
BROKER_OPERATION_ID: ${{ inputs.broker-operation-id }}
|
||||
COMPLETED_FENCE_ATTEMPT_ID: ${{ inputs.completed-fence-attempt-id }}
|
||||
COMPLETED_FENCE_COMMIT: ${{ inputs.completed-fence-commit }}
|
||||
COMPLETED_FENCE_OPERATION: ${{ inputs.completed-fence-operation }}
|
||||
COMPLETED_FENCE_STATE_SERIAL: ${{ inputs.completed-fence-state-serial }}
|
||||
COMPLETED_FENCE_PLAN_GENERATION: ${{ inputs.completed-fence-plan-generation }}
|
||||
COMPLETED_FENCE_STATE_GENERATION: ${{ inputs.completed-fence-state-generation }}
|
||||
COMPLETED_FENCE_STATE_SHA256: ${{ inputs.completed-fence-state-sha256 }}
|
||||
EXPECTED_LEASE_GENERATION: ${{ inputs.expected-lease-generation }}
|
||||
EXPECTED_LEASE_OPERATION_ID: ${{ inputs.expected-lease-operation-id }}
|
||||
EXPECTED_LEASE_REQUEST_DIGEST: ${{ inputs.expected-lease-request-digest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Require private fence-broker environment
|
||||
if: >-
|
||||
${{ inputs.mode == 'fence-source' ||
|
||||
inputs.mode == 'supersede-target' }}
|
||||
env:
|
||||
FENCE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
FENCE_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }}
|
||||
FENCE_BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
|
||||
run: |
|
||||
test -n "${FENCE_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${FENCE_SERVICE_ACCOUNT}"
|
||||
test -n "${FENCE_BROKER_URI}"
|
||||
|
||||
- name: Reject direct-runner Terraform fence aborts
|
||||
if: ${{ inputs.mode == 'abort-fence-source' }}
|
||||
run: |
|
||||
echo "Terraform fence aborts require a reviewed private-broker recovery path." >&2
|
||||
exit 1
|
||||
|
||||
- name: Require fresh dry-run evidence reference
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
run: |
|
||||
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
|
||||
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
|
||||
- name: Download private dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Verify dry-run artifact before cloud authentication
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run
|
||||
|
||||
- name: Reject previously consumed dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
COUNT="$(gh api \
|
||||
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
||||
--jq '.total_count')"
|
||||
test "${COUNT}" = "0"
|
||||
|
||||
- id: google-auth
|
||||
if: ${{ inputs.mode != 'supersede-target' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
|
||||
- name: Require explicit mutation confirmation
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if [[ "${DEPLOY_MODE}" = "cutover-admission" ]]; then
|
||||
test "${CONFIRMATION}" = "CUTOVER_SELECTOR"
|
||||
elif [[ "${DEPLOY_MODE}" = "add-migration-cells" ]]; then
|
||||
test "${CONFIRMATION}" = "ADD_MIGRATION_CELLS"
|
||||
elif [[ "${DEPLOY_MODE}" = "promote-general-cell" ]]; then
|
||||
test "${CONFIRMATION}" = "PROMOTE_GENERAL_CELL"
|
||||
elif [[ "${DEPLOY_MODE}" = "retire-migration-cell" ]]; then
|
||||
test "${CONFIRMATION}" = "RETIRE_MIGRATION_CELL"
|
||||
elif [[ "${DEPLOY_MODE}" = "execute" ]]; then
|
||||
test "${CONFIRMATION}" = "EVACUATE_MULTI"
|
||||
elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then
|
||||
test "${CONFIRMATION}" = "RECOVER_FORWARD"
|
||||
elif [[ "${DEPLOY_MODE}" = "fence-source" ]]; then
|
||||
test "${CONFIRMATION}" = "FENCE_SOURCE"
|
||||
elif [[ "${DEPLOY_MODE}" = "supersede-target" ]]; then
|
||||
test "${CONFIRMATION}" = "SUPERSEDE_TARGET"
|
||||
elif [[ "${DEPLOY_MODE}" = "abort-fence-source" ]]; then
|
||||
test "${CONFIRMATION}" = "ABORT_FENCE"
|
||||
else
|
||||
test "${CONFIRMATION}" = "FENCE_SOURCE"
|
||||
fi
|
||||
|
||||
- name: Require exact source-fence broker contract
|
||||
if: ${{ inputs.mode == 'fence-source' }}
|
||||
run: |
|
||||
test "${SOURCE_CELL_ID}" = "production-gce-c3"
|
||||
test "${TARGET_CELL_IDS}" = "production-gce-c7,production-gce-c8,production-gce-c10,production-gce-c13,production-gce-c17,production-gce-c18"
|
||||
[[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
|
||||
[[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
|
||||
test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}"
|
||||
[[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]]
|
||||
else
|
||||
test -z "${EXPECTED_LEASE_OPERATION_ID}"
|
||||
test -z "${EXPECTED_LEASE_REQUEST_DIGEST}"
|
||||
fi
|
||||
|
||||
- name: Require exact broker cell contract
|
||||
if: ${{ inputs.mode == 'supersede-target' }}
|
||||
run: |
|
||||
test "${SOURCE_CELL_ID}" = "production-gce-c3"
|
||||
test "${FAILED_TARGET_CELL_ID}" = "production-gce-c12"
|
||||
test "${REPLACEMENT_TARGET_CELL_ID}" = "production-gce-c13"
|
||||
test "${TARGET_CELL_IDS}" = "production-gce-c12,production-gce-c13"
|
||||
if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then
|
||||
[[ "${COMPLETED_FENCE_ATTEMPT_ID}" =~ ^[0-9a-f-]{36}$ ]]
|
||||
[[ "${COMPLETED_FENCE_COMMIT}" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "${COMPLETED_FENCE_OPERATION}" =~ ^[A-Za-z0-9._-]{1,256}$ ]]
|
||||
[[ "${COMPLETED_FENCE_STATE_SERIAL}" =~ ^[0-9]+$ ]]
|
||||
[[ "${COMPLETED_FENCE_PLAN_GENERATION}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${COMPLETED_FENCE_STATE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${COMPLETED_FENCE_STATE_SHA256}" =~ ^[0-9a-f]{64}$ ]]
|
||||
test -n "${EXPECTED_LEASE_GENERATION}"
|
||||
fi
|
||||
if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
|
||||
[[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
|
||||
test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}"
|
||||
[[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]]
|
||||
else
|
||||
test -z "${EXPECTED_LEASE_OPERATION_ID}"
|
||||
test -z "${EXPECTED_LEASE_REQUEST_DIGEST}"
|
||||
fi
|
||||
|
||||
- name: Read reviewed Terraform topology
|
||||
if: ${{ inputs.mode != 'supersede-target' }}
|
||||
run: |
|
||||
node dev/scripts/infra.mjs init --env production
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
|
||||
DIRECTOR_MIN_INSTANCES="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars <<< 'var.relay_min_instances')"
|
||||
[[ "${DIRECTOR_MIN_INSTANCES}" =~ ^[1-9][0-9]*$ ]]
|
||||
echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
|
||||
echo "DIRECTOR_MIN_INSTANCES=${DIRECTOR_MIN_INSTANCES}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify fresh dry-run evidence against live selector
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
SCOPED_RECOVERY_ARGS=()
|
||||
if [[ ("${DEPLOY_MODE}" = "execute" ||
|
||||
"${DEPLOY_MODE}" = "recover-forward") &&
|
||||
"${SOURCE_CELL_ID}" = "production-gce-c12" ]]; then
|
||||
SCOPED_RECOVERY_ARGS=(
|
||||
--scoped-recovery-source-cell-id
|
||||
production-gce-c3
|
||||
)
|
||||
fi
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run \
|
||||
--mutation-mode "${DEPLOY_MODE}" \
|
||||
--source-cell-id "${SOURCE_CELL_ID}" \
|
||||
"${SCOPED_RECOVERY_ARGS[@]}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}"
|
||||
|
||||
- name: Recheck all live safety signals
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
|
||||
|
||||
- name: Create single-use dry-run marker
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
|
||||
printf '%s\n' "${GITHUB_RUN_ID}" \
|
||||
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
|
||||
|
||||
- name: Consume dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
- id: google-fence-broker-auth
|
||||
if: >-
|
||||
${{ inputs.mode == 'fence-source' ||
|
||||
inputs.mode == 'supersede-target' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Invoke private target-supersession broker
|
||||
if: ${{ inputs.mode == 'supersede-target' }}
|
||||
env:
|
||||
BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }}
|
||||
BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
|
||||
run: |
|
||||
[[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then
|
||||
REQUEST="$(jq -cn \
|
||||
--arg operationId "${BROKER_OPERATION_ID}" \
|
||||
--arg fenceCommit "${GITHUB_SHA}" \
|
||||
--arg attemptId "${COMPLETED_FENCE_ATTEMPT_ID}" \
|
||||
--arg completedCommit "${COMPLETED_FENCE_COMMIT}" \
|
||||
--arg gceOperation "${COMPLETED_FENCE_OPERATION}" \
|
||||
--arg stateSerial "${COMPLETED_FENCE_STATE_SERIAL}" \
|
||||
--arg planGeneration "${COMPLETED_FENCE_PLAN_GENERATION}" \
|
||||
--arg stateGeneration "${COMPLETED_FENCE_STATE_GENERATION}" \
|
||||
--arg stateSha256 "${COMPLETED_FENCE_STATE_SHA256}" \
|
||||
--arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
|
||||
--arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
|
||||
--arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
|
||||
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
|
||||
completedFenceRecovery:{attemptId:$attemptId,fenceCommit:$completedCommit,
|
||||
gceOperation:$gceOperation,terraformStateSerial:($stateSerial|tonumber),
|
||||
planObjectGeneration:$planGeneration,
|
||||
terraformStateObjectGeneration:$stateGeneration,
|
||||
terraformStateObjectSha256:$stateSha256},
|
||||
expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
|
||||
requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')"
|
||||
elif [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
|
||||
REQUEST="$(jq -cn \
|
||||
--arg operationId "${BROKER_OPERATION_ID}" \
|
||||
--arg fenceCommit "${GITHUB_SHA}" \
|
||||
--arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
|
||||
--arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
|
||||
--arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
|
||||
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
|
||||
expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
|
||||
requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')"
|
||||
else
|
||||
REQUEST="$(jq -cn \
|
||||
--arg operationId "${BROKER_OPERATION_ID}" \
|
||||
--arg fenceCommit "${GITHUB_SHA}" \
|
||||
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,confirmation:"SUPERSEDE_TARGET"}')"
|
||||
fi
|
||||
curl --fail-with-body --max-time 1790 \
|
||||
--request POST "${BROKER_URI}/v1/supersede-target" \
|
||||
--header "Authorization: Bearer ${BROKER_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "${REQUEST}"
|
||||
|
||||
- name: Invoke private source-fence broker
|
||||
if: ${{ inputs.mode == 'fence-source' }}
|
||||
env:
|
||||
BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }}
|
||||
BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
|
||||
run: |
|
||||
if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
|
||||
REQUEST="$(jq -cn \
|
||||
--arg operationId "${BROKER_OPERATION_ID}" \
|
||||
--arg fenceCommit "${GITHUB_SHA}" \
|
||||
--arg targetCellIds "${TARGET_CELL_IDS}" \
|
||||
--arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
|
||||
--arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
|
||||
--arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
|
||||
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
|
||||
targetCellIds:($targetCellIds|split(",")),
|
||||
expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
|
||||
requestDigest:$leaseRequestDigest},confirmation:"FENCE_SOURCE"}')"
|
||||
else
|
||||
REQUEST="$(jq -cn \
|
||||
--arg operationId "${BROKER_OPERATION_ID}" \
|
||||
--arg fenceCommit "${GITHUB_SHA}" \
|
||||
--arg targetCellIds "${TARGET_CELL_IDS}" \
|
||||
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
|
||||
targetCellIds:($targetCellIds|split(",")),confirmation:"FENCE_SOURCE"}')"
|
||||
fi
|
||||
curl --fail-with-body --max-time 1790 \
|
||||
--request POST "${BROKER_URI}/v1/fence-source" \
|
||||
--header "Authorization: Bearer ${BROKER_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "${REQUEST}"
|
||||
|
||||
- name: Preflight or run multi-target evacuation
|
||||
if: >-
|
||||
${{ inputs.mode != 'fence-source' &&
|
||||
inputs.mode != 'supersede-target' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/deploy-relay-gce-multi-target.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--admin-audience "${ADMIN_AUDIENCE}" \
|
||||
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
|
||||
--source-cell-id "${SOURCE_CELL_ID}" \
|
||||
--target-cell-ids "${TARGET_CELL_IDS}" \
|
||||
--general-cell-ids "${GENERAL_CELL_IDS}" \
|
||||
--unobserved-connection-bound "${UNOBSERVED_CONNECTION_BOUND}" \
|
||||
--director-region "${{ vars.PRODUCTION_GCP_REGION }}" \
|
||||
--director-service "orca-cloud-relay" \
|
||||
--director-min-instances "${DIRECTOR_MIN_INSTANCES}" \
|
||||
--selector-attempt-id "${SELECTOR_ATTEMPT_ID}" \
|
||||
--failed-target-cell-id "${FAILED_TARGET_CELL_ID}" \
|
||||
--replacement-target-cell-id "${REPLACEMENT_TARGET_CELL_ID}" \
|
||||
--runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--environment production \
|
||||
--fence-commit "${GITHUB_SHA}" \
|
||||
--terraform-dir infra/terraform \
|
||||
--terraform-var-file environments/production.tfvars \
|
||||
--mode "${DEPLOY_MODE}" \
|
||||
--connection-ceiling 1000 \
|
||||
--minimum-lease-remaining-ms 600000
|
||||
@@ -1,930 +0,0 @@
|
||||
name: Deploy Relay Production Same-Cap Job
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
mode: { required: true, type: string }
|
||||
target-cell-id: { required: true, type: string }
|
||||
target-image-digest: { required: true, type: string }
|
||||
rollback-image-digest: { required: true, type: string }
|
||||
target-rehome-protocol: { required: true, type: string }
|
||||
rollback-rehome-protocol: { required: true, type: string }
|
||||
expected-selector-generation: { required: true, type: string }
|
||||
expected-existing-only-cells: { required: true, type: string }
|
||||
expected-migration-only-cells: { required: true, type: string }
|
||||
expected-general-cells: { required: true, type: string }
|
||||
expected-rehome-generation: { required: true, type: string }
|
||||
wave-index: { required: true, type: string }
|
||||
drain-pace-window-ms: { required: true, type: string }
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
rollout:
|
||||
if: ${{ github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
# The pre-drain sample adds 3-8 min (20 at most); a timeout cancels the job and skips the failsafe.
|
||||
# A 1200000 drain step alone may take 50 min.
|
||||
timeout-minutes: 120
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay
|
||||
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }}
|
||||
ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }}
|
||||
TARGET_REHOME_PROTOCOL: ${{ inputs.target-rehome-protocol }}
|
||||
ROLLBACK_REHOME_PROTOCOL: ${{ inputs.rollback-rehome-protocol }}
|
||||
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
||||
EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
|
||||
EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
|
||||
EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
|
||||
EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
||||
WAVE_INDEX: ${{ inputs.wave-index }}
|
||||
# The cell spreads its drain sends over this window, so each drained host re-dials
|
||||
# /v1/assign at hosts / window and is re-placed through the director's drain-return
|
||||
# lane. The wave validator admits only its closed set, and anything below the 300000
|
||||
# default only on US general cells: Asia drains are bound by their targets' accept
|
||||
# rate. The transition wait below is the 15-min migration lease plus this window. A
|
||||
# cell on an older image drains at once, or refuses a window above 300000.
|
||||
DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }}
|
||||
steps:
|
||||
- name: Require exact reusable-workflow configuration
|
||||
working-directory: .
|
||||
env:
|
||||
DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
CAPACITY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
[[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]]
|
||||
[[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
[[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}"
|
||||
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
|
||||
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
|
||||
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
# cell_1..cell_10 in the calling wave; the chain is static, so this range is too.
|
||||
[[ "${WAVE_INDEX}" =~ ^[0-9]$ ]]
|
||||
if test "${DEPLOY_MODE}" != verify && test "${GITHUB_RUN_ATTEMPT}" != 1; then
|
||||
echo "mutations are single-dispatch: recover each remaining cell" >&2
|
||||
echo "with its own fresh dispatch instead of a re-run" >&2
|
||||
exit 1
|
||||
fi
|
||||
test -n "${GCP_REGION}"
|
||||
test -n "${DEPLOY_WIF}"
|
||||
test -n "${DEPLOY_SERVICE_ACCOUNT}"
|
||||
test -n "${CAPACITY_WIF}"
|
||||
test -n "${CAPACITY_SERVICE_ACCOUNT}"
|
||||
test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with: { package-json-file: cloud/package.json }
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with: { terraform_wrapper: false }
|
||||
|
||||
# One approved-cell table, in the wave validator the dispatch gate already uses, so
|
||||
# a cell's class and its wave's selector delta cannot drift apart between the two.
|
||||
# Also re-checks the drain pace against this cell, before anything reads production.
|
||||
- name: Resolve this cell's admission class and wave selector delta
|
||||
run: |
|
||||
CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \
|
||||
--cell-id "${TARGET_CELL_ID}" --drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}")"
|
||||
ENTRY_ADMISSION="$(jq -er '.entryAdmission' <<< "${CELL_CLASS}")"
|
||||
SELECTOR_WAVE_DELTA="$(jq -er '.selectorWaveDelta' <<< "${CELL_CLASS}")"
|
||||
if test "${DEPLOY_MODE}" = verify; then
|
||||
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
|
||||
else
|
||||
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION \
|
||||
+ (SELECTOR_WAVE_DELTA * WAVE_INDEX)))"
|
||||
fi
|
||||
{
|
||||
echo "ENTRY_ADMISSION=${ENTRY_ADMISSION}"
|
||||
echo "SELECTOR_WAVE_DELTA=${SELECTOR_WAVE_DELTA}"
|
||||
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- id: deploy-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
release: 'false'
|
||||
|
||||
- name: Recheck aggregate SQL, pool, reconnect, migration, and selector safety
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
# Freshness-only failures are publish lag, not health, so the CLI retries
|
||||
# them a bounded number of times. The expected selector comes from the
|
||||
# dispatch inputs, which the rehome inspect below verifies against the
|
||||
# live director; the pre-drain sample later reads the same file.
|
||||
jq -n \
|
||||
--arg existingOnly "${EXPECTED_EXISTING_ONLY_CELLS/none/}" \
|
||||
--arg migrationOnly "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
|
||||
--arg general "${EXPECTED_GENERAL_CELLS/none/}" \
|
||||
'{existingOnly:$existingOnly,migrationOnly:$migrationOnly,general:$general}
|
||||
| map_values(split(",") | map(select(length > 0)))' \
|
||||
> "${RUNNER_TEMP}/relay-same-cap-selector.json"
|
||||
pnpm incident:relay-preflight -- \
|
||||
--no-monitor-state \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \
|
||||
--wave-index "${WAVE_INDEX}" \
|
||||
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
|
||||
|
||||
- name: Require durable rehome disabled and exact selector
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode inspect \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${EFFECTIVE_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
|
||||
| jq -e '.control.enabled == false' >/dev/null
|
||||
|
||||
- name: Initialize the exact production backend
|
||||
run: node dev/scripts/infra.mjs init --env production
|
||||
|
||||
- name: Resolve immutable same-cap cell configuration
|
||||
shell: bash
|
||||
run: |
|
||||
TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"
|
||||
case "${TARGET_HOSTNAME}" in
|
||||
c7|c8|c9|c10|c13|c14|c15|c16|c19|c20|c21|c22|c23|c24|c25|c26)
|
||||
EXPECTED_HARD_CAP=1000
|
||||
EXPECTED_REGION=us-central1
|
||||
EXPECTED_DATABASE_POOL_MAX=
|
||||
;;
|
||||
c17|c18)
|
||||
EXPECTED_HARD_CAP=600
|
||||
EXPECTED_REGION=us-central1
|
||||
EXPECTED_DATABASE_POOL_MAX=
|
||||
;;
|
||||
c27|c28|c29|c30|c31|c34|c35)
|
||||
EXPECTED_HARD_CAP=3000
|
||||
EXPECTED_REGION=asia-east2
|
||||
EXPECTED_DATABASE_POOL_MAX=16
|
||||
;;
|
||||
c32|c33)
|
||||
EXPECTED_HARD_CAP=3000
|
||||
EXPECTED_REGION=us-central1
|
||||
EXPECTED_DATABASE_POOL_MAX=
|
||||
;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
EXPECTED_UNOBSERVED_BOUND=60
|
||||
CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev"
|
||||
CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
|
||||
SOURCE_CELLS="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
<<< 'jsonencode(var.relay_region_rehome_source_cell_ids)' | jq -er '.')"
|
||||
CURRENT_SHAPE="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${CELLS_JSON}")"
|
||||
test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}"
|
||||
test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \
|
||||
"${EXPECTED_UNOBSERVED_BOUND}"
|
||||
# The startup script emits a pool line only off the root default, so an unpinned cell
|
||||
# must still be on that default or its plan would carry a line nothing reviews.
|
||||
test "$(jq -r '.database_pool_max' <<< "${CURRENT_SHAPE}")" = \
|
||||
"${EXPECTED_DATABASE_POOL_MAX:-10}"
|
||||
TARGET_ZONE="$(jq -r '.zone' <<< "${CURRENT_SHAPE}")"
|
||||
MIG_NAME="orca-cloud-relay-gce-${TARGET_HOSTNAME}"
|
||||
if test "${DEPLOY_MODE}" = rollback; then
|
||||
DESIRED_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}"
|
||||
CURRENT_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}"
|
||||
DESIRED_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
|
||||
CURRENT_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
|
||||
else
|
||||
DESIRED_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}"
|
||||
CURRENT_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}"
|
||||
DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
|
||||
CURRENT_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
|
||||
fi
|
||||
# The startup template emits rehome trust lines only for a declared source cell, so
|
||||
# require membership exactly when either side of this roll expects those lines.
|
||||
if test "${EXPECTED_REGION}" = us-central1 && {
|
||||
test "${DESIRED_REHOME_PROTOCOL}" != 0 || test "${CURRENT_REHOME_PROTOCOL}" != 0
|
||||
}; then
|
||||
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
|
||||
<<< "${SOURCE_CELLS}" >/dev/null
|
||||
fi
|
||||
DESIRED_IMAGE="${IMAGE_REPOSITORY}@${DESIRED_IMAGE_DIGEST}"
|
||||
OVERRIDE_CELLS_JSON="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
|
||||
--arg image "${DESIRED_IMAGE}" '.[$cell].image = $image' <<< "${CELLS_JSON}")"
|
||||
jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \
|
||||
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-same-cap.tfvars.json"
|
||||
SERVED_DIGEST="$(gcloud artifacts docker images describe "${DESIRED_IMAGE}" \
|
||||
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${SERVED_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"
|
||||
{
|
||||
echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}"
|
||||
echo "CELL_ORIGIN=${CELL_ORIGIN}"
|
||||
echo "TARGET_ZONE=${TARGET_ZONE}"
|
||||
echo "MIG_NAME=${MIG_NAME}"
|
||||
echo "EXPECTED_HARD_CAP=${EXPECTED_HARD_CAP}"
|
||||
echo "EXPECTED_UNOBSERVED_BOUND=${EXPECTED_UNOBSERVED_BOUND}"
|
||||
echo "EXPECTED_DATABASE_POOL_MAX=${EXPECTED_DATABASE_POOL_MAX}"
|
||||
echo "EXPECTED_REGION=${EXPECTED_REGION}"
|
||||
echo "DESIRED_IMAGE=${DESIRED_IMAGE}"
|
||||
echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}"
|
||||
echo "CURRENT_IMAGE_DIGEST=${CURRENT_IMAGE_DIGEST}"
|
||||
echo "DESIRED_REHOME_PROTOCOL=${DESIRED_REHOME_PROTOCOL}"
|
||||
echo "CURRENT_REHOME_PROTOCOL=${CURRENT_REHOME_PROTOCOL}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify exact current generation, digest, cap, and rollback point
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
# A single transient 5xx (LB warm-up behind a fresh instance) must not
|
||||
# fail a canary; 4xx (auth, generation mismatch) still fails fast.
|
||||
admin_post() {
|
||||
local out="${RUNNER_TEMP}/$1.json"
|
||||
if ! curl --fail-with-body --max-time 30 \
|
||||
--retry 3 --retry-delay 2 --retry-connrefused --output "${out}" \
|
||||
--request POST "$2" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data "$3"; then
|
||||
cat "${out}" >&2
|
||||
return 1
|
||||
fi
|
||||
cat "${out}"
|
||||
}
|
||||
CURRENT_RUNTIME="$(admin_post current-runtime \
|
||||
"${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
|
||||
# Two different failures leave the cell on the rollback image, and the image
|
||||
# alone cannot tell them apart. A rollback that failed between its template
|
||||
# apply and its admission restore restarted the cell, so that cell is not
|
||||
# draining and resumes. A wave that stopped after its drain and before its
|
||||
# template apply never restarted anything, so its cell is still draining and
|
||||
# is stranded: the drain flag only clears on a restart, so it has to be rolled.
|
||||
LIVE_IMAGE_DIGEST="$(jq -r '.imageDigest' <<< "${CURRENT_RUNTIME}")"
|
||||
LIVE_DRAINING="$(jq -r '.draining' <<< "${CURRENT_RUNTIME}")"
|
||||
if test "${DEPLOY_MODE}" = rollback \
|
||||
&& test "${LIVE_IMAGE_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"; then
|
||||
if test "${LIVE_DRAINING}" = true; then
|
||||
ROLLBACK_STAGE=stranded
|
||||
else
|
||||
ROLLBACK_STAGE=resume
|
||||
fi
|
||||
PREDECESSOR_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
|
||||
PREDECESSOR_REHOME_PROTOCOL="${DESIRED_REHOME_PROTOCOL}"
|
||||
else
|
||||
ROLLBACK_STAGE=roll
|
||||
PREDECESSOR_IMAGE_DIGEST="${CURRENT_IMAGE_DIGEST}"
|
||||
PREDECESSOR_REHOME_PROTOCOL="${CURRENT_REHOME_PROTOCOL}"
|
||||
fi
|
||||
if test "${ROLLBACK_STAGE}" = resume; then
|
||||
ROLLBACK_RESUME=true
|
||||
else
|
||||
ROLLBACK_RESUME=false
|
||||
fi
|
||||
# A stranded cell's template still carries the image the cell is serving, so that
|
||||
# is the predecessor its plan is reviewed against. A template already moved on to
|
||||
# the target is refused here rather than rolled backwards under a stale review.
|
||||
if test "${ROLLBACK_STAGE}" = stranded; then
|
||||
PLAN_ROLLBACK_IMAGE="${DESIRED_IMAGE}"
|
||||
else
|
||||
PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"
|
||||
fi
|
||||
# Rollback is the documented recovery from a failed canary, which
|
||||
# leaves the cell migration-only (and possibly still marked
|
||||
# draining); apply and verify still require the cell pristine in the
|
||||
# class it is declared to serve in.
|
||||
if test "${DEPLOY_MODE}" = rollback; then
|
||||
PRECHECK_ADMISSION=general-or-migration-only
|
||||
else
|
||||
PRECHECK_ADMISSION="${ENTRY_ADMISSION}"
|
||||
fi
|
||||
# Draining sheds connections, and a migration-only cell holds none, so the flag
|
||||
# carries no precondition there. It also outlives a failed wave, because the drain
|
||||
# that set it is followed by no restart, which is the state a failed canary leaves.
|
||||
if test "${DEPLOY_MODE}" = rollback \
|
||||
|| test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
PRECHECK_DRAINING=either
|
||||
else
|
||||
PRECHECK_DRAINING=forbidden
|
||||
fi
|
||||
# A resumed rollback already restarted, so its cell has to come back not draining;
|
||||
# that is what separates it from a wave that stopped before its template apply.
|
||||
if test "${PRECHECK_DRAINING}" = either && test "${ROLLBACK_RESUME}" != true; then
|
||||
PREDECESSOR_DRAINING_OK=true
|
||||
else
|
||||
PREDECESSOR_DRAINING_OK=false
|
||||
fi
|
||||
RESTORED_MIGRATION_CELLS="$(jq -rn \
|
||||
--arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
|
||||
--arg target "${TARGET_CELL_ID}" \
|
||||
'$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
|
||||
RESTORED_GENERAL_CELLS="$(jq -rn \
|
||||
--arg value "${EXPECTED_GENERAL_CELLS/none/}" \
|
||||
--arg target "${TARGET_CELL_ID}" \
|
||||
'$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')"
|
||||
test -n "${RESTORED_MIGRATION_CELLS}" || RESTORED_MIGRATION_CELLS=none
|
||||
test -n "${RESTORED_GENERAL_CELLS}" || RESTORED_GENERAL_CELLS=none
|
||||
ISOLATED_MIGRATION_CELLS="$(jq -rn \
|
||||
--arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
|
||||
--arg target "${TARGET_CELL_ID}" \
|
||||
'$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')"
|
||||
ISOLATED_GENERAL_CELLS="$(jq -rn \
|
||||
--arg value "${EXPECTED_GENERAL_CELLS/none/}" \
|
||||
--arg target "${TARGET_CELL_ID}" \
|
||||
'$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
|
||||
test -n "${ISOLATED_MIGRATION_CELLS}" || ISOLATED_MIGRATION_CELLS=none
|
||||
test -n "${ISOLATED_GENERAL_CELLS}" || ISOLATED_GENERAL_CELLS=none
|
||||
# A migration-only cell is already isolated and is handed back isolated, so both
|
||||
# halves of its wave see exactly the membership it entered with.
|
||||
if test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
RESTORED_MIGRATION_CELLS="${ISOLATED_MIGRATION_CELLS}"
|
||||
RESTORED_GENERAL_CELLS="${ISOLATED_GENERAL_CELLS}"
|
||||
fi
|
||||
{
|
||||
echo "ROLLBACK_RESUME=${ROLLBACK_RESUME}"
|
||||
echo "ROLLBACK_STAGE=${ROLLBACK_STAGE}"
|
||||
echo "PLAN_ROLLBACK_IMAGE=${PLAN_ROLLBACK_IMAGE}"
|
||||
# The drain wait and the plan review both read the image this cell actually
|
||||
# serves, which is the rollback image on a stranded cell and not the current one.
|
||||
echo "PREDECESSOR_IMAGE_DIGEST=${PREDECESSOR_IMAGE_DIGEST}"
|
||||
# The failsafe consumes these; deriving them here keeps them
|
||||
# defined for a failure in any later step.
|
||||
echo "ISOLATED_MIGRATION_CELLS=${ISOLATED_MIGRATION_CELLS}"
|
||||
echo "ISOLATED_GENERAL_CELLS=${ISOLATED_GENERAL_CELLS}"
|
||||
# No restart happens on resume, so isolate below is skipped and
|
||||
# cannot advance the selector generation.
|
||||
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${EFFECTIVE_SELECTOR_GENERATION}"
|
||||
# A failed-canary rollback enters with the target migration-only,
|
||||
# so the restore inspect cannot reuse the entry membership inputs.
|
||||
echo "RESTORED_MIGRATION_CELLS=${RESTORED_MIGRATION_CELLS}"
|
||||
echo "RESTORED_GENERAL_CELLS=${RESTORED_GENERAL_CELLS}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
if ! jq -e --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \
|
||||
--arg digest "${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
--arg region "${EXPECTED_REGION}" \
|
||||
--argjson hardCap "${EXPECTED_HARD_CAP}" \
|
||||
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
|
||||
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
|
||||
'.role == "cell" and .cellId == $cell and .cellUrl == $origin and
|
||||
(.region == $region or
|
||||
($region == "us-central1" and $protocol == 0 and .region == null)) and
|
||||
.imageDigest == $digest and
|
||||
.connectionCapacity.hardCap == $hardCap and
|
||||
.connectionCapacity.unobservedBound == $unobservedBound and
|
||||
(.draining == false or $drainingOk) and
|
||||
(.regionalRehomeProtocol // 0) == $protocol' <<< "${CURRENT_RUNTIME}" >/dev/null
|
||||
then
|
||||
jq -r --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \
|
||||
--arg digest "${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
--arg region "${EXPECTED_REGION}" \
|
||||
--argjson hardCap "${EXPECTED_HARD_CAP}" \
|
||||
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
|
||||
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
|
||||
'[
|
||||
if .role != "cell" then "role" else empty end,
|
||||
if .cellId != $cell then "cellId" else empty end,
|
||||
if .cellUrl != $origin then "cellUrl" else empty end,
|
||||
if (.region != $region and
|
||||
($region != "us-central1" or $protocol != 0 or .region != null))
|
||||
then "region" else empty end,
|
||||
if .imageDigest != $digest then "imageDigest" else empty end,
|
||||
if .connectionCapacity.hardCap != $hardCap then "hardCap" else empty end,
|
||||
if .connectionCapacity.unobservedBound != $unobservedBound then "unobservedBound" else empty end,
|
||||
if (.draining != false and ($drainingOk | not)) then "draining" else empty end,
|
||||
if (.regionalRehomeProtocol // 0) != $protocol then "regionalRehomeProtocol" else empty end
|
||||
] | "runtime predecessor mismatch fields=" + join(",")' \
|
||||
<<< "${CURRENT_RUNTIME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
# The exact legacy digest binds omitted pre-region fields to US and protocol 0.
|
||||
jq -r '[
|
||||
if .region == null then "region" else empty end,
|
||||
if .regionalRehomeProtocol == null then "regionalRehomeProtocol" else empty end
|
||||
] | if length > 0 then "runtime predecessor normalized legacy fields=" + join(",") else empty end' \
|
||||
<<< "${CURRENT_RUNTIME}"
|
||||
if ! CURRENT_DIRECTOR_STATUS="$(admin_post current-cell-status \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
"$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"; then
|
||||
if jq -e '.error == "cell_in_reserve_mode"' \
|
||||
"${RUNNER_TEMP}/current-cell-status.json" >/dev/null 2>&1; then
|
||||
echo "${TARGET_CELL_ID} is in reserve mode: flip it to admitMode=db with the cell-flags workflow and wait for it to record db before rolling it, then flip it back to reserve" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
SOURCE_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \
|
||||
<<< "${CURRENT_DIRECTOR_STATUS}")"
|
||||
if test "${ROLLBACK_RESUME}" = true && ! jq -e \
|
||||
'.status.admissionState == "migration-only"' \
|
||||
<<< "${CURRENT_DIRECTOR_STATUS}" >/dev/null; then
|
||||
echo 'resume requires the isolated migration-only cell a failed rollback leaves' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ "${SOURCE_INCARNATION}" =~ ^[0-9a-f-]{36}$ ]]
|
||||
echo "SOURCE_INCARNATION=${SOURCE_INCARNATION}" >> "${GITHUB_ENV}"
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh --admission "${PRECHECK_ADMISSION}" \
|
||||
--draining "${PRECHECK_DRAINING}" --activity allowed \
|
||||
--expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}"
|
||||
|
||||
# Hosts drained with nowhere to go keep redialling and pin the cell empty (c28, 2026-10-01).
|
||||
# Every run that drains (apply, rollback) checks first; verify reads the same answer; a resume drains nothing.
|
||||
- name: Require free general-cell slots for the selected cell's hosts
|
||||
if: ${{ env.ROLLBACK_RESUME != 'true' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
HEADROOM_STATUS=0
|
||||
HEADROOM="$(node dev/scripts/check-relay-same-cap-headroom.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --general-cells "${EXPECTED_GENERAL_CELLS}")" \
|
||||
|| HEADROOM_STATUS=$?
|
||||
echo "${HEADROOM}"
|
||||
test "${HEADROOM_STATUS}" = 0
|
||||
# The pre-drain sample sizes its window from this count.
|
||||
TARGET_HOSTS="$(jq -er '.targetHosts' <<< "${HEADROOM}")"
|
||||
echo "TARGET_HOSTS=${TARGET_HOSTS}" >> "${GITHUB_ENV}"
|
||||
|
||||
# The last read before isolation, so a roll drains on health measured minutes ago, not on a
|
||||
# separate monitor run's verdict. Its window grows with the hosts the drain will re-place;
|
||||
# every later wave samples again. Any trip fails here, before anything is changed. Rollback
|
||||
# skips it: getting off a crash-looping image must not wait for the crashes to stop.
|
||||
- name: Sample fleet health for a window sized to this drain
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-pre-drain-sample -- \
|
||||
--target-cell-id "${TARGET_CELL_ID}" \
|
||||
--target-hosts "${TARGET_HOSTS}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \
|
||||
--wave-index "${WAVE_INDEX}" \
|
||||
--selector-wave-delta "${SELECTOR_WAVE_DELTA}"
|
||||
|
||||
- name: Finish read-only verification
|
||||
if: ${{ inputs.mode == 'verify' }}
|
||||
run: echo 'Exact same-cap rollback point verified.'
|
||||
|
||||
# The sample can age the first one-hour token by up to 20 min; the drain's restart-safe
|
||||
# wait (20 min at the default pace, 35 at the slowest) gets a fresh one.
|
||||
- id: drain-auth
|
||||
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Reversibly isolate and drain only the selected cell
|
||||
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
|
||||
id: drain
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.drain-auth.outputs.id_token }}
|
||||
run: |
|
||||
# Before any mutation: a cell whose image caps the pace below this one stops untouched.
|
||||
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode pace-check \
|
||||
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}"
|
||||
# Opens the window the report-only shadow health gate below judges this cell over.
|
||||
echo "drain-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
|
||||
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
# A cell isolated by a failed canary is already migration-only, so
|
||||
# isolate is a no-op there that does not advance the selector; the
|
||||
# result's generation is authoritative either way.
|
||||
ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
|
||||
echo "${ISOLATE_RESULT}"
|
||||
# Isolating a migration-only cell must be a read-only no-op; a change here would
|
||||
# mean the live class is not the one this wave planned around.
|
||||
if test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
jq -e '.changed == false' <<< "${ISOLATE_RESULT}" >/dev/null
|
||||
fi
|
||||
# The director re-places hosts off a cell only when the isolate stamped it, so a
|
||||
# script too old to ask for the stamp produces today's behaviour and the canary
|
||||
# reads as "the fix did nothing" with nothing to tell that from a wrong premise.
|
||||
jq -e '.rollIsolated == true' <<< "${ISOLATE_RESULT}" >/dev/null
|
||||
ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
|
||||
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}"
|
||||
DRAIN_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain \
|
||||
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}")"
|
||||
echo "${DRAIN_RESULT}"
|
||||
# What the cell accepted, which is 0 when an older image fell back to an unpaced drain.
|
||||
echo "drain-applied-pace-window-ms=$(jq -er '.paceWindowMs' <<< "${DRAIN_RESULT}")" \
|
||||
>> "${GITHUB_OUTPUT}"
|
||||
# Restart-safe means an empty runtime that lasts past the drain's pace window; leases
|
||||
# held by unplaceable hosts are reported, not waited on (c28, 2026-10-01).
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat either --admission migration-only --draining required \
|
||||
--activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \
|
||||
--timeout-ms "$((900000 + DRAIN_PACE_WINDOW_MS))"
|
||||
echo "drain-settled-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- id: capacity-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
|
||||
- name: Require converged Terraform state and a stable MIG on resume
|
||||
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }}
|
||||
shell: bash
|
||||
env:
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
# A cell on the root pool default emits no pool line, so pin one only where it exists.
|
||||
POOL_ARGUMENTS=()
|
||||
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
|
||||
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
|
||||
fi
|
||||
# Zero resource changes prove the prior run's apply completed and no
|
||||
# restart will follow, keeping the incarnation check honest. Root
|
||||
# outputs may lag a targeted apply, so judge resource_changes only.
|
||||
# The cell backend service is deliberately not targeted: the capacity role has no
|
||||
# compute.backendServices.update, so naming it fails the apply after the MIG has
|
||||
# rolled. Its drain and log_config settings ship in a root apply by a principal
|
||||
# that holds the permission.
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/production.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
|
||||
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
-out="${RUNNER_TEMP}/relay-same-cap-resume.tfplan"
|
||||
if ! terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
|
||||
| jq -e '[.resource_changes[]?
|
||||
| select(.change.actions | any(. != "no-op" and . != "read"))]
|
||||
| length == 0' >/dev/null
|
||||
then
|
||||
# An apply that failed before its template apply also resumes here
|
||||
# (the cell still serves the rollback image), and repo drift since
|
||||
# the cell's last roll (for example newly added rehome trust
|
||||
# config) then legitimately replaces the template. Nothing is
|
||||
# applied on resume either way, so accept exactly the drift the
|
||||
# reviewed validator would let a real apply ship for the image the
|
||||
# cell already serves: the template leaves and re-enters the
|
||||
# rollback image, as exactly the template-and-MIG change pair.
|
||||
terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
|
||||
| jq -r '"resume found unconverged resources: " +
|
||||
([.resource_changes[]?
|
||||
| select(.change.actions | any(. != "no-op" and . != "read"))
|
||||
| .address] | join(","))'
|
||||
echo 'requiring reviewed rollback-image drift'
|
||||
RESUME_REVIEW="$(terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${DESIRED_IMAGE}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
"${POOL_ARGUMENTS[@]}")"
|
||||
echo "${RESUME_REVIEW}"
|
||||
jq -e '.changes == 2' <<< "${RESUME_REVIEW}" >/dev/null
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
|
||||
- name: Apply only the selected same-cap template and MIG
|
||||
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
|
||||
id: apply
|
||||
shell: bash
|
||||
env:
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
# A cell on the root pool default emits no pool line, so pin one only where it exists.
|
||||
POOL_ARGUMENTS=()
|
||||
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
|
||||
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
|
||||
fi
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/production.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
|
||||
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
-out="${RUNNER_TEMP}/relay-same-cap.tfplan"
|
||||
PLAN_REVIEW="$(terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-same-cap.tfplan" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${PLAN_ROLLBACK_IMAGE}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
"${POOL_ARGUMENTS[@]}")"
|
||||
echo "${PLAN_REVIEW}"
|
||||
# Stamped before the apply, not after it: the new container announces its listener while
|
||||
# the MIG is still converging, so a bound taken at the end of this step is already past
|
||||
# the announcement the shadow gate looks for.
|
||||
echo "apply-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
|
||||
terraform -chdir=infra/terraform apply -auto-approve \
|
||||
"${RUNNER_TEMP}/relay-same-cap.tfplan"
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
# A stranded cell already runs the reviewed template, so the apply above replaces
|
||||
# no instance and the drain flag, which only a restart clears, would survive the
|
||||
# whole wave. Fewer than the template-and-MIG pair means the template stayed put,
|
||||
# including a MIG-only reconciliation. Recreate its one instance from the MIG's
|
||||
# current template; a rolling action would rewrite the MIG's version name outside
|
||||
# Terraform, and every later targeted plan would carry that revert.
|
||||
if test "${ROLLBACK_STAGE}" = stranded \
|
||||
&& jq -e '.changes < 2' <<< "${PLAN_REVIEW}" >/dev/null; then
|
||||
STRANDED_INSTANCE="$(gcloud compute instance-groups managed list-instances \
|
||||
"${MIG_NAME}" --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
|
||||
--format=json \
|
||||
| jq -er 'if length == 1 then .[0].instance | split("/") | last
|
||||
else error("stranded cell MIG does not have exactly one instance") end')"
|
||||
gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \
|
||||
--instances "${STRANDED_INSTANCE}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --quiet
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
fi
|
||||
# Recorded for the operator comparing verdicts; the gate's boot search opens at the
|
||||
# apply-started-at stamp above, not here.
|
||||
echo "apply-completed-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- id: post-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Verify new incarnation, exact image, protocol, and durable safety
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
id: verify-target
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
|
||||
run: |
|
||||
# Rides out one DB stall: 5-7 s, then readiness grace and two 10 s LB
|
||||
# health checks (c18, 10-07 18:49Z: 503 "no healthy upstream", healthy
|
||||
# 16 s later). A cell still down after 60 s fails; other 4xx fail fast.
|
||||
# Retried in this loop: the runner's 7.81 client (Ubuntu 22.04) never
|
||||
# retries under --fail-with-body, so its --retry flags were inert.
|
||||
admin_post() {
|
||||
local out="${RUNNER_TEMP}/$1.json" window=60 delay=5 status
|
||||
local deadline=$((SECONDS + window))
|
||||
while true; do
|
||||
# A refused or timed-out attempt writes no body; never report the last one's.
|
||||
rm -f "${out}"
|
||||
status="$(curl --silent --show-error --max-time 30 --output "${out}" \
|
||||
--write-out '%{http_code}' --request POST "$2" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data "$3")" || status=000
|
||||
case "${status}" in
|
||||
2??) cat "${out}"; return 0 ;;
|
||||
000|408|429|5??) ((SECONDS + delay < deadline)) || break ;;
|
||||
*) break ;;
|
||||
esac
|
||||
sleep "${delay}"
|
||||
done
|
||||
echo "admin_post $1: HTTP ${status}" >&2
|
||||
cat "${out}" >&2 2>/dev/null || true
|
||||
return 1
|
||||
}
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh --admission migration-only --draining forbidden \
|
||||
--activity allowed --expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" --timeout-ms 900000
|
||||
TARGET_RUNTIME="$(admin_post target-runtime \
|
||||
"${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
|
||||
jq -e --arg digest "${DESIRED_IMAGE_DIGEST}" \
|
||||
--argjson protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
'.imageDigest == $digest and (.regionalRehomeProtocol // 0) == $protocol' \
|
||||
<<< "${TARGET_RUNTIME}" >/dev/null
|
||||
TARGET_DIRECTOR_STATUS="$(admin_post target-cell-status \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
"$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
|
||||
TARGET_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \
|
||||
<<< "${TARGET_DIRECTOR_STATUS}")"
|
||||
if test "${ROLLBACK_RESUME}" = true; then
|
||||
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
# No restart happened; the incarnation legitimately stays put.
|
||||
test "${TARGET_INCARNATION}" = "${SOURCE_INCARNATION}"
|
||||
else
|
||||
test "${TARGET_INCARNATION}" != "${SOURCE_INCARNATION}"
|
||||
fi
|
||||
echo "TARGET_INCARNATION=${TARGET_INCARNATION}" >> "${GITHUB_ENV}"
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_ISOLATE}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \
|
||||
--expected-general-cells "${ISOLATED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
|
||||
| jq -e '.control.enabled == false' >/dev/null
|
||||
echo "verify-ended-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Prove exact per-host trust and idempotent no-neighbor behavior
|
||||
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol != '0') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol != '0')) }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/probe-relay-rehome-trust.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-id "${TARGET_CELL_ID}" \
|
||||
--cell-incarnation "${TARGET_INCARNATION}"
|
||||
|
||||
- name: Restore only the verified selected cell to its entry admission
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
|
||||
run: |
|
||||
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
# Activating a migration-only cell would promote it to a serving cell for good, so
|
||||
# restore it with the idempotent isolate that reports the authoritative generation.
|
||||
if test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
RESTORE_MODE=isolate
|
||||
else
|
||||
RESTORE_MODE=activate
|
||||
fi
|
||||
RESTORE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode "${RESTORE_MODE}")"
|
||||
echo "${RESTORE_RESULT}"
|
||||
SELECTOR_GENERATION_AFTER_RESTORE="$(jq -er '.generation' \
|
||||
<<< "${RESTORE_RESULT}")"
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh --admission "${ENTRY_ADMISSION}" \
|
||||
--draining forbidden --activity allowed \
|
||||
--expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_RESTORE}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${RESTORED_MIGRATION_CELLS}" \
|
||||
--expected-general-cells "${RESTORED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
|
||||
| jq -e '.control.enabled == false' >/dev/null
|
||||
|
||||
# Report only: this evaluates the oracles an operator reads by hand after a cell and records
|
||||
# what it would have called, so its verdicts can be compared with the operator's over a full
|
||||
# roll before it is ever allowed to block. Two independent guarantees keep it inert: the
|
||||
# script exits 0 on every verdict, and continue-on-error keeps even a crash off the job's
|
||||
# outcome. The failsafe below therefore cannot fire on anything this step observes.
|
||||
#
|
||||
# It runs after the restore, not before it, for two reasons: the cell goes back into
|
||||
# admission on exactly today's schedule rather than waiting out a minute of log reads, and
|
||||
# the window it judges has closed by then, so Cloud Logging's ingestion lag is behind it.
|
||||
# These are fleet oracles anyway; when this does gate, what it gates is the next cell.
|
||||
- name: Shadow health gate (report only)
|
||||
id: shadow-gate
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
continue-on-error: true
|
||||
# continue-on-error bounds this step's contribution to the job outcome, not its clock,
|
||||
# and its reads are serialised. A timed-out step is a failed step, which continue-on-error
|
||||
# absorbs; without this bound a Logging 429 storm or an expired credential makes every
|
||||
# read cost its full retry budget and can push the job past timeout-minutes, and a
|
||||
# cancelled job takes the whole wave with it. A healthy gate is already minutes of
|
||||
# serial reads, so both bounds sit above that: the script settles at seven minutes and
|
||||
# reaching this eight is the pathological case. Eight on top of a cell of ~14 minutes
|
||||
# (~45 at the slowest pace) still leaves the job's 120 minutes intact.
|
||||
timeout-minutes: 8
|
||||
env:
|
||||
DRAIN_STARTED_AT: ${{ steps.drain.outputs.drain-started-at }}
|
||||
DRAIN_SETTLED_AT: ${{ steps.drain.outputs.drain-settled-at }}
|
||||
DRAIN_APPLIED_PACE_WINDOW_MS: ${{ steps.drain.outputs.drain-applied-pace-window-ms }}
|
||||
APPLY_STARTED_AT: ${{ steps.apply.outputs.apply-started-at }}
|
||||
APPLY_COMPLETED_AT: ${{ steps.apply.outputs.apply-completed-at }}
|
||||
VERIFY_ENDED_AT: ${{ steps.verify-target.outputs.verify-ended-at }}
|
||||
SHADOW_GATE_DIRECTORY: ${{ runner.temp }}/relay-same-cap-shadow-gate
|
||||
SHADOW_GATE_NAME: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json
|
||||
run: |
|
||||
mkdir -p "${SHADOW_GATE_DIRECTORY}"
|
||||
node dev/scripts/relay-same-cap-shadow-gate.mjs \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--cell-host "${TARGET_HOSTNAME}.relay.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" \
|
||||
--director-service orca-cloud-relay \
|
||||
--drain-started-at "${DRAIN_STARTED_AT}" \
|
||||
--apply-started-at "${APPLY_STARTED_AT}" \
|
||||
--apply-completed-at "${APPLY_COMPLETED_AT}" \
|
||||
--verify-ended-at "${VERIFY_ENDED_AT}" \
|
||||
--drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \
|
||||
--drain-applied-pace-window-ms "${DRAIN_APPLIED_PACE_WINDOW_MS}" \
|
||||
--drain-settled-at "${DRAIN_SETTLED_AT}" \
|
||||
--target-hosts "${TARGET_HOSTS:-}" \
|
||||
--summary-file "${GITHUB_STEP_SUMMARY}" \
|
||||
--output-file "${SHADOW_GATE_DIRECTORY}/${SHADOW_GATE_NAME}"
|
||||
|
||||
- name: Publish the shadow health gate verdict
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
continue-on-error: true
|
||||
# One small JSON file; a retrying upload must not spend the wave's remaining minutes either.
|
||||
timeout-minutes: 2
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json
|
||||
path: ${{ runner.temp }}/relay-same-cap-shadow-gate
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
overwrite: true
|
||||
|
||||
- id: cleanup-auth
|
||||
if: ${{ failure() && inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Keep a failed cell isolated and rehome disabled
|
||||
if: ${{ failure() && inputs.mode != 'verify' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }}
|
||||
run: |
|
||||
test "${MUTATION_STARTED:-false}" = true || exit 0
|
||||
ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
|
||||
echo "${ISOLATE_RESULT}"
|
||||
# Same reason as the isolate step above: a failed wave leaves this cell isolated
|
||||
# deliberately, and the stamp is what lets its hosts leave.
|
||||
jq -e '.rollIsolated == true' <<< "${ISOLATE_RESULT}" >/dev/null
|
||||
# The isolate result carries the authoritative post-isolate generation;
|
||||
# fixed offsets are wrong whenever an earlier isolate was a no-op.
|
||||
FAILSAFE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${FAILSAFE_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \
|
||||
--expected-general-cells "${ISOLATED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_REHOME_GENERATION}"
|
||||
@@ -1,433 +0,0 @@
|
||||
name: Deploy Relay Production Same-Cap
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Verify, roll one canary, roll a bounded batch, or roll back
|
||||
required: true
|
||||
default: verify
|
||||
type: choice
|
||||
options: [verify, canary-apply, batch-apply, rollback]
|
||||
cell-ids:
|
||||
description: Ordered comma-separated serving cells; one canary or two to ten batch cells
|
||||
required: true
|
||||
type: string
|
||||
target-image-digest:
|
||||
description: Exact immutable compatibility image digest
|
||||
required: true
|
||||
type: string
|
||||
rollback-image-digest:
|
||||
description: Exact immutable currently serving rollback digest
|
||||
required: true
|
||||
type: string
|
||||
target-rehome-protocol:
|
||||
description: Exact target regional-rehome protocol
|
||||
required: true
|
||||
default: '1'
|
||||
type: choice
|
||||
options: ['0', '1', '3']
|
||||
rollback-rehome-protocol:
|
||||
description: Exact rollback regional-rehome protocol
|
||||
required: true
|
||||
default: '0'
|
||||
type: choice
|
||||
options: ['0', '1', '3']
|
||||
expected-selector-generation:
|
||||
description: Exact selector generation before the first cell
|
||||
required: true
|
||||
type: string
|
||||
expected-existing-only-cells:
|
||||
description: Exact existing-only membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-migration-only-cells:
|
||||
description: Exact migration-only membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-general-cells:
|
||||
description: Exact general membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-rehome-generation:
|
||||
description: Exact durable regional-rehome control generation; it must be disabled
|
||||
required: true
|
||||
type: string
|
||||
drain-pace-window-ms:
|
||||
description: Window the cell spreads its drain over; below 300000 only for US general cells; any other value named in the confirmation
|
||||
required: true
|
||||
default: '300000'
|
||||
type: choice
|
||||
options: ['300000', '60000', '30000', '900000', '1200000']
|
||||
canary-run-id:
|
||||
description: Successful same-code canary in this rehome control generation; reusable across batches
|
||||
required: false
|
||||
type: string
|
||||
confirmation:
|
||||
description: Exact digest-and-cell-bound mutation confirmation
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
gate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
# Headroom for the full-history checkout the canary provenance check needs.
|
||||
timeout-minutes: 15
|
||||
environment: production
|
||||
outputs:
|
||||
cells: ${{ steps.wave.outputs.cells }}
|
||||
job-mode: ${{ steps.wave.outputs.job-mode }}
|
||||
steps:
|
||||
# Full history: the canary authority a batch verifies is sealed at an ancestor commit, and
|
||||
# the provenance check fails closed on a commit a shallow clone left out.
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with: { node-version: 24, check-latest: true }
|
||||
|
||||
- id: wave
|
||||
env:
|
||||
MODE: ${{ inputs.mode }}
|
||||
CELL_IDS: ${{ inputs.cell-ids }}
|
||||
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
||||
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
||||
DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }}
|
||||
run: |
|
||||
CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \
|
||||
--mode "${MODE}" --cell-ids "${CELL_IDS}" \
|
||||
--target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \
|
||||
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}" \
|
||||
--drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}")"
|
||||
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
|
||||
if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then
|
||||
echo 'job-mode=apply' >> "${GITHUB_OUTPUT}"
|
||||
else
|
||||
echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
|
||||
- name: Download exact prior canary authority
|
||||
if: ${{ inputs.mode == 'batch-apply' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-same-cap-canary-${{ inputs.canary-run-id }}
|
||||
path: ${{ runner.temp }}/relay-same-cap-canary
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.canary-run-id }}
|
||||
|
||||
- name: Verify canary authority against this batch
|
||||
if: ${{ inputs.mode == 'batch-apply' }}
|
||||
env:
|
||||
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
||||
CELL_IDS: ${{ inputs.cell-ids }}
|
||||
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
||||
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
||||
SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
||||
REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
||||
DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }}
|
||||
run: |
|
||||
node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
|
||||
--file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
|
||||
--commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
|
||||
--cell-ids "${CELL_IDS}" \
|
||||
--target-digest "${TARGET_DIGEST}" \
|
||||
--rollback-digest "${ROLLBACK_DIGEST}" \
|
||||
--drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \
|
||||
--selector-generation "${SELECTOR_GENERATION}" \
|
||||
--rehome-generation "${REHOME_GENERATION}"
|
||||
|
||||
cell_1:
|
||||
needs: gate
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[0] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '0'
|
||||
secrets: inherit
|
||||
|
||||
cell_2:
|
||||
if: ${{ needs.cell_1.result == 'success' && fromJSON(needs.gate.outputs.cells)[1] != null }}
|
||||
needs: [gate, cell_1]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[1] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '1'
|
||||
secrets: inherit
|
||||
|
||||
cell_3:
|
||||
if: ${{ needs.cell_2.result == 'success' && fromJSON(needs.gate.outputs.cells)[2] != null }}
|
||||
needs: [gate, cell_2]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[2] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '2'
|
||||
secrets: inherit
|
||||
|
||||
cell_4:
|
||||
if: ${{ needs.cell_3.result == 'success' && fromJSON(needs.gate.outputs.cells)[3] != null }}
|
||||
needs: [gate, cell_3]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[3] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '3'
|
||||
secrets: inherit
|
||||
|
||||
cell_5:
|
||||
if: ${{ needs.cell_4.result == 'success' && fromJSON(needs.gate.outputs.cells)[4] != null }}
|
||||
needs: [gate, cell_4]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[4] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '4'
|
||||
secrets: inherit
|
||||
|
||||
cell_6:
|
||||
if: ${{ needs.cell_5.result == 'success' && fromJSON(needs.gate.outputs.cells)[5] != null }}
|
||||
needs: [gate, cell_5]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[5] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '5'
|
||||
secrets: inherit
|
||||
|
||||
cell_7:
|
||||
if: ${{ needs.cell_6.result == 'success' && fromJSON(needs.gate.outputs.cells)[6] != null }}
|
||||
needs: [gate, cell_6]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[6] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '6'
|
||||
secrets: inherit
|
||||
|
||||
cell_8:
|
||||
if: ${{ needs.cell_7.result == 'success' && fromJSON(needs.gate.outputs.cells)[7] != null }}
|
||||
needs: [gate, cell_7]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[7] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '7'
|
||||
secrets: inherit
|
||||
|
||||
cell_9:
|
||||
if: ${{ needs.cell_8.result == 'success' && fromJSON(needs.gate.outputs.cells)[8] != null }}
|
||||
needs: [gate, cell_8]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[8] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '8'
|
||||
secrets: inherit
|
||||
|
||||
cell_10:
|
||||
if: ${{ needs.cell_9.result == 'success' && fromJSON(needs.gate.outputs.cells)[9] != null }}
|
||||
needs: [gate, cell_9]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[9] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '9'
|
||||
secrets: inherit
|
||||
|
||||
seal_canary:
|
||||
if: ${{ inputs.mode == 'canary-apply' }}
|
||||
needs: [gate, cell_1]
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with: { node-version: 24, check-latest: true }
|
||||
|
||||
# Its pace checks are what let a batch drain faster than the default; a missing artifact
|
||||
# seals UNVERIFIED, which only the default accepts. Named from the gate's normalized cell,
|
||||
# as cell_1 uploaded it, so a padded cell-ids input cannot miss it.
|
||||
- name: Download the canary cell's shadow health verdict
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-same-cap-shadow-gate-${{ fromJSON(needs.gate.outputs.cells)[0] }}-${{ github.run_id }}.json
|
||||
path: ${{ runner.temp }}/relay-same-cap-shadow-gate
|
||||
|
||||
- name: Seal exact successful canary authority
|
||||
run: |
|
||||
mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary"
|
||||
node dev/scripts/relay-production-same-cap-wave.mjs create-canary \
|
||||
--cell-id "${{ fromJSON(needs.gate.outputs.cells)[0] }}" \
|
||||
--target-digest "${{ inputs.target-image-digest }}" \
|
||||
--rollback-digest "${{ inputs.rollback-image-digest }}" \
|
||||
--confirmation "${{ inputs.confirmation }}" \
|
||||
--drain-pace-window-ms "${{ inputs.drain-pace-window-ms }}" \
|
||||
--shadow-report "${RUNNER_TEMP}/relay-same-cap-shadow-gate/relay-same-cap-shadow-gate-${{ fromJSON(needs.gate.outputs.cells)[0] }}-${GITHUB_RUN_ID}.json" \
|
||||
--commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \
|
||||
--selector-generation "${{ inputs.expected-selector-generation }}" \
|
||||
--rehome-generation "${{ inputs.expected-rehome-generation }}" \
|
||||
> "${RUNNER_TEMP}/relay-same-cap-canary/authority.json"
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-same-cap-canary-${{ github.run_id }}
|
||||
path: ${{ runner.temp }}/relay-same-cap-canary/authority.json
|
||||
retention-days: 30
|
||||
if-no-files-found: error
|
||||
|
||||
# Every cell job re-enters the run's lease with release: 'false'; only this job frees it.
|
||||
release_lease:
|
||||
if: always()
|
||||
needs:
|
||||
- gate
|
||||
- cell_1
|
||||
- cell_2
|
||||
- cell_3
|
||||
- cell_4
|
||||
- cell_5
|
||||
- cell_6
|
||||
- cell_7
|
||||
- cell_8
|
||||
- cell_9
|
||||
- cell_10
|
||||
- seal_canary
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 10
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
release: 'true'
|
||||
@@ -1,220 +0,0 @@
|
||||
name: Deploy Relay Production Candidate
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source-cell-id:
|
||||
description: Existing Terraform cell ID to evacuate
|
||||
required: true
|
||||
type: string
|
||||
target-cell-id:
|
||||
description: Distinct Terraform candidate cell ID
|
||||
required: true
|
||||
type: string
|
||||
mode:
|
||||
description: Audit/preflight are read-only; recover/continue resume committed work; disable/enable/reset/execute mutate admission
|
||||
required: true
|
||||
default: preflight
|
||||
type: choice
|
||||
options:
|
||||
- audit
|
||||
- preflight
|
||||
- recover-forward
|
||||
- continue-evacuation
|
||||
- disable-cell
|
||||
- enable-empty-cell
|
||||
- reset-empty-candidate
|
||||
- execute
|
||||
confirmation:
|
||||
description: Enter RECOVER_FORWARD, CONTINUE_EVACUATION, DISABLE_CELL, ENABLE_CELL, RESET_CANDIDATE, or EVACUATE for the matching mutation
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-id:
|
||||
description: Successful fresh dry-run monitor workflow run ID
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-attempt:
|
||||
description: Exact dry-run monitor workflow attempt
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
candidate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain
|
||||
SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
|
||||
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Require fresh dry-run evidence reference
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
run: |
|
||||
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
|
||||
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
|
||||
- name: Download private dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Verify dry-run artifact before cloud authentication
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run
|
||||
|
||||
- name: Reject previously consumed dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
COUNT="$(gh api \
|
||||
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
||||
--jq '.total_count')"
|
||||
test "${COUNT}" = "0"
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
|
||||
- name: Require explicit mutation confirmation
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if [[ "${DEPLOY_MODE}" = "execute" ]]; then
|
||||
test "${CONFIRMATION}" = "EVACUATE"
|
||||
elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then
|
||||
test "${CONFIRMATION}" = "RECOVER_FORWARD"
|
||||
elif [[ "${DEPLOY_MODE}" = "continue-evacuation" ]]; then
|
||||
test "${CONFIRMATION}" = "CONTINUE_EVACUATION"
|
||||
elif [[ "${DEPLOY_MODE}" = "disable-cell" ]]; then
|
||||
test "${CONFIRMATION}" = "DISABLE_CELL"
|
||||
elif [[ "${DEPLOY_MODE}" = "enable-empty-cell" ]]; then
|
||||
test "${CONFIRMATION}" = "ENABLE_CELL"
|
||||
else
|
||||
test "${CONFIRMATION}" = "RESET_CANDIDATE"
|
||||
fi
|
||||
|
||||
- name: Read reviewed Terraform topology
|
||||
run: |
|
||||
node dev/scripts/infra.mjs init --env production
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
|
||||
echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify fresh dry-run evidence against live selector
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run \
|
||||
--mutation-mode "${DEPLOY_MODE}" \
|
||||
--source-cell-id "${SOURCE_CELL_ID}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}"
|
||||
|
||||
- name: Recheck all live safety signals
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
|
||||
|
||||
- name: Create single-use dry-run marker
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
|
||||
printf '%s\n' "${GITHUB_RUN_ID}" \
|
||||
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
|
||||
|
||||
- name: Consume dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Preflight or evacuate exact GCE candidate
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/deploy-relay-gce-candidate.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--admin-audience "${ADMIN_AUDIENCE}" \
|
||||
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
|
||||
--source-cell-id "${SOURCE_CELL_ID}" \
|
||||
--target-cell-id "${TARGET_CELL_ID}" \
|
||||
--runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--mode "${DEPLOY_MODE}"
|
||||
@@ -1,112 +0,0 @@
|
||||
name: Deploy Relay Staging GCE Candidate
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source-cell-id:
|
||||
description: Existing Terraform cell ID to evacuate
|
||||
required: true
|
||||
type: string
|
||||
target-cell-id:
|
||||
description: Distinct Terraform candidate cell ID
|
||||
required: true
|
||||
type: string
|
||||
mode:
|
||||
description: Preflight is read-only; reset repairs an empty candidate; execute evacuates
|
||||
required: true
|
||||
default: preflight
|
||||
type: choice
|
||||
options:
|
||||
- preflight
|
||||
- reset-empty-candidate
|
||||
- execute
|
||||
confirmation:
|
||||
description: Enter RESET_CANDIDATE for reset or EVACUATE for execute
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
candidate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
ADMIN_AUDIENCE: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
|
||||
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Require explicit mutation confirmation
|
||||
if: ${{ inputs.mode != 'preflight' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if [[ "${DEPLOY_MODE}" = "execute" ]]; then
|
||||
test "${CONFIRMATION}" = "EVACUATE"
|
||||
else
|
||||
test "${CONFIRMATION}" = "RESET_CANDIDATE"
|
||||
fi
|
||||
|
||||
- name: Read reviewed Terraform topology
|
||||
run: |
|
||||
node dev/scripts/infra.mjs init --env staging
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
|
||||
echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Preflight or evacuate exact GCE candidate
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/deploy-relay-gce-candidate.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--admin-audience "${ADMIN_AUDIENCE}" \
|
||||
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
|
||||
--source-cell-id "${SOURCE_CELL_ID}" \
|
||||
--target-cell-id "${TARGET_CELL_ID}" \
|
||||
--runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--mode "${DEPLOY_MODE}"
|
||||
@@ -1,242 +0,0 @@
|
||||
name: Deploy Relay Staging
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
expected-image-digest:
|
||||
description: Exact checked-in production Relay sha256 digest to deploy
|
||||
required: true
|
||||
type: string
|
||||
reserve-placement:
|
||||
description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve
|
||||
required: true
|
||||
default: preserve
|
||||
type: choice
|
||||
options: [preserve, 'off', dry-run, 'on']
|
||||
expected-rehome-generation:
|
||||
description: Exact durable regional-rehome generation; it must remain disabled
|
||||
required: true
|
||||
type: string
|
||||
bootstrap-runtime-identity:
|
||||
description: One-time move from the stamped-cell identity to the director identity
|
||||
required: true
|
||||
default: false
|
||||
type: boolean
|
||||
predecessor-image-digest:
|
||||
description: Bootstrap only. Exact immutable serving predecessor digest
|
||||
required: false
|
||||
type: string
|
||||
capacity-cell-id:
|
||||
description: Register this cell's reviewed staging.tfvars cap and bound with the director
|
||||
required: true
|
||||
default: none
|
||||
type: choice
|
||||
options: [none, staging-gce-c1, staging-gce-c2, staging-gce-c3, staging-gce-c4]
|
||||
min-instances:
|
||||
description: Director floor; preserve keeps the serving revision's, so a reserve run's 2 survives
|
||||
required: true
|
||||
default: preserve
|
||||
type: choice
|
||||
options: [preserve, '0', '2']
|
||||
confirmation:
|
||||
description: BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY for a bootstrap, ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE with a shape baseline
|
||||
required: false
|
||||
type: string
|
||||
shape-baseline-revision:
|
||||
description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts
|
||||
required: false
|
||||
type: string
|
||||
shadow-seat-feed-cells:
|
||||
description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused
|
||||
required: true
|
||||
default: preserve
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# Staging deploy, candidate, auth, and power operations must never overlap.
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
|
||||
DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging
|
||||
REPOSITORY_ID: orca-cloud
|
||||
IMAGE_NAME: relay
|
||||
EXPECTED_IMAGE_DIGEST: ${{ inputs.expected-image-digest }}
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
# The director must trust the identity every staging Relay workflow authenticates as.
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
ASIA_PROOF_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
|
||||
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: orca-cloud-staging-relay-dir@onorca-cloud-staging.iam.gserviceaccount.com
|
||||
PREDECESSOR_RUNTIME_SERVICE_ACCOUNT: orca-cloud-staging-relay@onorca-cloud-staging.iam.gserviceaccount.com
|
||||
REHOME_AUDIENCE: https://relay-staging.onorca.dev/v1/admin/host-drain
|
||||
EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
||||
BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }}
|
||||
PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }}
|
||||
CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }}
|
||||
MIN_INSTANCES: ${{ inputs.min-instances }}
|
||||
SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Require the expected immutable image and identity request
|
||||
shell: bash
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${EXPECTED_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${BOOTSTRAP_RUNTIME_IDENTITY}" =~ ^(true|false)$ ]]
|
||||
[[ "${MIN_INSTANCES}" =~ ^(preserve|0|2)$ ]]
|
||||
if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then
|
||||
[[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test "${CAPACITY_CELL_ID}" = none
|
||||
test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY
|
||||
else
|
||||
test -z "${PREDECESSOR_IMAGE_DIGEST}"
|
||||
fi
|
||||
if test -n "${SHAPE_BASELINE_REVISION}"; then
|
||||
[[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-staging-[0-9]{5}-[a-z0-9]{3}$ ]]
|
||||
test "${BOOTSTRAP_RUNTIME_IDENTITY}" = false
|
||||
test "${CAPACITY_CELL_ID}" = none
|
||||
test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE
|
||||
fi
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
terraform_wrapper: false
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Bind the request to the checked-in staging C4 image
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
terraform -chdir=infra/terraform init -reconfigure \
|
||||
-backend-config=backend/staging.hcl -input=false
|
||||
IMAGE="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'var.relay_gce_cells["staging-gce-c4"].image' | jq -er '.')"
|
||||
test "${IMAGE}" = \
|
||||
"${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${EXPECTED_IMAGE_DIGEST}"
|
||||
echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
|
||||
if test "${CAPACITY_CELL_ID}" != none; then
|
||||
# The deploy script refuses any topology change beyond this one cell's cap and bound.
|
||||
DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'local.relay_director_cells_json' | jq -er '.')"
|
||||
jq -e --arg cell "${CAPACITY_CELL_ID}" \
|
||||
'any(.[]; .id == $cell and .connectionHardCap != null)' \
|
||||
<<< "${DESIRED_CELLS_JSON}" >/dev/null
|
||||
echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Require the mirrored immutable image
|
||||
run: |
|
||||
DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \
|
||||
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${DIGEST}" = "${EXPECTED_IMAGE_DIGEST}"
|
||||
|
||||
- name: Deploy director blue/green
|
||||
env:
|
||||
# An image change reads every cell's admit mode before the traffic move.
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
RESERVE_PLACEMENT: ${{ inputs.reserve-placement }}
|
||||
SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }}
|
||||
run: |
|
||||
regional_version="$(gcloud secrets versions describe latest \
|
||||
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \
|
||||
--format='value(name)' | awk -F/ '{print $NF}')"
|
||||
[[ "${regional_version}" =~ ^[1-9][0-9]*$ ]]
|
||||
RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
|
||||
identity_args=()
|
||||
if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then
|
||||
identity_args=(
|
||||
--bootstrap-runtime-identity true
|
||||
--predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}"
|
||||
--predecessor-image-digest "${PREDECESSOR_IMAGE_DIGEST}"
|
||||
)
|
||||
fi
|
||||
# Omitted, the deploy script inherits the serving revision's floor.
|
||||
floor_args=()
|
||||
if test "${MIN_INSTANCES}" != preserve; then
|
||||
floor_args=(--min-instances "${MIN_INSTANCES}")
|
||||
fi
|
||||
shape_args=()
|
||||
if test -n "${SHAPE_BASELINE_REVISION}"; then
|
||||
shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}")
|
||||
fi
|
||||
topology_args=()
|
||||
if test "${CAPACITY_CELL_ID}" != none; then
|
||||
topology_args=(
|
||||
--capacity-cell-id "${CAPACITY_CELL_ID}"
|
||||
--director-cells-json "${DESIRED_CELLS_JSON}"
|
||||
)
|
||||
fi
|
||||
node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service "${DIRECTOR_SERVICE_NAME}" \
|
||||
--image "${IMAGE}" \
|
||||
--role director \
|
||||
--max-instances 2 \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--deploy-service-account "${DEPLOY_SERVICE_ACCOUNT}" \
|
||||
--asia-proof-service-account "${ASIA_PROOF_SERVICE_ACCOUNT}" \
|
||||
--regional-placement-secret-version "${regional_version}" \
|
||||
"${floor_args[@]}" \
|
||||
--runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
"${identity_args[@]}" \
|
||||
"${shape_args[@]}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience "${REHOME_AUDIENCE}" \
|
||||
--rehome-control-origin https://relay-staging.onorca.dev \
|
||||
--expected-rehome-generation "${EXPECTED_REHOME_GENERATION}" \
|
||||
"${topology_args[@]}" \
|
||||
--admin-audience https://relay-staging.onorca.dev/v1/admin/drain \
|
||||
--reserve-placement "${RESERVE_PLACEMENT}" \
|
||||
--shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}" \
|
||||
--release-id "${RELEASE_ID}"
|
||||
|
||||
- name: Smoke director health
|
||||
run: |
|
||||
URL="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(status.url)')"
|
||||
node dev/scripts/smoke-relay.mjs "${URL}"
|
||||
@@ -1,90 +0,0 @@
|
||||
name: Monitor Relay Cell Clock Skew
|
||||
|
||||
# Why: the 2026-08-01 sustained HOST_OFFLINE incident traced to one cell's
|
||||
# clock running ~100ms ahead, which deterministically failed every host
|
||||
# challenge under a zero-tolerance freshness check. /health and /ready cannot
|
||||
# see clock skew; this monitor alarms before drift reaches the (now 2s)
|
||||
# client tolerance.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '17 * * * *'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
skew:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: cloud/infra/terraform/environments/production.tfvars
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
|
||||
- name: Measure Date-header skew for every relay cell
|
||||
run: |
|
||||
set -u
|
||||
# Date headers carry whole seconds; compare floored seconds on both
|
||||
# sides so perfect sync reads 0/±1 and never flaps. An absolute
|
||||
# floor-skew of >= 2 means real drift of at least ~1s — approaching
|
||||
# the client's 2s challenge tolerance. Millisecond-precision deltas
|
||||
# come from the desktop's named-check logging when activations fail.
|
||||
ALARM_S=2
|
||||
failures=0
|
||||
reachable=0
|
||||
unserved=0
|
||||
# Every relay_gce_cells hostname (the block relayOpsCellsFromTerraform reads), fenced
|
||||
# cells included, so a newly declared cell is probed from its first run.
|
||||
cells="$(awk '
|
||||
/^relay_gce_cells[ \t]*=[ \t]*\{/ { inside = 1; next }
|
||||
inside && /^\}/ { exit }
|
||||
inside && /^[ \t]*hostname[ \t]*=/ { split($0, quoted, "\""); print quoted[2] }
|
||||
' infra/terraform/environments/production.tfvars || true)"
|
||||
if [ -z "${cells}" ]; then
|
||||
echo "::error::read no relay cells from production.tfvars; monitor blind"
|
||||
exit 1
|
||||
fi
|
||||
for cell in ${cells}; do
|
||||
url="https://${cell}.relay.onorca.dev/health"
|
||||
# Why: *.relay.onorca.dev is a wildcard, so the load balancer answers with its own
|
||||
# accurate Date for a fenced, dead, or never-provisioned cell. Timing that reads as
|
||||
# perfect sync. Only an HTTP 200 is the relay process itself answering, so only a
|
||||
# 200 carries a clock worth judging.
|
||||
response="$(curl -sS -D - -o /dev/null --max-time 8 "${url}" 2>/dev/null || true)"
|
||||
status="$(printf '%s' "${response}" | awk 'NR==1 {print $2}')"
|
||||
header="$(printf '%s' "${response}" | tr -d '\r' | grep -i '^date:' || true)"
|
||||
if [ "${status:-000}" != "200" ] || [ -z "${header}" ]; then
|
||||
# Expected for the fenced cells; a dead unfenced cell is caught by the heartbeat
|
||||
# and readiness alerts, not here. Named either way so it is never invisible.
|
||||
echo "${cell}: not serving (status ${status:-none}); no relay clock to judge"
|
||||
unserved=$((unserved + 1))
|
||||
continue
|
||||
fi
|
||||
reachable=$((reachable + 1))
|
||||
server_s="$(date -d "${header#*: }" +%s)"
|
||||
local_s="$(date +%s)"
|
||||
skew=$((server_s - local_s))
|
||||
abs=${skew#-}
|
||||
if [ "${abs}" -ge "${ALARM_S}" ]; then
|
||||
echo "::error::${cell}: clock skew ${skew}s reaches ±${ALARM_S}s alarm"
|
||||
failures=$((failures + 1))
|
||||
else
|
||||
echo "${cell}: skew ${skew}s"
|
||||
fi
|
||||
done
|
||||
echo "cells serving: ${reachable}, not serving: ${unserved}, alarms: ${failures}"
|
||||
if [ "${reachable}" -eq 0 ]; then
|
||||
# Now meaningful: previously the load balancer answered for every name, so this
|
||||
# could never fire and a total fleet outage reported "all healthy".
|
||||
echo "::error::no relay cell is serving; monitor blind"
|
||||
exit 1
|
||||
fi
|
||||
exit "$((failures > 0 ? 1 : 0))"
|
||||
@@ -1,216 +0,0 @@
|
||||
name: Monitor Relay Production Job
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
mode:
|
||||
required: true
|
||||
type: string
|
||||
expected-selector-generation:
|
||||
required: true
|
||||
type: string
|
||||
expected-existing-only-cells:
|
||||
required: true
|
||||
type: string
|
||||
expected-migration-only-cells:
|
||||
required: true
|
||||
type: string
|
||||
expected-general-cells:
|
||||
required: true
|
||||
type: string
|
||||
migration-policy:
|
||||
required: true
|
||||
type: string
|
||||
recovery-source-cell-id:
|
||||
required: true
|
||||
type: string
|
||||
capacity-cell-id:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
monitor:
|
||||
if: ${{ github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 100
|
||||
environment: production
|
||||
env:
|
||||
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
||||
EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
|
||||
EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
|
||||
EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
|
||||
MIGRATION_POLICY: ${{ inputs.migration-policy }}
|
||||
RECOVERY_SOURCE_CELL_ID: ${{ inputs.recovery-source-cell-id }}
|
||||
CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }}
|
||||
INCIDENT_ID: relay-${{ github.run_id }}-${{ inputs.mode }}
|
||||
MONITOR_MODE: ${{ inputs.mode }}
|
||||
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-incident
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- id: prior-attempt
|
||||
if: ${{ github.run_attempt > 1 }}
|
||||
run: echo "value=$((GITHUB_RUN_ATTEMPT - 1))" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Restore prior private monitor state
|
||||
if: ${{ github.run_attempt > 1 }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ steps.prior-attempt.outputs.value }}
|
||||
path: ${{ github.workspace }}/relay-incident
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ github.run_id }}
|
||||
|
||||
- name: Verify restored state provenance
|
||||
if: ${{ github.run_attempt > 1 }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
|
||||
--directory "${OUTPUT_DIRECTORY}" \
|
||||
--incident-id "${INCIDENT_ID}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--run-attempt "${{ steps.prior-attempt.outputs.value }}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode "${MONITOR_MODE}"
|
||||
echo "RESTART_FLAG=--restart" >> "${GITHUB_ENV}"
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- name: Verify exact-audience admin identity
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
|
||||
|
||||
- name: Run read-only relay dry-run
|
||||
if: ${{ inputs.mode == 'dry-run' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm --filter @orca-cloud/relay-ops incident:monitor \
|
||||
--environment production \
|
||||
--incident-id "${INCIDENT_ID}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--migration-policy "${MIGRATION_POLICY}" \
|
||||
--recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
|
||||
--capacity-cell-id "${CAPACITY_CELL_ID}" \
|
||||
--interval-seconds 60 \
|
||||
--output-directory "${OUTPUT_DIRECTORY}" \
|
||||
--duration-minutes 15 \
|
||||
--pre-drain-dry-run \
|
||||
${RESTART_FLAG:-}
|
||||
|
||||
- name: Run first relay monitor segment
|
||||
if: ${{ inputs.mode == 'monitor' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm --filter @orca-cloud/relay-ops incident:monitor \
|
||||
--environment production \
|
||||
--incident-id "${INCIDENT_ID}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--migration-policy "${MIGRATION_POLICY}" \
|
||||
--recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
|
||||
--capacity-cell-id "${CAPACITY_CELL_ID}" \
|
||||
--interval-seconds 60 \
|
||||
--output-directory "${OUTPUT_DIRECTORY}" \
|
||||
--duration-minutes 90 \
|
||||
--max-samples-this-run 45 \
|
||||
${RESTART_FLAG:-}
|
||||
|
||||
- id: google-auth-refresh
|
||||
if: ${{ inputs.mode == 'monitor' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Run remaining relay monitor window
|
||||
if: ${{ inputs.mode == 'monitor' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-refresh.outputs.id_token }}
|
||||
run: |
|
||||
node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
|
||||
pnpm --filter @orca-cloud/relay-ops incident:monitor \
|
||||
--environment production \
|
||||
--incident-id "${INCIDENT_ID}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--migration-policy "${MIGRATION_POLICY}" \
|
||||
--recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
|
||||
--capacity-cell-id "${CAPACITY_CELL_ID}" \
|
||||
--interval-seconds 60 \
|
||||
--output-directory "${OUTPUT_DIRECTORY}" \
|
||||
--duration-minutes 90 \
|
||||
--restart
|
||||
|
||||
- name: Seal private evidence provenance
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs create \
|
||||
--directory "${OUTPUT_DIRECTORY}" \
|
||||
--incident-id "${INCIDENT_ID}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--run-attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode "${MONITOR_MODE}"
|
||||
|
||||
- name: Publish aggregate job summary
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
if [[ -f "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" ]]; then
|
||||
cat "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
else
|
||||
echo "Relay monitor failed before its first aggregate checkpoint." \
|
||||
>> "${GITHUB_STEP_SUMMARY}"
|
||||
fi
|
||||
|
||||
- name: Upload private aggregate evidence
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ github.workspace }}/relay-incident
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
@@ -1,75 +0,0 @@
|
||||
name: Monitor Relay Production
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Run the required 15-minute pre-drain gate or a 90-minute incident watch
|
||||
required: true
|
||||
default: dry-run
|
||||
type: choice
|
||||
options:
|
||||
- dry-run
|
||||
- monitor
|
||||
expected-selector-generation:
|
||||
description: Exact durable admission-selector generation
|
||||
required: true
|
||||
type: string
|
||||
expected-existing-only-cells:
|
||||
description: Exact comma-separated existing-only cells, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-migration-only-cells:
|
||||
description: Exact comma-separated migration-only cells, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-general-cells:
|
||||
description: Exact comma-separated general cells, or none
|
||||
required: true
|
||||
type: string
|
||||
migration-policy:
|
||||
description: Migration checks matched to the intended mutation
|
||||
required: true
|
||||
default: strict
|
||||
type: choice
|
||||
options:
|
||||
- strict
|
||||
- recover-forward
|
||||
- capacity-transition
|
||||
recovery-source-cell-id:
|
||||
description: Existing-only recovery source cell, or none for strict monitoring
|
||||
required: true
|
||||
default: none
|
||||
type: string
|
||||
capacity-cell-id:
|
||||
description: General cell for a capacity-transition monitor, or none
|
||||
required: true
|
||||
default: none
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
monitor:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
uses: ./.github/workflows/cloud-monitor-relay-production-job.yml
|
||||
with:
|
||||
mode: ${{ inputs.mode }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
migration-policy: ${{ inputs.migration-policy }}
|
||||
recovery-source-cell-id: ${{ inputs.recovery-source-cell-id }}
|
||||
capacity-cell-id: ${{ inputs.capacity-cell-id }}
|
||||
@@ -1,558 +0,0 @@
|
||||
name: Operate Relay Asia Admission
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
environment:
|
||||
description: Target Relay environment
|
||||
required: true
|
||||
type: choice
|
||||
options: [staging, production]
|
||||
mode:
|
||||
description: Inspect, initialize, verify, atomically register, promote, or roll back admission
|
||||
required: true
|
||||
default: verify
|
||||
type: choice
|
||||
options: [inspect, initialize, verify, register, configure, promote, rollback]
|
||||
cell-ids:
|
||||
description: Exact reviewed comma-separated cell wave (Asia, or a US cell at the Asia shape)
|
||||
required: true
|
||||
type: string
|
||||
selector-generation:
|
||||
description: Exact live selector generation; leave empty only for inspect
|
||||
required: false
|
||||
type: string
|
||||
selector-membership-sha256:
|
||||
description: Exact fingerprint printed by inspect; required only for initialize
|
||||
required: false
|
||||
type: string
|
||||
selector-attempt-id:
|
||||
description: Durable unique attempt ID; empty for inspect, verify, and configure
|
||||
required: false
|
||||
type: string
|
||||
image-digest:
|
||||
description: Expected compatible Relay sha256 digest
|
||||
required: true
|
||||
type: string
|
||||
director-image-digest:
|
||||
description: Director sha256 digest; required only for configure
|
||||
required: false
|
||||
type: string
|
||||
evidence-run-id:
|
||||
description: Staging evidence run ID for C27, C27 canary run ID for C28/C29; C30-C35 take none and each proves itself by its own canary
|
||||
required: false
|
||||
type: string
|
||||
evidence-run-attempt:
|
||||
description: Exact evidence workflow run attempt; required with an evidence run ID
|
||||
required: false
|
||||
type: string
|
||||
confirmation:
|
||||
description: Exact typed confirmation for a mutation
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
admission:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 30
|
||||
environment: ${{ inputs.environment }}
|
||||
env:
|
||||
DIRECTOR_ORIGIN: ${{ inputs.environment == 'production' && 'https://relay.onorca.dev' || 'https://relay-staging.onorca.dev' }}
|
||||
AUTH_ORIGIN: ${{ inputs.environment == 'production' && 'https://login.onorca.dev' || 'https://auth-staging.onorca.dev' }}
|
||||
DIRECTOR_SERVICE: ${{ inputs.environment == 'production' && 'orca-cloud-relay' || 'orca-cloud-relay-staging' }}
|
||||
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
|
||||
GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }}
|
||||
GCP_REGION: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_REGION || vars.STAGING_GCP_REGION }}
|
||||
DIRECTOR_MAX_INSTANCES: ${{ inputs.environment == 'production' && '5' || '2' }}
|
||||
TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }}
|
||||
TARGET_ENVIRONMENT: ${{ inputs.environment }}
|
||||
OPERATION_MODE: ${{ inputs.mode }}
|
||||
TARGET_CELL_IDS: ${{ inputs.cell-ids }}
|
||||
EXPECTED_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
|
||||
EXPECTED_SELECTOR_MEMBERSHIP_SHA256: ${{ inputs.selector-membership-sha256 }}
|
||||
SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }}
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }}
|
||||
EVIDENCE_RUN_ID: ${{ inputs.evidence-run-id }}
|
||||
EVIDENCE_RUN_ATTEMPT: ${{ inputs.evidence-run-attempt }}
|
||||
OPERATION_CONFIRMATION: ${{ inputs.confirmation }}
|
||||
DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Validate exact operation inputs before authentication
|
||||
id: inputs
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${DEPLOY_SERVICE_ACCOUNT}"
|
||||
[[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
evidence_kind=none
|
||||
canary_cell=none
|
||||
canary_region=asia-east2
|
||||
artifact_name=none
|
||||
if test "${OPERATION_MODE}" = inspect; then
|
||||
test -z "${EXPECTED_SELECTOR_GENERATION}"
|
||||
test -z "${SELECTOR_ATTEMPT_ID}"
|
||||
test -z "${OPERATION_CONFIRMATION}"
|
||||
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
|
||||
test -z "${DIRECTOR_IMAGE_DIGEST}"
|
||||
else
|
||||
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
fi
|
||||
if test "${OPERATION_MODE}" = initialize; then
|
||||
test "${EXPECTED_SELECTOR_GENERATION}" = 0
|
||||
[[ "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" =~ ^[a-f0-9]{64}$ ]]
|
||||
test -z "${DIRECTOR_IMAGE_DIGEST}"
|
||||
[[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
test "${OPERATION_CONFIRMATION}" = INITIALIZE_ADMISSION_SELECTOR
|
||||
elif test "${OPERATION_MODE}" = verify; then
|
||||
test -z "${SELECTOR_ATTEMPT_ID}"
|
||||
test -z "${OPERATION_CONFIRMATION}"
|
||||
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
|
||||
test -z "${DIRECTOR_IMAGE_DIGEST}"
|
||||
elif test "${OPERATION_MODE}" = inspect; then
|
||||
:
|
||||
elif test "${OPERATION_MODE}" = configure; then
|
||||
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
|
||||
test -z "${SELECTOR_ATTEMPT_ID}"
|
||||
[[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
test "${OPERATION_CONFIRMATION}" = CONFIGURE_ASIA_DIRECTOR
|
||||
else
|
||||
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
|
||||
test -z "${DIRECTOR_IMAGE_DIGEST}"
|
||||
[[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
case "${OPERATION_MODE}:${OPERATION_CONFIRMATION}" in
|
||||
register:REGISTER_ASIA_MIGRATION_ONLY) ;;
|
||||
promote:PROMOTE_ASIA_GENERAL) ;;
|
||||
rollback:ROLLBACK_ASIA_MIGRATION_ONLY) ;;
|
||||
*) echo "typed confirmation does not match the requested mutation" >&2; exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
if test "${TARGET_ENVIRONMENT}:${OPERATION_MODE}" = production:promote; then
|
||||
case "${TARGET_CELL_IDS}" in
|
||||
production-gce-c27)
|
||||
evidence_kind=staging
|
||||
artifact_name="relay-asia-staging-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}"
|
||||
canary_cell=production-gce-c27
|
||||
;;
|
||||
production-gce-c28,production-gce-c29)
|
||||
evidence_kind=c27
|
||||
artifact_name="relay-asia-c27-canary-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}"
|
||||
;;
|
||||
production-gce-c30|production-gce-c31|production-gce-c34|production-gce-c35)
|
||||
# No earlier proof binds a later cell's generation; its own canary below rolls it back on failure.
|
||||
canary_cell="${TARGET_CELL_IDS}"
|
||||
;;
|
||||
production-gce-c32|production-gce-c33)
|
||||
# A US cell at the Asia shape proves itself the same way, with load aimed at its own region.
|
||||
canary_cell="${TARGET_CELL_IDS}"
|
||||
canary_region=us-central1
|
||||
;;
|
||||
*) echo "production promotion wave is not reviewed" >&2; exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
if test "${evidence_kind}" = none; then
|
||||
test -z "${EVIDENCE_RUN_ID}"
|
||||
test -z "${EVIDENCE_RUN_ATTEMPT}"
|
||||
else
|
||||
[[ "${EVIDENCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${EVIDENCE_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
fi
|
||||
canary=false
|
||||
if test "${canary_cell}" != none; then
|
||||
canary=true
|
||||
# Leaves room for the rollback attempt's -rollback suffix within 128 characters.
|
||||
test "${#SELECTOR_ATTEMPT_ID}" -le 119
|
||||
fi
|
||||
{
|
||||
echo "evidence_kind=${evidence_kind}"
|
||||
echo "artifact_name=${artifact_name}"
|
||||
echo "canary=${canary}"
|
||||
echo "canary_cell=${canary_cell}"
|
||||
echo "canary_region=${canary_region}"
|
||||
echo "canary_hostname=${canary_cell##*-}"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
|
||||
- name: Install exact canary dependencies
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build the canary Relay contract
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
run: pnpm --filter @orca-cloud/relay-contract build
|
||||
|
||||
- name: Download immutable rollout evidence
|
||||
if: ${{ steps.inputs.outputs.evidence_kind != 'none' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ steps.inputs.outputs.artifact_name }}
|
||||
path: ${{ runner.temp }}/relay-asia-input-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.evidence-run-id }}
|
||||
|
||||
- name: Verify evidence provenance and rollout binding before authentication
|
||||
if: ${{ steps.inputs.outputs.evidence_kind != 'none' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
EVIDENCE_KIND: ${{ steps.inputs.outputs.evidence_kind }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
run_json="${RUNNER_TEMP}/relay-asia-evidence-run.json"
|
||||
verified="${RUNNER_TEMP}/relay-asia-evidence-verified"
|
||||
gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${EVIDENCE_RUN_ID}/attempts/${EVIDENCE_RUN_ATTEMPT}" > "${run_json}"
|
||||
evidence_commit_sha="$(
|
||||
jq -er '.head_sha | select(type == "string" and test("^[a-f0-9]{40}$"))' "${run_json}"
|
||||
)"
|
||||
command=(node dev/scripts/relay-asia-rollout-evidence.mjs "verify-${EVIDENCE_KIND}"
|
||||
--evidence "${RUNNER_TEMP}/relay-asia-input-evidence/evidence.json"
|
||||
--run-json "${run_json}"
|
||||
--commit-sha "${evidence_commit_sha}"
|
||||
--image-digest "${IMAGE_DIGEST}"
|
||||
--now "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
--output "${verified}")
|
||||
if test "${EVIDENCE_KIND}" = c27; then
|
||||
command+=(--selector-generation "${EXPECTED_SELECTOR_GENERATION}")
|
||||
fi
|
||||
"${command[@]}"
|
||||
test "$(< "${verified}")" = verified
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ env.DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ env.DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: ${{ env.DIRECTOR_ORIGIN }}/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Require the exact director image before promotion
|
||||
if: ${{ inputs.mode == 'promote' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
runtime="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data '{"v":1}')"
|
||||
test "$(jq -r '.role' <<< "${runtime}")" = director
|
||||
test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}"
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }}
|
||||
object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }}
|
||||
if: ${{ inputs.mode == 'configure' || steps.inputs.outputs.canary == 'true' }}
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
if: ${{ inputs.mode == 'configure' }}
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Run the exact generation-bound admission operation
|
||||
id: admission-operation
|
||||
if: ${{ inputs.mode != 'configure' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
run: |
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--mode "${OPERATION_MODE}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-membership-sha256 "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" \
|
||||
--attempt-id "${SELECTOR_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
generation="$(jq -er '.generation' <<< "${result}")"
|
||||
states="$(jq -cS '.states // {}' <<< "${result}")"
|
||||
membership="$(jq -cS '.membership // empty' <<< "${result}")"
|
||||
membership_sha256="$(jq -r '.membershipSha256 // empty' <<< "${result}")"
|
||||
echo "generation=${generation}" >> "${GITHUB_OUTPUT}"
|
||||
result_dir="${RUNNER_TEMP}/relay-asia-admission-result"
|
||||
mkdir -p "${result_dir}"
|
||||
node dev/scripts/sanitize-relay-asia-admission-result.mjs \
|
||||
<<< "${result}" > "${result_dir}/result.json"
|
||||
{
|
||||
echo "### Relay Asia admission"
|
||||
echo "- Mode: ${OPERATION_MODE}"
|
||||
echo "- Cells: ${TARGET_CELL_IDS}"
|
||||
echo "- Result generation: ${generation}"
|
||||
echo "- States: \`${states}\`"
|
||||
if test -n "${membership}"; then echo "- Membership: \`${membership}\`"; fi
|
||||
if test -n "${membership_sha256}"; then
|
||||
echo "- Membership SHA-256: \`${membership_sha256}\`"
|
||||
fi
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Verify the canary cell state and start the timed canary
|
||||
id: canary-start
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# A later cell's launch cells were checked general by the promotion; verify reads only its digest.
|
||||
case "${CANARY_CELL}" in
|
||||
production-gce-c27)
|
||||
verify_cells=production-gce-c27,production-gce-c28,production-gce-c29
|
||||
expected_states='{"production-gce-c27":"general","production-gce-c28":"migration-only","production-gce-c29":"migration-only"}'
|
||||
;;
|
||||
production-gce-c30|production-gce-c31|production-gce-c32|production-gce-c33|production-gce-c34|production-gce-c35)
|
||||
verify_cells="${CANARY_CELL}"
|
||||
expected_states="{\"${CANARY_CELL}\":\"general\"}"
|
||||
;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment production \
|
||||
--mode verify \
|
||||
--cell-ids "${verify_cells}" \
|
||||
--expected-generation "${{ steps.admission-operation.outputs.generation }}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -cS '.states' <<< "${result}")" = "$(jq -cS '.' <<< "${expected_states}")"
|
||||
echo "verify_cells=${verify_cells}" >> "${GITHUB_OUTPUT}"
|
||||
echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Run a real five-minute canary control and splice
|
||||
id: canary-load
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
env:
|
||||
CANARY_REGION: ${{ steps.inputs.outputs.canary_region }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
log="${RUNNER_TEMP}/relay-asia-canary-load.jsonl"
|
||||
report="${RUNNER_TEMP}/relay-asia-canary-load.json"
|
||||
node dev/scripts/load-relay-controls.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--auth-origin "${AUTH_ORIGIN}" \
|
||||
--preferred-region "${CANARY_REGION}" \
|
||||
--relay-asia-load-principals 1 \
|
||||
--controls 1 \
|
||||
--splices 1 \
|
||||
--capacity-hard-cap 3000 \
|
||||
--ramp-seconds 0 \
|
||||
--duration-seconds 300 \
|
||||
--splice-hold-seconds 60 \
|
||||
--required-lease-horizons 2 > "${log}"
|
||||
jq -cer 'select(.event == "relay_load_complete")' "${log}" | tail -n 1 > "${report}"
|
||||
echo "ended_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Collect regional, Relay SQL, and Cloud SQL canary evidence
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }}
|
||||
CANARY_STARTED_AT: ${{ steps.canary-start.outputs.started_at }}
|
||||
CANARY_VERIFY_CELLS: ${{ steps.canary-start.outputs.verify_cells }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment production \
|
||||
--mode verify \
|
||||
--cell-ids "${CANARY_VERIFY_CELLS}" \
|
||||
--expected-generation "${{ steps.admission-operation.outputs.generation }}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r --arg cell "${CANARY_CELL}" '.states[$cell]' <<< "${result}")" = general
|
||||
ended_at="${{ steps.canary-load.outputs.ended_at }}"
|
||||
sleep 60
|
||||
output="${RUNNER_TEMP}/relay-asia-output-evidence"
|
||||
logs="${RUNNER_TEMP}/relay-asia-canary-runtime-metrics.json"
|
||||
mkdir -p "${output}"
|
||||
gcloud logging read \
|
||||
"timestamp>=\"${CANARY_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--limit 20000 \
|
||||
--format json > "${logs}"
|
||||
node dev/scripts/relay-asia-rollout-evidence.mjs create-canary \
|
||||
--cell-id "${CANARY_CELL}" \
|
||||
--repository "${GITHUB_REPOSITORY}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--run-attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--image-digest "${IMAGE_DIGEST}" \
|
||||
--selector-generation "${{ steps.admission-operation.outputs.generation }}" \
|
||||
--started-at "${CANARY_STARTED_AT}" \
|
||||
--ended-at "${ended_at}" \
|
||||
--load-report "${RUNNER_TEMP}/relay-asia-canary-load.json" \
|
||||
--logs-json "${logs}" \
|
||||
--output "${output}/evidence.json"
|
||||
jq -r '.metrics | to_entries[] | "- \(.key): \(.value)"' \
|
||||
"${output}/evidence.json" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload immutable canary evidence
|
||||
id: canary-evidence-upload
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-asia-${{ steps.inputs.outputs.canary_hostname }}-canary-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/relay-asia-output-evidence/evidence.json
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload sanitized admission result
|
||||
if: ${{ inputs.mode != 'configure' && steps.admission-operation.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-asia-admission-result-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/relay-asia-admission-result/result.json
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: Return an unproven canary cell to migration-only
|
||||
if: ${{ always() && steps.inputs.outputs.canary == 'true' && steps.admission-operation.outcome != 'skipped' && steps.canary-evidence-upload.outcome != 'success' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment production \
|
||||
--mode recover-promotion \
|
||||
--cell-ids "${CANARY_CELL}" \
|
||||
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--attempt-id "${SELECTOR_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
|
||||
promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment production \
|
||||
--mode rollback \
|
||||
--cell-ids "${CANARY_CELL}" \
|
||||
--expected-generation "${promoted_generation}" \
|
||||
--attempt-id "${SELECTOR_ATTEMPT_ID}-rollback" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r --arg cell "${CANARY_CELL}" '.states[$cell]' <<< "${result}")" = migration-only
|
||||
|
||||
- name: Require registered migration-only cells before director configuration
|
||||
if: ${{ inputs.mode == 'configure' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
run: |
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--mode registered \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true
|
||||
|
||||
- name: Build the additive director cell configuration
|
||||
if: ${{ inputs.mode == 'configure' }}
|
||||
id: director-config
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}"
|
||||
topology="${RUNNER_TEMP}/relay-asia-state-topology.json"
|
||||
current="${RUNNER_TEMP}/relay-current-director-cells.json"
|
||||
desired="${RUNNER_TEMP}/relay-asia-director-cells.json"
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${topology}"
|
||||
service="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
revision="$(jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end' \
|
||||
<<< "${service}")"
|
||||
gcloud run revisions describe "${revision}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -er '.spec.containers[0].env[] | select(.name == "ORCA_RELAY_CELLS_JSON") | .value | fromjson' \
|
||||
> "${current}"
|
||||
node dev/scripts/prepare-relay-asia-director-cells.mjs \
|
||||
--current-json "${current}" \
|
||||
--topology-json "${topology}" \
|
||||
--output "${desired}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--image-digest "${IMAGE_DIGEST}"
|
||||
echo "file=${desired}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Deploy the registered additive director topology
|
||||
if: ${{ inputs.mode == 'configure' }}
|
||||
env:
|
||||
DIRECTOR_CELLS_FILE: ${{ steps.director-config.outputs.file }}
|
||||
run: |
|
||||
current="$(gcloud secrets versions access latest \
|
||||
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")"
|
||||
[[ "${current}" =~ ^(true|false)$ ]]
|
||||
image="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}"
|
||||
release_id="asia-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
|
||||
node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service "${DIRECTOR_SERVICE}" \
|
||||
--image "${image}" \
|
||||
--role director \
|
||||
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
|
||||
--release-id "${release_id}" \
|
||||
--director-cells-json "$(< "${DIRECTOR_CELLS_FILE}")" \
|
||||
--prune-revisions false
|
||||
|
||||
- name: Verify selector and heartbeats after director configuration
|
||||
if: ${{ inputs.mode == 'configure' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
run: |
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--mode verify \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true
|
||||
revision="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end')"
|
||||
revision_json="$(gcloud run revisions describe "${revision}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
jq -e --arg image "us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}" \
|
||||
'.spec.containers[0].image == $image' <<< "${revision_json}" > /dev/null
|
||||
jq -er --arg secret "${REGIONAL_PLACEMENT_SECRET}" \
|
||||
'[.spec.containers[0].env[] |
|
||||
select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
|
||||
(.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
|
||||
{secret: (.secret // .name), version: (.version // .key)} |
|
||||
select(.secret == $secret and (.version | test("^[1-9][0-9]*$")))] |
|
||||
if length == 1 then .[0].version else error("regional switch version missing") end' \
|
||||
<<< "${revision_json}" > "${RUNNER_TEMP}/relay-regional-placement-version"
|
||||
regional_version="$(< "${RUNNER_TEMP}/relay-regional-placement-version")"
|
||||
[[ "$(gcloud secrets versions access "${regional_version}" --project "${GCP_PROJECT_ID}" \
|
||||
--secret "${REGIONAL_PLACEMENT_SECRET}")" =~ ^(true|false)$ ]]
|
||||
echo "Director configuration now lists the registered migration-only Asia cells." >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,127 +0,0 @@
|
||||
name: Operate Relay Production Cell Flags
|
||||
|
||||
# Writes one cell's runtime switch object and waits for the cell to report it applied.
|
||||
# One cell per run; never fleet-wide. dry-run writes nothing.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: dry-run prints the change; write applies it
|
||||
required: true
|
||||
default: dry-run
|
||||
type: choice
|
||||
options: [dry-run, write]
|
||||
cell-id:
|
||||
description: Exact cell id, e.g. production-gce-c26
|
||||
required: true
|
||||
type: string
|
||||
break-glass:
|
||||
description: >-
|
||||
Only for a dead or unreachable reserve cell, with set admitMode=default. Typed as
|
||||
BREAK GLASS <cell-id>; records db without the cell's read-back
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
confirm-reserve:
|
||||
description: Required for admitMode=reserve, typed as RESERVE <cell-id>
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
set:
|
||||
description: >-
|
||||
Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a
|
||||
key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck,
|
||||
admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs,
|
||||
reregisterInFlight, phonePath (memory only takes effect in reserve; a reserve flip
|
||||
must name phonePath)
|
||||
required: true
|
||||
type: string
|
||||
expected-generation:
|
||||
description: Exact current object generation from a dry run, or 0 if none exists
|
||||
required: true
|
||||
type: string
|
||||
confirmation:
|
||||
description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# One write at a time across the fleet keeps the audit trail in order.
|
||||
concurrency:
|
||||
group: relay-production-cell-flags
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
# Explicit bash runs with pipefail, so the script's exit code survives the tee.
|
||||
shell: bash
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
operate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
|
||||
timeout-minutes: 60
|
||||
environment: production
|
||||
env:
|
||||
MODE: ${{ inputs.mode }}
|
||||
CELL_ID: ${{ inputs.cell-id }}
|
||||
FLAG_CHANGES: ${{ inputs.set }}
|
||||
EXPECTED_GENERATION: ${{ inputs.expected-generation }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
CONFIRM_RESERVE: ${{ inputs.confirm-reserve }}
|
||||
BREAK_GLASS: ${{ inputs.break-glass }}
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- name: Write or preview the cell's switch object
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
|
||||
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
|
||||
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
|
||||
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
|
||||
node dev/scripts/operate-relay-cell-flags.mjs \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" \
|
||||
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" \
|
||||
--set "${FLAG_CHANGES}" \
|
||||
--director-origin "https://relay.onorca.dev" --confirm-reserve "${CONFIRM_RESERVE}" \
|
||||
--break-glass "${BREAK_GLASS}" \
|
||||
--expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-cell-flags.jsonl"
|
||||
|
||||
- name: Audit summary
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
{
|
||||
echo "### Relay cell flags: ${MODE} ${CELL_ID}"
|
||||
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
|
||||
echo '```json'
|
||||
cat "${RUNNER_TEMP}/relay-cell-flags.jsonl" 2>/dev/null || echo '(no output)'
|
||||
echo '```'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,342 +0,0 @@
|
||||
name: Operate Relay Production Rehome Job
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
mode: { required: true, type: string }
|
||||
director-image-digest: { required: true, type: string }
|
||||
rollback-image-digest: { required: true, type: string }
|
||||
expected-selector-generation: { required: true, type: string }
|
||||
expected-existing-only-cells: { required: true, type: string }
|
||||
expected-migration-only-cells: { required: true, type: string }
|
||||
expected-general-cells: { required: true, type: string }
|
||||
expected-control-generation: { required: true, type: string }
|
||||
not-before: { required: true, type: string }
|
||||
rate-per-minute: { required: true, type: string }
|
||||
preference-max-age-ms: { required: true, type: string }
|
||||
host-cooldown-ms: { required: true, type: string }
|
||||
drain-grace-ms: { required: true, type: string }
|
||||
confirmation: { required: true, type: string }
|
||||
monitor-run-id: { required: true, type: string }
|
||||
monitor-run-attempt: { required: true, type: string }
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
defaults:
|
||||
run:
|
||||
# `shell: bash` adds pipefail; without it `node ... | tee` reports tee's exit code and a
|
||||
# thrown inspect/apply passed green (Aug 28-29 and Sep 3 2026 runs).
|
||||
shell: bash
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
control:
|
||||
if: ${{ github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 30
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
DIRECTOR_SERVICE: orca-cloud-relay
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain
|
||||
MODE: ${{ inputs.mode }}
|
||||
DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }}
|
||||
ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }}
|
||||
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
||||
EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
|
||||
EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
|
||||
EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
|
||||
EXPECTED_CONTROL_GENERATION: ${{ inputs.expected-control-generation }}
|
||||
NOT_BEFORE: ${{ inputs.not-before }}
|
||||
RATE_PER_MINUTE: ${{ inputs.rate-per-minute }}
|
||||
PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }}
|
||||
HOST_COOLDOWN_MS: ${{ inputs.host-cooldown-ms }}
|
||||
DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
||||
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
|
||||
steps:
|
||||
- name: Require exact reusable-workflow configuration
|
||||
working-directory: .
|
||||
env:
|
||||
DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
[[ "${MODE}" =~ ^(inspect|enable|pause|disable)$ ]]
|
||||
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${EXPECTED_CONTROL_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
test -n "${DEPLOY_WIF}"
|
||||
test -n "${DEPLOY_SERVICE_ACCOUNT}"
|
||||
if [[ "${MODE}" =~ ^(inspect|enable)$ ]]; then
|
||||
[[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
[[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test -n "${GCP_REGION}"
|
||||
test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
|
||||
fi
|
||||
case "${MODE}" in
|
||||
inspect)
|
||||
test -z "${CONFIRMATION}"
|
||||
;;
|
||||
enable)
|
||||
test "${CONFIRMATION}" = ENABLE_REGIONAL_REHOMING
|
||||
test "${RATE_PER_MINUTE}" = 10
|
||||
[[ "${NOT_BEFORE}" =~ ^[1-9][0-9]*$ ]]
|
||||
;;
|
||||
pause)
|
||||
test "${CONFIRMATION}" = PAUSE_REGIONAL_REHOMING
|
||||
;;
|
||||
disable)
|
||||
test "${CONFIRMATION}" = DISABLE_REGIONAL_REHOMING
|
||||
;;
|
||||
esac
|
||||
|
||||
# Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
|
||||
# and the provenance check fails closed on a commit a shallow clone left out.
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Apply emergency durable pause or disable before diagnostics
|
||||
if: ${{ inputs.mode == 'pause' || inputs.mode == 'disable' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
|
||||
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
|
||||
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
|
||||
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
with: { package-json-file: cloud/package.json }
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
|
||||
- name: Download fresh aggregate safety evidence
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ github.workspace }}/relay-monitor-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
- name: Verify enable evidence provenance
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
run: |
|
||||
[[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-authority \
|
||||
--directory "${OUTPUT_DIRECTORY}" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" --run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" --mode dry-run \
|
||||
--required-migration-policy strict
|
||||
|
||||
- name: Reject previously consumed enable safety evidence
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
COUNT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
||||
--jq '.total_count')"
|
||||
test "${COUNT}" = 0
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
|
||||
- name: Verify exact serving and rollback director identities
|
||||
if: ${{ inputs.mode == 'inspect' || inputs.mode == 'enable' }}
|
||||
env:
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
SERVING_REVISION="$(jq -er \
|
||||
'[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName
|
||||
else error("director does not have one serving revision") end' \
|
||||
<<< "${SERVICE_JSON}")"
|
||||
ROLLBACK_REVISION="$(jq -er \
|
||||
'[.status.traffic[] | select(.tag == "selector-rollback")] |
|
||||
if length == 1 then .[0].revisionName else error("rollback tag missing") end' \
|
||||
<<< "${SERVICE_JSON}")"
|
||||
verify_revision() {
|
||||
local revision="$1" expected_digest="$2"
|
||||
local json
|
||||
json="$(gcloud run revisions describe "${revision}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
test "$(jq -r '.spec.serviceAccountName' <<< "${json}")" = \
|
||||
"${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
|
||||
test "$(jq -r '.spec.containers[0].image | split("@") | last' <<< "${json}")" = \
|
||||
"${expected_digest}"
|
||||
test "$(jq -r '[.spec.containers[0].env[] | select(.name ==
|
||||
"ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT") | .value] | if length == 1
|
||||
then .[0] else empty end' <<< "${json}")" = "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
|
||||
test "$(jq -r '[.spec.containers[0].env[] | select(.name ==
|
||||
"ORCA_RELAY_REHOME_AUDIENCE") | .value] | if length == 1 then .[0]
|
||||
else empty end' <<< "${json}")" = "${REHOME_AUDIENCE}"
|
||||
}
|
||||
verify_revision "${SERVING_REVISION}" "${DIRECTOR_IMAGE_DIGEST}"
|
||||
verify_revision "${ROLLBACK_REVISION}" "${ROLLBACK_IMAGE_DIGEST}"
|
||||
|
||||
- name: Seal 24-hour aggregate region observation evidence
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
run: |
|
||||
mkdir -p "${RUNNER_TEMP}/relay-region-observation"
|
||||
# 6 director instances x 120 samples/hour x 25h = 18000; a clipped
|
||||
# read empties the oldest hourly buckets and fails the seal.
|
||||
gcloud logging read \
|
||||
'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND jsonPayload.event="orca_relay_runtime_metrics" AND jsonPayload.role="director"' \
|
||||
--project "${GCP_PROJECT_ID}" --freshness=25h --limit=30000 --format=json \
|
||||
| node dev/scripts/relay-region-observation-evidence.mjs create \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \
|
||||
--selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
> "${RUNNER_TEMP}/relay-region-observation/evidence.json"
|
||||
|
||||
- name: Upload sealed 24-hour aggregate region evidence
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-region-observation-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/relay-region-observation/evidence.json
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Verify sealed 24-hour enable authority
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
run: |
|
||||
node dev/scripts/relay-region-observation-evidence.mjs verify \
|
||||
--file "${RUNNER_TEMP}/relay-region-observation/evidence.json" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \
|
||||
--selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--control-generation "${EXPECTED_CONTROL_GENERATION}"
|
||||
|
||||
- name: Recheck every aggregate safety signal before enable
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json"
|
||||
|
||||
- name: Seal single-use enable safety authority
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
mkdir -p "${RUNNER_TEMP}/relay-rehome-enable-authority"
|
||||
printf '%s\n' "${GITHUB_RUN_ID}" \
|
||||
> "${RUNNER_TEMP}/relay-rehome-enable-authority/${MARKER_NAME}"
|
||||
|
||||
- name: Consume enable safety evidence before durable mutation
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-rehome-enable-authority/relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Inspect regional rehome control
|
||||
if: ${{ inputs.mode == 'inspect' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
|
||||
- name: Apply exact durable regional rehome enable
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
|
||||
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
|
||||
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
|
||||
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
|
||||
- name: Read fresh aggregate completion and abort evidence
|
||||
run: |
|
||||
gcloud logging read \
|
||||
'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND textPayload:"[orca-relay] regional rehome inventory"' \
|
||||
--project "${GCP_PROJECT_ID}" --freshness=15m --limit=20 --format=json \
|
||||
| node dev/scripts/relay-rehome-aggregate-evidence.mjs --max-age-ms 900000 \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-inventory.json"
|
||||
|
||||
- name: Publish aggregate control evidence
|
||||
run: |
|
||||
{
|
||||
echo '### Regional rehome control'
|
||||
jq -r '"- mode: `\(.mode)`\n- generation: `\(.control.generation)`\n- enabled: `\(.control.enabled)`"' \
|
||||
"${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
jq -r '"- active: `\(.active)`\n- awaiting receipt: `\(.awaitingReceipt)`\n- target registered: `\(.targetRegistered)`\n- completed (24h): `\(.completedLast24Hours)`\n- aborted (24h): `\(.abortedLast24Hours)`\n- host not arrived (24h): `\(.hostNotArrivedLast24Hours // "not reported")`\n- oldest active age (ms): `\(.oldestActiveAgeMs // "none")`"' \
|
||||
"${RUNNER_TEMP}/relay-rehome-inventory.json"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Fail closed after an unsuccessful enable run
|
||||
if: ${{ failure() && inputs.mode == 'enable' && steps.google-auth.outcome == 'success' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode recover-enable --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
--confirmation RECOVER_FAILED_REGIONAL_REHOME_ENABLE \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-enable-recovery.json"
|
||||
jq -e \
|
||||
'.mode == "recover-enable" and .control.enabled == false' \
|
||||
"${RUNNER_TEMP}/relay-rehome-enable-recovery.json" >/dev/null
|
||||
@@ -1,112 +0,0 @@
|
||||
name: Operate Relay Production Rehome
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Inspect or apply the durable regional-rehome switch
|
||||
required: true
|
||||
default: inspect
|
||||
type: choice
|
||||
options: [inspect, enable, pause, disable]
|
||||
director-image-digest:
|
||||
description: Exact immutable serving director digest
|
||||
required: true
|
||||
type: string
|
||||
rollback-image-digest:
|
||||
description: Exact immutable selector-rollback director digest
|
||||
required: true
|
||||
type: string
|
||||
expected-selector-generation:
|
||||
description: Exact admission selector generation
|
||||
required: true
|
||||
type: string
|
||||
expected-existing-only-cells:
|
||||
description: Exact existing-only membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-migration-only-cells:
|
||||
description: Exact migration-only membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-general-cells:
|
||||
description: Exact general membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-control-generation:
|
||||
description: Exact durable rehome generation
|
||||
required: true
|
||||
type: string
|
||||
not-before:
|
||||
description: Exact epoch milliseconds; ignored only by inspect
|
||||
required: true
|
||||
default: '0'
|
||||
type: string
|
||||
rate-per-minute:
|
||||
description: Exact global host rate; initial enable is fixed at 10
|
||||
required: true
|
||||
default: '10'
|
||||
type: string
|
||||
preference-max-age-ms:
|
||||
description: Maximum fresh preference age
|
||||
required: true
|
||||
default: '86400000'
|
||||
type: string
|
||||
host-cooldown-ms:
|
||||
description: Minimum gap between two rehomes of the same host
|
||||
required: true
|
||||
default: '604800000'
|
||||
type: string
|
||||
drain-grace-ms:
|
||||
description: Per-host source drain grace
|
||||
required: true
|
||||
default: '3600000'
|
||||
type: string
|
||||
monitor-run-id:
|
||||
description: Fresh successful aggregate dry-run required only by enable
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-attempt:
|
||||
description: Exact monitor attempt required only by enable
|
||||
required: false
|
||||
type: string
|
||||
confirmation:
|
||||
description: ENABLE_REGIONAL_REHOMING, PAUSE_REGIONAL_REHOMING, or DISABLE_REGIONAL_REHOMING
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
operate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
uses: ./.github/workflows/cloud-operate-relay-production-rehome-job.yml
|
||||
with:
|
||||
mode: ${{ inputs.mode }}
|
||||
director-image-digest: ${{ inputs.director-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-control-generation: ${{ inputs.expected-control-generation }}
|
||||
not-before: ${{ inputs.not-before }}
|
||||
rate-per-minute: ${{ inputs.rate-per-minute }}
|
||||
preference-max-age-ms: ${{ inputs.preference-max-age-ms }}
|
||||
host-cooldown-ms: ${{ inputs.host-cooldown-ms }}
|
||||
drain-grace-ms: ${{ inputs.drain-grace-ms }}
|
||||
confirmation: ${{ inputs.confirmation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
secrets: inherit
|
||||
@@ -1,341 +0,0 @@
|
||||
name: Operate Relay Production Reserve Watch
|
||||
|
||||
# Watches one cell already switched to reserve mode and flips it back to db on the first stop
|
||||
# rule it trips (step 5 design §9). Manual only. dry-run evaluates and logs, writes nothing.
|
||||
# An operator watches the run. A refused watch or a flip back that did not land raises an ::error::
|
||||
# annotation, as the clock-skew monitor does, and in write mode logs the line the reserve-watch
|
||||
# alert pages on. Refuses to start while a rollout holds the Cloud SQL rollout lease.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: dry-run logs what a trip would write; write flips the cell back on a trip
|
||||
required: true
|
||||
default: dry-run
|
||||
type: choice
|
||||
options: [dry-run, write]
|
||||
cell-id:
|
||||
description: Exact cell id already in reserve mode, e.g. production-gce-c26
|
||||
required: true
|
||||
type: string
|
||||
duration-minutes:
|
||||
description: How long to watch, 5 to 300
|
||||
required: true
|
||||
default: '120'
|
||||
type: string
|
||||
flipped-at:
|
||||
description: >-
|
||||
When the reserve (or phonePath=memory) write applied, ISO with a zone, e.g.
|
||||
2026-10-14T12:00:00Z. The non-drain 503 background and the phone rules' background are
|
||||
the 10 minutes before it
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
background-503-per-minute:
|
||||
description: >-
|
||||
Director non-drain 503s per minute before the flip (all 503s less drain-return deferrals
|
||||
and a host's own early or duplicate re-dials, from the director's runtime metrics);
|
||||
overrides the one measured before flipped-at. One of the two is required
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
unverifiable-limit:
|
||||
description: Consecutive samples with any unreadable signal that count as a trip
|
||||
required: true
|
||||
default: '3'
|
||||
type: string
|
||||
confirmation:
|
||||
description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# One watch per cell; why not the flag workflow's group: cloud/docs/relay-workflows.md.
|
||||
concurrency:
|
||||
group: relay-production-reserve-watch-${{ inputs.cell-id }}
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
# Explicit bash runs with pipefail, so the script's exit code survives the tee.
|
||||
shell: bash
|
||||
working-directory: cloud
|
||||
|
||||
env:
|
||||
MODE: ${{ inputs.mode }}
|
||||
CELL_ID: ${{ inputs.cell-id }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
|
||||
jobs:
|
||||
watch:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
# Six 55-minute segments cover the longest watch; under the 360-minute job limit.
|
||||
timeout-minutes: 350
|
||||
environment: production
|
||||
outputs:
|
||||
state: ${{ steps.outcome.outputs.state }}
|
||||
verdict: ${{ steps.outcome.outputs.verdict }}
|
||||
# phonePath when a step-6 phone rule tripped; anything else flips the cell back whole.
|
||||
flip: ${{ steps.outcome.outputs.flip }}
|
||||
env:
|
||||
DURATION_MINUTES: ${{ inputs.duration-minutes }}
|
||||
FLIPPED_AT: ${{ inputs.flipped-at }}
|
||||
BACKGROUND_503_PER_MINUTE: ${{ inputs.background-503-per-minute }}
|
||||
UNVERIFIABLE_LIMIT: ${{ inputs.unverifiable-limit }}
|
||||
# One segment, after the auth step before it: fixed text, the inputs arrive as env vars.
|
||||
WATCH_SEGMENT: |
|
||||
set -euo pipefail
|
||||
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
|
||||
node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
|
||||
# A phone-rule trip writes phonePath=db mid-watch, through the flag tool.
|
||||
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
|
||||
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
|
||||
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs watch \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" \
|
||||
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \
|
||||
--director-service orca-cloud-relay --confirmation "${CONFIRMATION}" \
|
||||
--duration-minutes "${DURATION_MINUTES}" --segment-minutes 55 \
|
||||
--flipped-at "${FLIPPED_AT}" --background-503-per-minute "${BACKGROUND_503_PER_MINUTE}" \
|
||||
--unverifiable-limit "${UNVERIFIABLE_LIMIT}" \
|
||||
--state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}" \
|
||||
| tee -a "${RUNNER_TEMP}/relay-reserve-watch.jsonl"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- id: google-auth-1
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- id: watch-1
|
||||
name: Watch segment 1 of 6
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-1.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-2
|
||||
if: ${{ steps.watch-1.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-2
|
||||
name: Watch segment 2 of 6
|
||||
if: ${{ steps.watch-1.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-2.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-3
|
||||
if: ${{ steps.watch-2.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-3
|
||||
name: Watch segment 3 of 6
|
||||
if: ${{ steps.watch-2.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-3.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-4
|
||||
if: ${{ steps.watch-3.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-4
|
||||
name: Watch segment 4 of 6
|
||||
if: ${{ steps.watch-3.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-4.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-5
|
||||
if: ${{ steps.watch-4.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-5
|
||||
name: Watch segment 5 of 6
|
||||
if: ${{ steps.watch-4.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-5.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-6
|
||||
if: ${{ steps.watch-5.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-6
|
||||
name: Watch segment 6 of 6
|
||||
if: ${{ steps.watch-5.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-6.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: outcome
|
||||
name: Record the watch outcome
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs outcome \
|
||||
--state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}"
|
||||
|
||||
# The documented flow writes reserve before dispatching, so a refusal can leave it unwatched.
|
||||
- name: Alarm on a refused watch
|
||||
if: ${{ always() && steps.outcome.outputs.state == 'refused' }}
|
||||
env:
|
||||
WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }}
|
||||
run: |
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind refused \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${WATCH_VERDICT}"
|
||||
|
||||
- name: Audit summary
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
WATCH_STATE: ${{ steps.outcome.outputs.state }}
|
||||
WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }}
|
||||
run: |
|
||||
{
|
||||
echo "### Relay reserve watch: ${MODE} ${CELL_ID}"
|
||||
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "Outcome: ${WATCH_STATE}: ${WATCH_VERDICT}"
|
||||
echo '```json'
|
||||
tail -n 20 "${RUNNER_TEMP}/relay-reserve-watch.jsonl" 2>/dev/null || echo '(no output)'
|
||||
echo '```'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
flip-back:
|
||||
needs: watch
|
||||
# Missing data is never healthy: a watch that crashed, timed out or was cancelled (by an
|
||||
# operator too) flips back like a trip. In dry-run this only plans the write.
|
||||
if: ${{ always() && vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && needs.watch.result != 'skipped' && !contains(fromJSON('["completed","stopped","refused"]'), needs.watch.outputs.state) }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
|
||||
timeout-minutes: 60
|
||||
environment: production
|
||||
# Only the run's per-cell group above, which a running run never yields (cancel-in-progress off).
|
||||
# Every step runs on always() too, so an ordinary cancel cannot skip the switch-file write; a force-cancel
|
||||
# can, and leaves the cell in reserve for the operator to flip back by hand. A cancel during the
|
||||
# lease wait can still kill the process before Postgres records db: the file says db, Postgres reserve.
|
||||
env:
|
||||
VERDICT: ${{ needs.watch.outputs.verdict || format('the watch ended {0} without a verdict', needs.watch.result) }}
|
||||
FLIP: ${{ needs.watch.outputs.flip || 'admitMode' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- id: google-auth
|
||||
if: ${{ always() }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- name: Flip the cell back to db, or preview it
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
|
||||
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
|
||||
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
|
||||
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs flip-back \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" \
|
||||
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \
|
||||
--confirmation "${CONFIRMATION}" --verdict "${VERDICT}" --flip "${FLIP}" \
|
||||
| tee "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"
|
||||
|
||||
# The flip-back line is logged only once the write (or its dry-run plan) finished; without
|
||||
# it a missing token, a failed login or a silent cell left the cell in reserve, unwatched.
|
||||
- name: Alarm when the flip back did not land
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
if grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"; then
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${FLIP}" == phonePath ]]; then
|
||||
echo "## FLIP BACK FAILED: ${CELL_ID} may still have phonePath=memory" >> "${GITHUB_STEP_SUMMARY}"
|
||||
echo "Set phonePath=default by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}"
|
||||
else
|
||||
echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode" >> "${GITHUB_STEP_SUMMARY}"
|
||||
echo "Flip it to db by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}"
|
||||
fi
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind flip-back-failed \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}" --flip "${FLIP}"
|
||||
|
||||
- name: Audit summary
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
{
|
||||
echo "### Relay reserve watch flip back: ${MODE} ${CELL_ID}"
|
||||
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "Verdict: ${VERDICT}"
|
||||
echo '```json'
|
||||
cat "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl" 2>/dev/null || echo '(no output)'
|
||||
echo '```'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,129 +0,0 @@
|
||||
name: Operate Relay Staging Cell Flags
|
||||
|
||||
# The staging twin of Operate Relay Production Cell Flags: writes one staging cell's runtime
|
||||
# switch object and waits for the cell to report it applied. One cell per run. dry-run writes
|
||||
# nothing.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: dry-run prints the change; write applies it
|
||||
required: true
|
||||
default: dry-run
|
||||
type: choice
|
||||
options: [dry-run, write]
|
||||
cell-id:
|
||||
description: Exact cell id, e.g. staging-gce-c2
|
||||
required: true
|
||||
type: string
|
||||
break-glass:
|
||||
description: >-
|
||||
Only for a dead or unreachable reserve cell, with set admitMode=default. Typed as
|
||||
BREAK GLASS <cell-id>; records db without the cell's read-back
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
confirm-reserve:
|
||||
description: Required for admitMode=reserve, typed as RESERVE <cell-id>
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
set:
|
||||
description: >-
|
||||
Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a
|
||||
key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck,
|
||||
admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs,
|
||||
reregisterInFlight, phonePath (memory only takes effect in reserve; a reserve flip
|
||||
must name phonePath)
|
||||
required: true
|
||||
type: string
|
||||
expected-generation:
|
||||
description: Exact current object generation from a dry run, or 0 if none exists
|
||||
required: true
|
||||
type: string
|
||||
confirmation:
|
||||
description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# Not relay-staging-mutation: a switch write must run while a staging deploy or load run is
|
||||
# in flight, and it touches neither Cloud SQL nor a revision.
|
||||
concurrency:
|
||||
group: relay-staging-cell-flags
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
# Explicit bash runs with pipefail, so the script's exit code survives the tee.
|
||||
shell: bash
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
operate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
|
||||
timeout-minutes: 60
|
||||
environment: staging
|
||||
env:
|
||||
MODE: ${{ inputs.mode }}
|
||||
CELL_ID: ${{ inputs.cell-id }}
|
||||
FLAG_CHANGES: ${{ inputs.set }}
|
||||
EXPECTED_GENERATION: ${{ inputs.expected-generation }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
CONFIRM_RESERVE: ${{ inputs.confirm-reserve }}
|
||||
BREAK_GLASS: ${{ inputs.break-glass }}
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- name: Write or preview the cell's switch object
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
[[ "${CELL_ID}" =~ ^staging-gce-c[0-9]{1,3}$ ]]
|
||||
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
|
||||
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
|
||||
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
|
||||
node dev/scripts/operate-relay-cell-flags.mjs \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" \
|
||||
--cell-origin "https://${CELL_ID#staging-gce-}.relay-staging.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" \
|
||||
--set "${FLAG_CHANGES}" \
|
||||
--director-origin "https://relay-staging.onorca.dev" --confirm-reserve "${CONFIRM_RESERVE}" \
|
||||
--break-glass "${BREAK_GLASS}" \
|
||||
--expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-cell-flags.jsonl"
|
||||
|
||||
- name: Audit summary
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
{
|
||||
echo "### Relay staging cell flags: ${MODE} ${CELL_ID}"
|
||||
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
|
||||
echo '```json'
|
||||
cat "${RUNNER_TEMP}/relay-cell-flags.jsonl" 2>/dev/null || echo '(no output)'
|
||||
echo '```'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,104 +0,0 @@
|
||||
name: Power Relay Staging
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# A zero-activity guard makes this a no-op when an internal test is still running.
|
||||
- cron: '0 9 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Inspect, wake, or sleep the staging Relay data plane
|
||||
required: true
|
||||
default: status
|
||||
type: choice
|
||||
options:
|
||||
- status
|
||||
- wake
|
||||
- sleep
|
||||
wake-cells:
|
||||
description: Wake configured admission cells, or include disabled candidate cells
|
||||
required: true
|
||||
default: configured
|
||||
type: choice
|
||||
options:
|
||||
- configured
|
||||
- all
|
||||
confirmation:
|
||||
description: Enter WAKE_STAGING or SLEEP_STAGING for a manual mutation
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
power:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: staging
|
||||
env:
|
||||
POWER_MODE: ${{ github.event_name == 'schedule' && 'sleep' || inputs.mode }}
|
||||
WAKE_CELLS: ${{ inputs.wake-cells || 'configured' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Require explicit manual mutation confirmation
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && inputs.mode != 'status' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if [[ "${POWER_MODE}" = "wake" ]]; then
|
||||
test "${CONFIRMATION}" = "WAKE_STAGING"
|
||||
else
|
||||
test "${CONFIRMATION}" = "SLEEP_STAGING"
|
||||
fi
|
||||
|
||||
- name: Read reviewed staging topology
|
||||
run: |
|
||||
node dev/scripts/infra.mjs init --env staging
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
|
||||
- name: Inspect or change staging power state
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/power-staging-relay.mjs \
|
||||
--mode "${POWER_MODE}" \
|
||||
--wake-cells "${WAKE_CELLS}" \
|
||||
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
@@ -1,500 +0,0 @@
|
||||
name: Prove Relay Asia Staging
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: prove runs the fixed C4 proof; load runs the generator with free-form arguments
|
||||
required: true
|
||||
default: prove
|
||||
type: choice
|
||||
options: [prove, load]
|
||||
image-digest:
|
||||
description: Prove only. Exact immutable Relay digest deployed on staging C4
|
||||
required: false
|
||||
type: string
|
||||
selector-generation:
|
||||
description: Prove only. Exact selector generation with C4 migration-only
|
||||
required: false
|
||||
type: string
|
||||
promote-attempt-id:
|
||||
description: Prove only. Durable unique C4 promotion attempt ID
|
||||
required: false
|
||||
type: string
|
||||
rollback-attempt-id:
|
||||
description: Prove only. Durable unique C4 rollback attempt ID
|
||||
required: false
|
||||
type: string
|
||||
load-region:
|
||||
description: >-
|
||||
Load only. asia-east2 runs on the Asia load runner; asia-east2-hosted asks for asia-east2
|
||||
from a hosted runner; us-central1 runs on a hosted runner
|
||||
required: false
|
||||
default: asia-east2
|
||||
type: choice
|
||||
options: [asia-east2, asia-east2-hosted, us-central1]
|
||||
load-shards:
|
||||
description: Load only. Generator processes, each with its own shard of the 32 load principals
|
||||
required: false
|
||||
default: '4'
|
||||
type: choice
|
||||
options: ['1', '2', '3', '4']
|
||||
load-runners:
|
||||
description: >-
|
||||
Load only, hosted runners only. Runner machines, each with load-shards generators; one
|
||||
hosted runner holds about 2,000 connections to one cell
|
||||
required: false
|
||||
default: '1'
|
||||
type: choice
|
||||
options: ['1', '2']
|
||||
load-args:
|
||||
description: >-
|
||||
Load only. Per-shard generator arguments, e.g. --controls 150 --splices 15
|
||||
--reconnect-max-seconds 30 --duration-seconds 900. Origins, region, principals and
|
||||
sharding are set by the workflow
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
confirmation:
|
||||
description: Enter PROVE_ASIA_STAGING (prove) or LOAD_RELAY_STAGING (load)
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# A load run drives traffic only, so it must not queue behind (or hold up) the staging deploys
|
||||
# and switch writes it is there to exercise.
|
||||
concurrency:
|
||||
group: ${{ inputs.mode == 'load' && 'relay-staging-load' || 'relay-staging-mutation' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
prove:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') && inputs.mode == 'prove' }}
|
||||
runs-on: [self-hosted, linux, x64, relay-asia-east2-load]
|
||||
timeout-minutes: 75
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
AUTH_ORIGIN: https://auth-staging.onorca.dev
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
|
||||
PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }}
|
||||
ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Validate the exact staging proof request
|
||||
shell: bash
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "${CONFIRMATION}" = PROVE_ASIA_STAGING
|
||||
[[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
[[ "${INITIAL_SELECTOR_GENERATION}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${PROMOTE_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
[[ "${ROLLBACK_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
test "${PROMOTE_ATTEMPT_ID}" != "${ROLLBACK_ATTEMPT_ID}"
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- name: Require the exact staging director image before promotion
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
runtime="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data '{"v":1}')"
|
||||
test "$(jq -r '.role' <<< "${runtime}")" = director
|
||||
test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}"
|
||||
|
||||
- name: Install exact load-harness dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build the Relay load-harness contract
|
||||
run: pnpm --filter @orca-cloud/relay-contract build
|
||||
|
||||
- name: Promote only staging C4
|
||||
id: promote
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging \
|
||||
--mode promote \
|
||||
--cell-ids staging-gce-c4 \
|
||||
--expected-generation "${INITIAL_SELECTOR_GENERATION}" \
|
||||
--attempt-id "${PROMOTE_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
generation="$(jq -er '.generation' <<< "${result}")"
|
||||
test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = general
|
||||
echo "generation=${generation}" >> "${GITHUB_OUTPUT}"
|
||||
echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Run sharded two-horizon and mixed splice proofs
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof"
|
||||
mkdir -p "${proof_dir}"
|
||||
fd_limit="$(ulimit -n)"
|
||||
if test "${fd_limit}" != unlimited; then
|
||||
[[ "${fd_limit}" =~ ^[0-9]+$ ]]
|
||||
test "${fd_limit}" -ge 4096
|
||||
fi
|
||||
run_phase() {
|
||||
phase="$1"
|
||||
controls="$2"
|
||||
splices="$3"
|
||||
pids=()
|
||||
stop_shards() {
|
||||
for pid in "${pids[@]}"; do kill "${pid}" 2>/dev/null || true; done
|
||||
for pid in "${pids[@]}"; do wait "${pid}" 2>/dev/null || true; done
|
||||
}
|
||||
trap stop_shards EXIT
|
||||
for shard in 0 1 2 3; do
|
||||
slow=0
|
||||
wedged=0
|
||||
boundary_args=()
|
||||
request_unit_args=()
|
||||
if test "${phase}" = launch && test "${shard}" = 0; then
|
||||
slow=4
|
||||
wedged=1
|
||||
fi
|
||||
if test "${phase}" = launch && test "${shard}" = 0; then
|
||||
boundary_args=(
|
||||
--region-behavior-probes 1
|
||||
--capacity-cell-id staging-gce-c4
|
||||
--capacity-cell-origin https://c4.relay-staging.onorca.dev
|
||||
--capacity-unobserved-bound 60
|
||||
--rebind-probes 2
|
||||
--skip-rebind-overflow-check
|
||||
)
|
||||
fi
|
||||
node dev/scripts/load-relay-controls.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--auth-origin "${AUTH_ORIGIN}" \
|
||||
--preferred-region asia-east2 \
|
||||
--relay-asia-load-principals 32 \
|
||||
--controls "${controls}" \
|
||||
--splices "${splices}" \
|
||||
--slow-reader-splices "${slow}" \
|
||||
--wedged-reader-splices "${wedged}" \
|
||||
--capacity-hard-cap 3000 \
|
||||
"${boundary_args[@]}" \
|
||||
"${request_unit_args[@]}" \
|
||||
--phase-barrier-dir "${proof_dir}/${phase}-barrier" \
|
||||
--aggregate-controls "$((controls * 4))" \
|
||||
--aggregate-splices "$((splices * 4))" \
|
||||
--aggregate-reader-splices "$([[ "${phase}" = launch ]] && echo 5 || echo 0)" \
|
||||
--aggregate-reader-bytes "$([[ "${phase}" = launch ]] && echo 12582912 || echo 0)" \
|
||||
--required-lease-horizons 2 \
|
||||
--splice-ramp-seconds 120 \
|
||||
--max-generator-rss-growth-mib 512 \
|
||||
--ramp-seconds 180 \
|
||||
--duration-seconds 210 \
|
||||
--shard-count 4 \
|
||||
--shard-index "${shard}" \
|
||||
> "${proof_dir}/${phase}-${shard}.jsonl" &
|
||||
pids+=("$!")
|
||||
done
|
||||
failed=0
|
||||
for pid in "${pids[@]}"; do
|
||||
if ! wait "${pid}"; then failed=1; break; fi
|
||||
done
|
||||
if test "${failed}" = 1; then
|
||||
stop_shards
|
||||
for shard in 0 1 2 3; do
|
||||
jq -cer 'select(.event == "relay_load_progress" or .event == "relay_load_complete") |
|
||||
{event, shardIndex, active, peakActive, connected, connectionFailures,
|
||||
rampConnectionFailures, connectionFailuresByReason, unexpectedCloses,
|
||||
protocolErrors, refreshErrors, socketErrors, elapsedSeconds}' \
|
||||
"${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1 || true
|
||||
done
|
||||
trap - EXIT
|
||||
return 1
|
||||
fi
|
||||
trap - EXIT
|
||||
for shard in 0 1 2 3; do
|
||||
jq -cer 'select(.event == "relay_load_complete")' \
|
||||
"${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1
|
||||
done | jq -s . > "${proof_dir}/${phase}.json"
|
||||
}
|
||||
run_phase launch 5 5
|
||||
|
||||
- name: Collect and validate aggregate staging proof evidence
|
||||
env:
|
||||
PROOF_STARTED_AT: ${{ steps.promote.outputs.started_at }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof"
|
||||
ended_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
sleep 60
|
||||
gcloud logging read \
|
||||
"timestamp>=\"${PROOF_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \
|
||||
--project "${GCP_PROJECT_ID}" --limit 20000 --format json \
|
||||
> "${proof_dir}/runtime-metrics.json"
|
||||
node dev/scripts/relay-asia-rollout-evidence.mjs create-staging \
|
||||
--repository "${GITHUB_REPOSITORY}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--run-attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--image-digest "${IMAGE_DIGEST}" \
|
||||
--selector-generation "${{ steps.promote.outputs.generation }}" \
|
||||
--started-at "${PROOF_STARTED_AT}" \
|
||||
--ended-at "${ended_at}" \
|
||||
--launch-report "${proof_dir}/launch.json" \
|
||||
--logs-json "${proof_dir}/runtime-metrics.json" \
|
||||
--output "${proof_dir}/evidence.json"
|
||||
|
||||
- name: Return staging C4 to migration-only
|
||||
if: ${{ always() && steps.promote.outcome != 'skipped' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging \
|
||||
--mode recover-promotion \
|
||||
--cell-ids staging-gce-c4 \
|
||||
--expected-generation "${INITIAL_SELECTOR_GENERATION}" \
|
||||
--attempt-id "${PROMOTE_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
|
||||
promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging \
|
||||
--mode rollback \
|
||||
--cell-ids staging-gce-c4 \
|
||||
--expected-generation "${promoted_generation}" \
|
||||
--attempt-id "${ROLLBACK_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only
|
||||
|
||||
- name: Upload immutable staging readiness evidence
|
||||
if: ${{ success() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-asia-staging-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/relay-asia-staging-proof/evidence.json
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
recover:
|
||||
if: ${{ always() && github.ref == 'refs/heads/main' && inputs.mode == 'prove' }}
|
||||
needs: prove
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 15
|
||||
environment: staging
|
||||
env:
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
|
||||
PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }}
|
||||
ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Recover staging C4 with a fresh identity
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging \
|
||||
--mode recover-promotion \
|
||||
--cell-ids staging-gce-c4 \
|
||||
--expected-generation "${INITIAL_SELECTOR_GENERATION}" \
|
||||
--attempt-id "${PROMOTE_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
|
||||
promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging \
|
||||
--mode rollback \
|
||||
--cell-ids staging-gce-c4 \
|
||||
--expected-generation "${promoted_generation}" \
|
||||
--attempt-id "${ROLLBACK_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only
|
||||
|
||||
# The staging auth service mints relay load principals only for this workflow file, so the
|
||||
# free-form staging load lives here rather than in a workflow of its own.
|
||||
load:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') && inputs.mode == 'load' }}
|
||||
runs-on: ${{ inputs.load-region == 'asia-east2' && fromJSON('["self-hosted","linux","x64","relay-asia-east2-load"]') || 'ubuntu-latest' }}
|
||||
strategy:
|
||||
# One runner's failure must not drop the other half of the load mid-run.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: ${{ fromJSON(inputs.load-runners == '2' && '[0, 1]' || '[0]') }}
|
||||
timeout-minutes: 180
|
||||
environment: staging
|
||||
env:
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
AUTH_ORIGIN: https://auth-staging.onorca.dev
|
||||
LOAD_REGION: ${{ inputs.load-region }}
|
||||
LOAD_SHARDS: ${{ inputs.load-shards }}
|
||||
LOAD_RUNNERS: ${{ inputs.load-runners }}
|
||||
LOAD_RUNNER_INDEX: ${{ matrix.runner }}
|
||||
LOAD_ARGS: ${{ inputs.load-args }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Validate the staging load request
|
||||
shell: bash
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "${CONFIRMATION}" = LOAD_RELAY_STAGING
|
||||
[[ "${LOAD_REGION}" =~ ^(asia-east2|asia-east2-hosted|us-central1)$ ]]
|
||||
[[ "${LOAD_SHARDS}" =~ ^[1-4]$ ]]
|
||||
[[ "${LOAD_RUNNERS}" =~ ^[12]$ ]]
|
||||
# The Asia runner is one machine; a second matrix job would only queue behind the first.
|
||||
if test "${LOAD_RUNNERS}" != 1 && test "${LOAD_REGION}" = asia-east2; then
|
||||
echo "load-runners 2 needs a hosted runner" >&2
|
||||
exit 1
|
||||
fi
|
||||
read -r -a args <<< "${LOAD_ARGS}"
|
||||
for arg in "${args[@]}"; do
|
||||
[[ "${arg}" =~ ^[A-Za-z0-9=:/._,-]+$ ]] || { echo "unsupported characters in load-args" >&2; exit 1; }
|
||||
case "${arg}" in
|
||||
--target-origin|--director-origin|--auth-origin|--signing-key-file|--preferred-region|\
|
||||
--relay-asia-load-principals|--shard-count|--shard-index|--phase-barrier-dir)
|
||||
echo "${arg} is set by the workflow" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- name: Install exact load-harness dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build the Relay load-harness contract
|
||||
run: pnpm --filter @orca-cloud/relay-contract build
|
||||
|
||||
- name: Run the sharded staging load
|
||||
env:
|
||||
# Probes that read director capacity need an admin token; it lasts about an hour.
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
load_dir="${RUNNER_TEMP}/relay-staging-load"
|
||||
mkdir -p "${load_dir}"
|
||||
fd_limit="$(ulimit -n)"
|
||||
if test "${fd_limit}" != unlimited; then
|
||||
[[ "${fd_limit}" =~ ^[0-9]+$ ]]
|
||||
test "${fd_limit}" -ge 4096
|
||||
fi
|
||||
read -r -a args <<< "${LOAD_ARGS}"
|
||||
pids=()
|
||||
stop_shards() {
|
||||
for pid in "${pids[@]}"; do kill "${pid}" 2>/dev/null || true; done
|
||||
for pid in "${pids[@]}"; do wait "${pid}" 2>/dev/null || true; done
|
||||
}
|
||||
trap stop_shards EXIT
|
||||
for ((shard = 0; shard < LOAD_SHARDS; shard++)); do
|
||||
node dev/scripts/load-relay-controls.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--auth-origin "${AUTH_ORIGIN}" \
|
||||
--preferred-region "${LOAD_REGION%-hosted}" \
|
||||
--relay-asia-load-principals 32 \
|
||||
--shard-count "$((LOAD_SHARDS * LOAD_RUNNERS))" \
|
||||
--shard-index "$((LOAD_RUNNER_INDEX * LOAD_SHARDS + shard))" \
|
||||
"${args[@]}" \
|
||||
> "${load_dir}/shard-${shard}.jsonl" &
|
||||
pids+=("$!")
|
||||
done
|
||||
failed=0
|
||||
for pid in "${pids[@]}"; do
|
||||
if ! wait "${pid}"; then failed=1; fi
|
||||
done
|
||||
trap - EXIT
|
||||
for ((shard = 0; shard < LOAD_SHARDS; shard++)); do
|
||||
jq -c 'select(.event == "relay_load_complete" or .event == "relay_load_progress")' \
|
||||
"${load_dir}/shard-${shard}.jsonl" | tail -n 1 || true
|
||||
done
|
||||
exit "${failed}"
|
||||
|
||||
- name: Upload the load reports
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-staging-load-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.runner }}
|
||||
path: ${{ runner.temp }}/relay-staging-load/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
@@ -1,928 +0,0 @@
|
||||
name: Prove Relay Staging Capacity
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Verify or change C3 capacity, restore admission, or refresh empty Asia C4
|
||||
required: true
|
||||
default: verify
|
||||
type: choice
|
||||
options:
|
||||
- verify
|
||||
- apply
|
||||
- restore-admission
|
||||
- refresh-asia-c4-image
|
||||
expected-hard-cap:
|
||||
description: Exact cap declared for staging-gce-c3 in the reviewed staging tfvars
|
||||
required: true
|
||||
default: '600'
|
||||
type: choice
|
||||
options:
|
||||
- '1000'
|
||||
- '600'
|
||||
expected-unobserved-bound:
|
||||
description: Exact bound declared for staging-gce-c3 in the reviewed staging tfvars
|
||||
required: true
|
||||
default: '60'
|
||||
type: choice
|
||||
options:
|
||||
- '60'
|
||||
- '0'
|
||||
confirmation:
|
||||
description: Exact confirmation required for a mutation
|
||||
required: false
|
||||
type: string
|
||||
expected-selector-generation:
|
||||
description: Exact staging selector generation for a C4 image refresh
|
||||
required: false
|
||||
type: string
|
||||
predecessor-image-digest:
|
||||
description: Exact current C4 sha256 digest
|
||||
required: false
|
||||
type: string
|
||||
target-image-digest:
|
||||
description: Exact desired C4 sha256 digest
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
capacity:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && inputs.mode != 'refresh-asia-c4-image' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
|
||||
DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
CELL_ORIGIN: https://c3.relay-staging.onorca.dev
|
||||
# These drains are unpaced, but restart-safe still waits out the production pace window
|
||||
# from the drain, the same one the same-cap drain uses.
|
||||
RESTART_SAFE_PACE_WINDOW_MS: '300000'
|
||||
TARGET_CELL_ID: staging-gce-c3
|
||||
FALLBACK_CELL_ORIGIN: https://c2.relay-staging.onorca.dev
|
||||
FALLBACK_CELL_ID: staging-gce-c2
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
EXPECTED_HARD_CAP: ${{ inputs.expected-hard-cap }}
|
||||
EXPECTED_UNOBSERVED_BOUND: ${{ inputs.expected-unobserved-bound }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
if: ${{ inputs.mode != 'restore-admission' }}
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Initialize the exact staging backend
|
||||
if: ${{ inputs.mode != 'restore-admission' }}
|
||||
run: node dev/scripts/infra.mjs init --env staging
|
||||
|
||||
- name: Require reviewed desired capacity and image
|
||||
if: ${{ inputs.mode != 'restore-admission' }}
|
||||
shell: bash
|
||||
run: |
|
||||
CAP_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_hard_cap"
|
||||
BOUND_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_unobserved_bound"
|
||||
IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image"
|
||||
ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone"
|
||||
DESIRED_CAP="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars <<< "${CAP_EXPRESSION}")"
|
||||
DESIRED_BOUND="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars <<< "${BOUND_EXPRESSION}")"
|
||||
DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars <<< "${IMAGE_EXPRESSION}" | jq -r '.')"
|
||||
TARGET_ZONE="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars <<< "${ZONE_EXPRESSION}" | jq -r '.')"
|
||||
MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
|
||||
| jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
|
||||
DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'local.relay_director_cells_json' | jq -r '.')"
|
||||
CELL_ORIGIN="$(jq -r --arg cell "${TARGET_CELL_ID}" \
|
||||
'.[] | select(.id == $cell) | .url' <<< "${DESIRED_CELLS_JSON}")"
|
||||
test "${DESIRED_CAP}" = "${EXPECTED_HARD_CAP}"
|
||||
test "${DESIRED_BOUND}" = "${EXPECTED_UNOBSERVED_BOUND}"
|
||||
[[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]]
|
||||
[[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]]
|
||||
[[ "${MIG_NAME}" =~ ^[a-z0-9-]+$ ]]
|
||||
test "${CELL_ORIGIN}" = "https://c3.relay-staging.onorca.dev"
|
||||
jq -e \
|
||||
--arg cell "${TARGET_CELL_ID}" \
|
||||
--arg fallback "${FALLBACK_CELL_ID}" \
|
||||
--argjson cap "${EXPECTED_HARD_CAP}" \
|
||||
--argjson bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
'(any(.[]; .id == $cell and .connectionHardCap == $cap and .connectionUnobservedBound == $bound)) and
|
||||
(any(.[]; .id == $fallback and .connectionHardCap == 600 and .connectionUnobservedBound == 60))' \
|
||||
<<< "${DESIRED_CELLS_JSON}" >/dev/null
|
||||
echo "DESIRED_IMAGE=${DESIRED_IMAGE}" >> "${GITHUB_ENV}"
|
||||
echo "TARGET_ZONE=${TARGET_ZONE}" >> "${GITHUB_ENV}"
|
||||
echo "MIG_NAME=${MIG_NAME}" >> "${GITHUB_ENV}"
|
||||
echo "CELL_ORIGIN=${CELL_ORIGIN}" >> "${GITHUB_ENV}"
|
||||
echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify exact compatible director image
|
||||
if: ${{ inputs.mode != 'restore-admission' }}
|
||||
shell: bash
|
||||
run: |
|
||||
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
ACTIVE_REVISION="$(jq -r \
|
||||
'[.status.traffic[] | select((.percent // 0) > 0)]
|
||||
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \
|
||||
<<< "${SERVICE_JSON}")"
|
||||
test -n "${ACTIVE_REVISION}"
|
||||
ACTIVE_IMAGE="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format='value(spec.containers[0].image)')"
|
||||
test "${ACTIVE_IMAGE}" = "${DESIRED_IMAGE}"
|
||||
echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Require explicit transition confirmation
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: test "${CONFIRMATION}" = "FENCE_AND_TRANSITION_STAGING_C3"
|
||||
|
||||
- name: Require explicit admission restore confirmation
|
||||
if: ${{ inputs.mode == 'restore-admission' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: test "${CONFIRMATION}" = "RESTORE_STAGING_C2_C3_GENERAL"
|
||||
|
||||
- name: Require capacity identity and exact predecessor state
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "${EXPECTED_HARD_CAP}/${EXPECTED_UNOBSERVED_BOUND}" in
|
||||
1000/0)
|
||||
PREDECESSOR_C3_CAP=600
|
||||
PREDECESSOR_C3_BOUND=60
|
||||
;;
|
||||
1000/60)
|
||||
PREDECESSOR_C3_CAP=1000
|
||||
PREDECESSOR_C3_BOUND=0
|
||||
;;
|
||||
600/60)
|
||||
PREDECESSOR_C3_CAP=1000
|
||||
PREDECESSOR_C3_BOUND=60
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported staging capacity transition" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
ACTIVE_REVISION="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '
|
||||
[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test -n "${ACTIVE_REVISION}"
|
||||
CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -cer '
|
||||
[.spec.containers[0].env[]? |
|
||||
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
|
||||
if length == 1 then .[0] | fromjson else error("missing director topology") end')"
|
||||
PREDECESSOR_CELLS_JSON="$(jq -ce \
|
||||
--arg cell "${TARGET_CELL_ID}" \
|
||||
--argjson cap "${PREDECESSOR_C3_CAP}" \
|
||||
--argjson bound "${PREDECESSOR_C3_BOUND}" \
|
||||
'map(if .id == $cell then . + {
|
||||
connectionHardCap: $cap,
|
||||
connectionUnobservedBound: $bound
|
||||
} else . end)' <<< "${DESIRED_CELLS_JSON}")"
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${FALLBACK_CELL_ORIGIN}" \
|
||||
--cell-id "${FALLBACK_CELL_ID}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission either \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
if jq -ne \
|
||||
--argjson current "${CURRENT_CELLS_JSON}" \
|
||||
--argjson expected "${DESIRED_CELLS_JSON}" \
|
||||
'$current == $expected'; then
|
||||
if node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed; then
|
||||
TRANSITION_PHASE=cell-active
|
||||
elif node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining forbidden \
|
||||
--activity allowed; then
|
||||
TRANSITION_PHASE=cell-ready
|
||||
else
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
|
||||
TRANSITION_PHASE=director-ready
|
||||
fi
|
||||
else
|
||||
jq -ne \
|
||||
--argjson current "${CURRENT_CELLS_JSON}" \
|
||||
--argjson expected "${PREDECESSOR_CELLS_JSON}" \
|
||||
'$current == $expected'
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${PREDECESSOR_C3_CAP}" \
|
||||
--unobserved-bound "${PREDECESSOR_C3_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission either \
|
||||
--draining either \
|
||||
--activity allowed
|
||||
TRANSITION_PHASE=predecessor
|
||||
fi
|
||||
echo "TRANSITION_PHASE=${TRANSITION_PHASE}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Restore C2 as the safe placement fallback
|
||||
if: ${{ inputs.mode == 'restore-admission' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${FALLBACK_CELL_ORIGIN}" \
|
||||
--cell-id "${FALLBACK_CELL_ID}" \
|
||||
--heartbeat fresh \
|
||||
--admission either \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode restore-fallback \
|
||||
--general-cell-ids "${FALLBACK_CELL_ID}"
|
||||
|
||||
- name: Require a healthy non-draining C3 before restoring it
|
||||
if: ${{ inputs.mode == 'restore-admission' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission either \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Require a healthy general C2 before isolating C3
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
if test "${TRANSITION_PHASE}" = cell-active; then
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode restore-fallback \
|
||||
--general-cell-ids "${FALLBACK_CELL_ID}"
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${FALLBACK_CELL_ORIGIN}" \
|
||||
--cell-id "${FALLBACK_CELL_ID}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Reversibly isolate and drain C3
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
if test "${TRANSITION_PHASE}" = cell-ready; then
|
||||
exit 0
|
||||
fi
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode isolate
|
||||
|
||||
- name: Verify restart-safe migration-only target
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
if test "${TRANSITION_PHASE}" = cell-ready; then
|
||||
exit 0
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
|
||||
|
||||
- name: Verify current capacity
|
||||
if: ${{ inputs.mode == 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Deploy reviewed director topology and remove pre-protocol revisions
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
run: |
|
||||
if test "${TRANSITION_PHASE}" != predecessor; then
|
||||
echo "DIRECTOR_CONFIG_CHANGED=false" >> "${GITHUB_ENV}"
|
||||
exit 0
|
||||
fi
|
||||
RELEASE_ID="capacity-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
|
||||
DEPLOY_RESULT="$(node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service "${DIRECTOR_SERVICE_NAME}" \
|
||||
--image "${ACTIVE_IMAGE}" \
|
||||
--role director \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--capacity-cell-id "${TARGET_CELL_ID}" \
|
||||
--director-cells-json "${DESIRED_CELLS_JSON}" \
|
||||
--prune-revisions true \
|
||||
--release-id "${RELEASE_ID}")"
|
||||
echo "${DEPLOY_RESULT}"
|
||||
DIRECTOR_CONFIG_CHANGED="$(jq -r '.topologyChanged' <<< "${DEPLOY_RESULT}")"
|
||||
[[ "${DIRECTOR_CONFIG_CHANGED}" =~ ^(true|false)$ ]]
|
||||
echo "DIRECTOR_CONFIG_CHANGED=${DIRECTOR_CONFIG_CHANGED}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Require fail-closed director transition
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
if test "${TRANSITION_PHASE}" = cell-ready; then
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
exit 0
|
||||
fi
|
||||
HEARTBEAT_EXPECTATION=either
|
||||
if test "${DIRECTOR_CONFIG_CHANGED}" = true; then
|
||||
HEARTBEAT_EXPECTATION=stale
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat "${HEARTBEAT_EXPECTATION}" \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
|
||||
|
||||
- name: Plan and apply only the exact empty cell
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
shell: bash
|
||||
run: |
|
||||
recreate_fixed_one_instance() {
|
||||
local instance
|
||||
instance="$(gcloud compute instance-groups managed list-instances \
|
||||
"${MIG_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" \
|
||||
--format=json \
|
||||
| jq -er '
|
||||
if length == 1 and .[0].instanceStatus == "RUNNING" and
|
||||
.[0].currentAction == "NONE"
|
||||
then .[0].instance | split("/") | last
|
||||
else error("capacity cell does not have one stable running instance") end')"
|
||||
gcloud compute instance-groups managed recreate-instances \
|
||||
"${MIG_NAME}" \
|
||||
--instances "${instance}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" \
|
||||
--quiet
|
||||
}
|
||||
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c3"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]' \
|
||||
-out="${RUNNER_TEMP}/relay-capacity-cell.tfplan"
|
||||
PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-capacity-cell.tfplan" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode cell \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--image "${DESIRED_IMAGE}")"
|
||||
echo "${PLAN_RESULT}"
|
||||
CELL_PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")"
|
||||
[[ "${CELL_PLAN_CHANGES}" =~ ^(0|2)$ ]]
|
||||
if test "${TRANSITION_PHASE}" = cell-ready; then
|
||||
test "${CELL_PLAN_CHANGES}" = 0
|
||||
elif test "${TRANSITION_PHASE}" = cell-active; then
|
||||
test "${CELL_PLAN_CHANGES}" = 0
|
||||
recreate_fixed_one_instance
|
||||
elif test "${CELL_PLAN_CHANGES}" = 0; then
|
||||
recreate_fixed_one_instance
|
||||
else
|
||||
terraform -chdir=infra/terraform apply \
|
||||
-auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan"
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until \
|
||||
"${MIG_NAME}" \
|
||||
--stable \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" \
|
||||
--timeout 900
|
||||
|
||||
- name: Verify exact live cap and fresh matching heartbeat
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Make C3 the only staging placement cell
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode activate
|
||||
|
||||
- name: Verify the sole general canary after transition
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Restore the reviewed C2 and C3 placement set
|
||||
if: ${{ inputs.mode == 'restore-admission' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode restore \
|
||||
--general-cell-ids staging-gce-c2,staging-gce-c3
|
||||
|
||||
- name: Verify restored C3 admission and capacity
|
||||
if: ${{ inputs.mode == 'restore-admission' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Preserve C2 as the safe fallback after a failed transition
|
||||
if: ${{ failure() && inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode restore-fallback \
|
||||
--general-cell-ids "${FALLBACK_CELL_ID}"
|
||||
|
||||
refresh-asia-c4-image:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && github.ref == 'refs/heads/main' && inputs.mode == 'refresh-asia-c4-image' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 90
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
CELL_ORIGIN: https://c4.relay-staging.onorca.dev
|
||||
TARGET_CELL_ID: staging-gce-c4
|
||||
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
||||
PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }}
|
||||
TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }}
|
||||
APPROVED_PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Require the exact bounded C4 refresh
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "${CONFIRMATION}" = REFRESH_STAGING_ASIA_C4_IMAGE
|
||||
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
[[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test "${PREDECESSOR_IMAGE_DIGEST}" != "${TARGET_IMAGE_DIGEST}"
|
||||
test "${PREDECESSOR_IMAGE_DIGEST}" = "${APPROVED_PREDECESSOR_IMAGE_DIGEST}"
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Initialize the exact staging backend
|
||||
run: node dev/scripts/infra.mjs init --env staging
|
||||
|
||||
- name: Resolve the reviewed C4 image and shape
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cells="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
|
||||
shape="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${cells}")"
|
||||
test "$(jq -r '.hostname' <<< "${shape}")" = c4
|
||||
test "$(jq -r '.region' <<< "${shape}")" = asia-east2
|
||||
test "$(jq -r '.zone' <<< "${shape}")" = asia-east2-a
|
||||
test "$(jq -r '.machine_type' <<< "${shape}")" = e2-standard-4
|
||||
test "$(jq -r '.capacity_requests' <<< "${shape}")" = 6000
|
||||
test "$(jq -r '.database_pool_max' <<< "${shape}")" = 10
|
||||
test "$(jq -r '.connection_hard_cap' <<< "${shape}")" = 3000
|
||||
test "$(jq -r '.connection_unobserved_bound' <<< "${shape}")" = 60
|
||||
test "$(jq -r '.initially_enabled' <<< "${shape}")" = false
|
||||
desired_image="$(jq -r '.image' <<< "${shape}")"
|
||||
test "${desired_image}" = \
|
||||
"us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${TARGET_IMAGE_DIGEST}"
|
||||
served_digest="$(gcloud artifacts docker images describe "${desired_image}" \
|
||||
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${served_digest}" = "${TARGET_IMAGE_DIGEST}"
|
||||
mig_name="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
|
||||
| jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
|
||||
test "${mig_name}" = orca-cloud-staging-relay-gce-c4
|
||||
{
|
||||
echo "DESIRED_IMAGE=${desired_image}"
|
||||
echo "ROLLBACK_IMAGE=us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${PREDECESSOR_IMAGE_DIGEST}"
|
||||
echo "TARGET_ZONE=asia-east2-a"
|
||||
echo "MIG_NAME=${mig_name}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Save, validate, and classify the exact C4 plan
|
||||
id: plan
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
plan="${RUNNER_TEMP}/relay-c4-image-refresh.tfplan"
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
|
||||
-out="${plan}"
|
||||
result="$(terraform -chdir=infra/terraform show -json "${plan}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
|
||||
--unobserved-bound 60 --image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${ROLLBACK_IMAGE}")"
|
||||
case "$(jq -r '[.changes,.changeKind] | join(":")' <<< "${result}")" in
|
||||
2:replacement) refresh_phase=predecessor ;;
|
||||
0:none|*:obsolete-template-delete) refresh_phase=applied ;;
|
||||
*:manager-convergence|*:replacement-with-obsolete-template) refresh_phase=converging ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
{
|
||||
echo "PLAN_CHANGES=$(jq -r '.changes' <<< "${result}")"
|
||||
echo "REFRESH_PHASE=${refresh_phase}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- id: state-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Verify the exact selector and current C4 state
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.state-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \
|
||||
--expected-generation '' --expected-membership-sha256 '' --attempt-id '' \
|
||||
--image-digest "${TARGET_IMAGE_DIGEST}")"
|
||||
test "$(jq -r '.generation' <<< "${inspect}")" = "${EXPECTED_SELECTOR_GENERATION}"
|
||||
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \
|
||||
migration-only
|
||||
expected_digests="${TARGET_IMAGE_DIGEST}"
|
||||
if test "${REFRESH_PHASE}" = predecessor; then
|
||||
expected_digests="${PREDECESSOR_IMAGE_DIGEST}"
|
||||
elif test "${REFRESH_PHASE}" = converging; then
|
||||
expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
|
||||
fi
|
||||
if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \
|
||||
"${CELL_ORIGIN}/v1/admin/runtime-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data '{"v":1}')"; then
|
||||
current_digest="$(jq -er '.imageDigest' <<< "${runtime}")"
|
||||
case ",${expected_digests}," in
|
||||
*,"${current_digest}",*) ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
source_incarnation=''
|
||||
draining=forbidden
|
||||
if test "${REFRESH_PHASE}" != applied; then
|
||||
status="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
|
||||
source_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")"
|
||||
[[ "${source_incarnation}" =~ ^[0-9a-f-]{36}$ ]]
|
||||
if test "$(jq -r '.draining' <<< "${runtime}")" = true; then
|
||||
draining=required
|
||||
fi
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
|
||||
--heartbeat fresh --admission migration-only --draining "${draining}" \
|
||||
--activity quiescent --expected-image-digests "${expected_digests}"
|
||||
runtime_available=true
|
||||
else
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \
|
||||
--admission migration-only --draining either --activity restart-safe \
|
||||
--timeout-ms 300000
|
||||
source_incarnation=''
|
||||
runtime_available=false
|
||||
fi
|
||||
{
|
||||
echo "MIG_STABLE_AT_MS=0"
|
||||
echo "MUTATION_STARTED=false"
|
||||
echo "REPLACEMENT_STARTED_AT_MS=0"
|
||||
echo "RUNTIME_AVAILABLE=${runtime_available}"
|
||||
echo "SOURCE_INCARNATION=${source_incarnation}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- id: fence-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Fence C4 and prove it stayed empty before replacement
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test "${REFRESH_PHASE}" = applied; then exit 0; fi
|
||||
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
if test "${RUNTIME_AVAILABLE}" = false; then exit 0; fi
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --mode isolate
|
||||
fence_digests="${PREDECESSOR_IMAGE_DIGEST}"
|
||||
if test "${REFRESH_PHASE}" = converging; then
|
||||
fence_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
|
||||
--heartbeat fresh --admission migration-only --draining required \
|
||||
--activity quiescent --expected-image-digests "${fence_digests}"
|
||||
|
||||
- name: Apply the exact saved C4 plan
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test "${PLAN_CHANGES}" = 0 && test "${RUNTIME_AVAILABLE}" = true; then exit 0; fi
|
||||
if test "${REFRESH_PHASE}" = applied && test "${RUNTIME_AVAILABLE}" = false; then
|
||||
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
if test "${REFRESH_PHASE}" != applied; then
|
||||
replacement_started_at_ms="$(date -u +%s%3N)"
|
||||
echo "REPLACEMENT_STARTED_AT_MS=${replacement_started_at_ms}" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
if test "${PLAN_CHANGES}" != 0; then
|
||||
terraform -chdir=infra/terraform apply -auto-approve \
|
||||
"${RUNNER_TEMP}/relay-c4-image-refresh.tfplan"
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
if test "${RUNTIME_AVAILABLE}" = false && test "${REFRESH_PHASE}" = applied; then
|
||||
instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \
|
||||
| jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and
|
||||
.[0].currentAction == "NONE" then .[0].instance | split("/") | last
|
||||
else error("C4 is not one stable running instance") end')"
|
||||
gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \
|
||||
--instances "${instance}" --project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" --quiet
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
fi
|
||||
echo "MIG_STABLE_AT_MS=$(date -u +%s%3N)" >> "${GITHUB_ENV}"
|
||||
|
||||
- id: post-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Verify new C4 incarnation, image, and unchanged isolation
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
|
||||
--heartbeat fresh --admission migration-only --draining forbidden \
|
||||
--activity quiescent --expected-image-digests "${TARGET_IMAGE_DIGEST}" \
|
||||
--timeout-ms 240000
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \
|
||||
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-membership-sha256 '' --attempt-id '' \
|
||||
--image-digest "${TARGET_IMAGE_DIGEST}")"
|
||||
test "$(jq -r '.generation' <<< "${result}")" = "${EXPECTED_SELECTOR_GENERATION}"
|
||||
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \
|
||||
migration-only
|
||||
for _ in $(seq 1 36); do
|
||||
status="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
|
||||
if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \
|
||||
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
|
||||
-ge "${MIG_STABLE_AT_MS}"; then break; fi
|
||||
sleep 5
|
||||
done
|
||||
target_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")"
|
||||
test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true
|
||||
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
|
||||
-ge "${MIG_STABLE_AT_MS}"
|
||||
if test "${REFRESH_PHASE}" != applied; then
|
||||
if test -n "${SOURCE_INCARNATION}"; then
|
||||
test "${target_incarnation}" != "${SOURCE_INCARNATION}"
|
||||
fi
|
||||
test "$(jq -r '.status.runtime.startedAt' <<< "${status}")" \
|
||||
-ge "${REPLACEMENT_STARTED_AT_MS}"
|
||||
fi
|
||||
|
||||
- name: Require an empty targeted Terraform readback
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
plan="${RUNNER_TEMP}/relay-c4-image-readback.tfplan"
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
|
||||
-out="${plan}"
|
||||
result="$(terraform -chdir=infra/terraform show -json "${plan}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
|
||||
--unobserved-bound 60 --image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${ROLLBACK_IMAGE}")"
|
||||
test "$(jq -r '.changes' <<< "${result}")" = 0
|
||||
@@ -1,133 +0,0 @@
|
||||
name: Publish Relay Production Image
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Publish a new production image or mirror an existing immutable image to staging
|
||||
required: true
|
||||
default: publish
|
||||
type: choice
|
||||
options: [publish, mirror-staging]
|
||||
image-digest:
|
||||
description: Exact existing production digest for mirror-staging mode
|
||||
required: false
|
||||
type: string
|
||||
confirmation:
|
||||
description: Enter MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING for mirror-staging mode
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: publish-relay-production
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
REPOSITORY_ID: orca-cloud
|
||||
IMAGE_NAME: relay
|
||||
PUBLISH_MODE: ${{ inputs.mode }}
|
||||
MIRROR_DIGEST: ${{ inputs.image-digest }}
|
||||
MIRROR_CONFIRMATION: ${{ inputs.confirmation }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Validate the exact publish request before authentication
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test "${PUBLISH_MODE}" = mirror-staging; then
|
||||
[[ "${MIRROR_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test "${MIRROR_CONFIRMATION}" = MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING
|
||||
else
|
||||
test "${PUBLISH_MODE}" = publish
|
||||
test -z "${MIRROR_DIGEST}"
|
||||
test -z "${MIRROR_CONFIRMATION}"
|
||||
fi
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Configure Docker auth
|
||||
run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
|
||||
|
||||
- name: Build and publish immutable image
|
||||
if: ${{ inputs.mode == 'publish' }}
|
||||
run: |
|
||||
IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${GITHUB_SHA}"
|
||||
docker build -f apps/relay/Dockerfile -t "${IMAGE_TAG}" .
|
||||
docker push "${IMAGE_TAG}"
|
||||
DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')"
|
||||
test -n "${DIGEST}"
|
||||
IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${DIGEST}"
|
||||
{
|
||||
echo '### Terraform candidate image'
|
||||
echo
|
||||
echo "\`${IMAGE}\`"
|
||||
echo
|
||||
echo 'Declare this digest on a distinct disabled candidate cell in a reviewed Terraform PR.'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Build and publish immutable fence broker
|
||||
if: ${{ inputs.mode == 'publish' }}
|
||||
run: |
|
||||
IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker:sha-${GITHUB_SHA}"
|
||||
docker build \
|
||||
--build-arg "ORCA_RELAY_FENCE_IMAGE_COMMIT=${GITHUB_SHA}" \
|
||||
-f apps/relay-fence-broker/Dockerfile \
|
||||
-t "${IMAGE_TAG}" .
|
||||
docker push "${IMAGE_TAG}"
|
||||
DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')"
|
||||
test -n "${DIGEST}"
|
||||
IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker@${DIGEST}"
|
||||
{
|
||||
echo
|
||||
echo '### Terraform fence broker image'
|
||||
echo
|
||||
echo "\`${IMAGE}\`"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Mirror the exact production manifest to staging
|
||||
if: ${{ inputs.mode == 'mirror-staging' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
source_image="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${MIRROR_DIGEST}"
|
||||
target_tag="${GCP_REGION}-docker.pkg.dev/onorca-cloud-staging/${REPOSITORY_ID}/${IMAGE_NAME}:production-${MIRROR_DIGEST#sha256:}"
|
||||
source_digest="$(gcloud artifacts docker images describe "${source_image}" \
|
||||
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${source_digest}" = "${MIRROR_DIGEST}"
|
||||
docker pull "${source_image}"
|
||||
docker tag "${source_image}" "${target_tag}"
|
||||
docker push "${target_tag}"
|
||||
target_digest="$(gcloud artifacts docker images describe "${target_tag}" \
|
||||
--project onorca-cloud-staging --format='value(image_summary.digest)')"
|
||||
test "${target_digest}" = "${MIRROR_DIGEST}"
|
||||
{
|
||||
echo '### Mirrored Relay image'
|
||||
echo
|
||||
printf 'Production and staging now resolve the same immutable digest: %s.\n' \
|
||||
"${MIRROR_DIGEST}"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,568 +0,0 @@
|
||||
name: Deploy Push Gateway Production
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_sha:
|
||||
description: Full reviewed commit SHA to build (feature may remain unmerged)
|
||||
required: true
|
||||
type: string
|
||||
confirmation:
|
||||
description: Enter DEPLOY_PUSH_GATEWAY to shift production traffic
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# Serialize push traffic changes independently of Relay and the shared database.
|
||||
concurrency:
|
||||
group: production-push-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: >-
|
||||
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
|
||||
github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
SERVICE_NAME: orca-cloud-push
|
||||
REPOSITORY_ID: orca-cloud
|
||||
IMAGE_NAME: push
|
||||
PUSH_ORIGIN: https://push.onorca.dev
|
||||
PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud.iam.gserviceaccount.com
|
||||
# Scaling the serving revision must already hold, matching push_min_instances and
|
||||
# push_max_instances. Terraform owns both, and the candidate inherits them from the
|
||||
# service, so this deploy never passes a scaling flag: doing so would write a
|
||||
# Terraform-owned field that `lifecycle.ignore_changes` does not cover, and a later
|
||||
# `push_max_instances` raise would then be reverted by every deploy. These two values
|
||||
# are the expected shape, asserted before the candidate is created and again on the
|
||||
# candidate itself, so a deploy that would change the gateway's Cloud SQL draw fails.
|
||||
PUSH_MIN_INSTANCES: 1
|
||||
PUSH_MAX_INSTANCES: 2
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
SOURCE_SHA: ${{ inputs.source_sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Require the explicit deploy confirmation
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "${CONFIRMATION}" = DEPLOY_PUSH_GATEWAY
|
||||
[[ "${SOURCE_SHA}" =~ ^[a-f0-9]{40}$ ]]
|
||||
|
||||
# Keep the workflow and rollout lease on main; only the Docker build uses candidate code.
|
||||
- name: Fetch the immutable gateway source
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin "${SOURCE_SHA}"
|
||||
test "$(git rev-parse FETCH_HEAD)" = "${SOURCE_SHA}"
|
||||
mkdir -p "${RUNNER_TEMP}/push-source"
|
||||
git -C "${GITHUB_WORKSPACE}" archive "${SOURCE_SHA}" cloud \
|
||||
| tar -x -C "${RUNNER_TEMP}/push-source"
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Configure Docker auth
|
||||
run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
|
||||
|
||||
# Building an image does not need the deployment lease.
|
||||
- name: Build and publish the immutable gateway image
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
image_tag="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${SOURCE_SHA}"
|
||||
docker buildx build --push --platform linux/amd64 --provenance=false --metadata-file "${RUNNER_TEMP}/push-image.json" \
|
||||
-f "${RUNNER_TEMP}/push-source/cloud/apps/push/Dockerfile" \
|
||||
-t "${image_tag}" "${RUNNER_TEMP}/push-source/cloud"
|
||||
digest="$(jq -er '."containerimage.digest"' "${RUNNER_TEMP}/push-image.json")"
|
||||
[[ "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
echo "IMAGE=${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${digest}" \
|
||||
>> "${GITHUB_ENV}"
|
||||
echo "IMAGE_DIGEST=${digest}" >> "${GITHUB_ENV}"
|
||||
|
||||
# Refuse older images before they ever boot against production.
|
||||
- name: Require image support for inert validation
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker run --rm --network none --entrypoint node "${IMAGE}" --input-type=module -e '
|
||||
import { loadPushConfig } from "./apps/push/dist/config.js";
|
||||
const env = { ORCA_PUSH_PUBLIC_URL: "https://push.onorca.dev", ORCA_PUSH_MODE: "validation", ORCA_PUSH_FCM_PROJECT_ID: "onorca-cloud" };
|
||||
if (loadPushConfig(env).mode !== "validation") throw new Error("validation_mode_unsupported");
|
||||
let rejected = false;
|
||||
try { loadPushConfig({ ...env, ORCA_PUSH_MODE: "invalid" }); } catch { rejected = true; }
|
||||
if (!rejected) throw new Error("validation_mode_not_fail_closed");
|
||||
'
|
||||
|
||||
# Held across the deploy, not just a separate schema step: the gateway opens its pool and
|
||||
# applies its schema while the new revision starts, so the revision is the schema step.
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/push-rollout/production.lock
|
||||
|
||||
# Why: the candidate inherits the serving revision's scaling. A serving revision that has
|
||||
# drifted below the floor would hand the candidate a cold start on every notification, and
|
||||
# one that has drifted above the ceiling would hand it a larger Cloud SQL draw than the
|
||||
# rollout lease was taken for. Refuse to inherit either rather than latch it.
|
||||
- name: Record the serving revision and require its Terraform-owned scaling
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
|
||||
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test -n "${serving}"
|
||||
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
|
||||
if test "${revisions}" != "${serving}"; then
|
||||
echo 'Retire leftover revisions under the rollout lease before deploying; three pools are the limit.' >&2
|
||||
exit 1
|
||||
fi
|
||||
floor="$(gcloud run revisions describe "${serving}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
|
||||
if [[ "${floor:-0}" -lt "${PUSH_MIN_INSTANCES}" ]]; then
|
||||
echo "serving revision ${serving} holds ${floor:-0} minimum instances," \
|
||||
"below ${PUSH_MIN_INSTANCES}; deploying would inherit and latch it." >&2
|
||||
echo "Restore the floor first: gcloud run services update ${SERVICE_NAME}" \
|
||||
"--region ${GCP_REGION} --min-instances=${PUSH_MIN_INSTANCES}" >&2
|
||||
exit 1
|
||||
fi
|
||||
ceiling="$(gcloud run revisions describe "${serving}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
|
||||
test "${ceiling}" = "${PUSH_MAX_INSTANCES}"
|
||||
echo "serving revision ${serving} holds ${floor} minimum and ${ceiling} maximum instances"
|
||||
echo "ROLLBACK_REVISION=${serving}" >> "${GITHUB_ENV}"
|
||||
gcloud run revisions describe "${serving}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
> "${RUNNER_TEMP}/push-rollback-revision.json"
|
||||
image="$(jq -er '.status.imageDigest' "${RUNNER_TEMP}/push-rollback-revision.json")"
|
||||
[[ "${image}" =~ @sha256:[a-f0-9]{64}$ ]]
|
||||
echo "ROLLBACK_IMAGE=${image}" >> "${GITHUB_ENV}"
|
||||
jq -e 'all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE" or .value == "active")' \
|
||||
"${RUNNER_TEMP}/push-rollback-revision.json" > /dev/null
|
||||
|
||||
# Validation has no schema writes, HTTP mutations, worker, or pruners; tags alone do not
|
||||
# isolate background consumers from production.
|
||||
- name: Deploy the candidate revision with no traffic
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag="c${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
{
|
||||
echo "VALIDATION_DEPLOY_ATTEMPTED=true"
|
||||
echo "VALIDATION_REVISION=${SERVICE_NAME}-${tag}"
|
||||
echo "VALIDATION_TAG=${tag}"
|
||||
echo "CANDIDATE_TAG=${tag}"
|
||||
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
gcloud run deploy "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--image "${IMAGE}" \
|
||||
--tag "${tag}" \
|
||||
--revision-suffix "${tag}" \
|
||||
--no-traffic \
|
||||
--update-env-vars ORCA_PUSH_MODE=validation \
|
||||
--quiet
|
||||
candidate="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -er --arg tag "${tag}" \
|
||||
'[.status.traffic[] | select(.tag == $tag)]
|
||||
| if length == 1 then .[0] else error("tagged candidate is not unique") end')"
|
||||
test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
|
||||
echo "CANDIDATE_URL=$(jq -r '.url' <<< "${candidate}")" >> "${GITHUB_ENV}"
|
||||
|
||||
# A tagged revision is directly addressable and sits outside the service-wide cap, so the
|
||||
# candidate and the serving revision each draw up to the ceiling during the probe window.
|
||||
# Successor creation later requires three revision pools; assert the inherited ceiling.
|
||||
- name: Require the candidate to serve the exact image and inherited scaling
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
served="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format='value(spec.containers[0].image)')"
|
||||
test "${served}" = "${IMAGE}"
|
||||
test "${CANDIDATE_REVISION}" != "${ROLLBACK_REVISION}"
|
||||
candidate_ceiling="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
|
||||
test "${candidate_ceiling}" = "${PUSH_MAX_INSTANCES}"
|
||||
|
||||
- name: Probe the candidate readiness endpoint
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${CANDIDATE_URL}" =~ ^https://[^/]+$ ]]
|
||||
for attempt in $(seq 1 30); do
|
||||
code="$(curl -sS -o "${RUNNER_TEMP}/push-ready.json" -w '%{http_code}' \
|
||||
--max-time 10 "${CANDIDATE_URL}/ready" || true)"
|
||||
if test "${code}" = 200; then
|
||||
jq -e . < "${RUNNER_TEMP}/push-ready.json" > /dev/null
|
||||
curl --fail --silent --show-error --max-time 10 "${CANDIDATE_URL}/health" \
|
||||
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "validation"' > /dev/null
|
||||
echo "candidate ${CANDIDATE_REVISION} is ready after ${attempt} attempt(s)"
|
||||
exit 0
|
||||
fi
|
||||
echo "attempt ${attempt}: /ready returned ${code}"
|
||||
sleep 5
|
||||
done
|
||||
echo "candidate ${CANDIDATE_REVISION} never reported ready" >&2
|
||||
exit 1
|
||||
|
||||
# Why: a gateway that boots and answers /ready can still be unable to send. This proves the
|
||||
# runtime account's FCM grant end to end without delivering anything: validate_only stops
|
||||
# Google before any push, and the deliberately invalid token means a healthy credential
|
||||
# answers INVALID_ARGUMENT. PERMISSION_DENIED is the failure this step exists to catch.
|
||||
#
|
||||
# Only the four verdicts below are conclusive. A 429, a 5xx, or a transport failure says
|
||||
# nothing about the credential, so it is retried rather than treated as either answer; a
|
||||
# denied credential still fails on the first attempt, without burning the retries.
|
||||
- name: Prove the runtime identity can reach FCM
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
token="$(gcloud auth print-access-token \
|
||||
--impersonate-service-account "${PUSH_RUNTIME_SERVICE_ACCOUNT}")"
|
||||
test -n "${token}"
|
||||
echo "::add-mask::${token}"
|
||||
body='{"validate_only":true,"message":{"token":"orca-push-deploy-probe-invalid-token","notification":{"title":"Orca","body":"deploy probe"}}}'
|
||||
for attempt in $(seq 1 5); do
|
||||
code="$(curl -sS -o "${RUNNER_TEMP}/push-fcm.json" -w '%{http_code}' --max-time 20 \
|
||||
-X POST "https://fcm.googleapis.com/v1/projects/${GCP_PROJECT_ID}/messages:send" \
|
||||
-H "Authorization: Bearer ${token}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
--data "${body}" || true)"
|
||||
status="$(jq -r '.error.status // empty' < "${RUNNER_TEMP}/push-fcm.json" || true)"
|
||||
echo "attempt ${attempt}: FCM validate-only send returned HTTP ${code} status ${status:-OK}"
|
||||
if test "${status}" = PERMISSION_DENIED || test "${status}" = INVALID_ARGUMENT ||
|
||||
test "${code}" = 401 || test "${code}" = 403; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
if test "${status}" = PERMISSION_DENIED || test "${code}" = 401 || test "${code}" = 403; then
|
||||
echo "the push runtime identity cannot send through FCM" >&2
|
||||
exit 1
|
||||
fi
|
||||
test "${status}" = INVALID_ARGUMENT
|
||||
|
||||
# Cloud Run requires a successor before the latest revision can be deleted.
|
||||
- name: Retire inert validation and activate the verified image
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag="a${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
{
|
||||
echo "CANDIDATE_TAG=${tag}"
|
||||
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
|
||||
echo "ACTIVATION_ATTEMPTED=true"
|
||||
} >> "${GITHUB_ENV}"
|
||||
# This is the production-effect boundary: schema, pruners and workers start here.
|
||||
gcloud run deploy "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--image "${IMAGE}" --tag "${tag}" --revision-suffix "${tag}" \
|
||||
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--remove-tags "${VALIDATION_TAG}" --quiet
|
||||
gcloud run revisions delete "${VALIDATION_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
||||
echo "VALIDATION_RETIRED=true" >> "${GITHUB_ENV}"
|
||||
revision="$(gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
jq -e --arg image "${IMAGE}" --arg account "${PUSH_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--arg ceiling "${PUSH_MAX_INSTANCES}" --arg floor "${PUSH_MIN_INSTANCES}" \
|
||||
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" '
|
||||
def shape: del(.containers[0].image) |
|
||||
.containers[0].env = ((.containers[0].env // []) |
|
||||
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
|
||||
.spec.containers[0].image == $image and .spec.serviceAccountName == $account and
|
||||
all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
|
||||
(.spec | shape) == ($prior[0].spec | shape) and
|
||||
.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
|
||||
(.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber)' \
|
||||
<<< "${revision}" > /dev/null
|
||||
candidate="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -er --arg tag "${tag}" '[.status.traffic[] | select(.tag == $tag)]
|
||||
| if length == 1 then .[0] else error("active candidate is not unique") end')"
|
||||
test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
|
||||
url="$(jq -er '.url' <<< "${candidate}")"
|
||||
[[ "${url}" =~ ^https://[^/]+$ ]]
|
||||
curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
|
||||
curl --fail --silent --show-error --max-time 10 "${url}/health" \
|
||||
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
|
||||
|
||||
- name: Shift all traffic to the verified candidate
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "TRAFFIC_SHIFT_ATTEMPTED=true" >> "${GITHUB_ENV}"
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--to-revisions "${CANDIDATE_REVISION}=100" \
|
||||
--quiet
|
||||
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
|
||||
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test "${serving}" = "${CANDIDATE_REVISION}"
|
||||
echo "TRAFFIC_SHIFTED=true" >> "${GITHUB_ENV}"
|
||||
|
||||
# Why: the summary is written before the origin check, not after it. Once traffic has
|
||||
# moved, the rollback target is the single thing an operator needs, and a summary that only
|
||||
# appeared on success would be missing in exactly the run that needs it.
|
||||
- name: Publish the rollout summary
|
||||
if: ${{ always() && env.CANDIDATE_REVISION != '' && env.ROLLBACK_REVISION != '' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
{
|
||||
echo '### Push gateway rollout'
|
||||
echo
|
||||
echo "Source: ${SOURCE_SHA}"
|
||||
echo
|
||||
echo "Revision: \`${CANDIDATE_REVISION}\`"
|
||||
echo
|
||||
echo "Image: \`${IMAGE_DIGEST}\`"
|
||||
echo "Known-good image: \`${ROLLBACK_IMAGE}\`"
|
||||
echo
|
||||
echo "Recovery: deploy \`${ROLLBACK_IMAGE}\` as a new revision with" \
|
||||
"\`--remove-env-vars ORCA_PUSH_MODE --no-traffic --tag <unique-recovery-tag> --revision-suffix <unique-recovery-suffix>\`."
|
||||
echo 'Verify its exact digest, configuration, readiness and active mode, then promote and check the public origin.'
|
||||
echo 'Only then remove obsolete tags and delete rejected/previous revisions; never delete the latest revision.'
|
||||
echo 'The previous revision is retired after public checks; retain this immutable image for recovery under the rollout lease.'
|
||||
echo "Activation attempted: ${ACTIVATION_ATTEMPTED:-false}; traffic rollback cannot undo schema or deliveries."
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Verify the public origin after the shift
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for attempt in $(seq 1 30); do
|
||||
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \
|
||||
"${PUSH_ORIGIN}/ready" || true)"
|
||||
if test "${code}" = 200; then
|
||||
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
|
||||
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' > /dev/null
|
||||
echo "ROLLOUT_VERIFIED=true" >> "${GITHUB_ENV}"
|
||||
echo "${PUSH_ORIGIN} is ready after ${attempt} attempt(s)"
|
||||
exit 0
|
||||
fi
|
||||
echo "attempt ${attempt}: ${PUSH_ORIGIN}/ready returned ${code}"
|
||||
sleep 5
|
||||
done
|
||||
echo "${PUSH_ORIGIN} never reported ready after the shift" >&2
|
||||
exit 1
|
||||
|
||||
# Why: everything after the shift runs with production on the candidate. A failure there
|
||||
# is not a failure to deploy, it is a live gateway that has to go back, so the traffic move
|
||||
# is undone here rather than left to whoever reads the run.
|
||||
- name: Roll traffic back to the previous revision
|
||||
if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -n "${ROLLBACK_REVISION:-}"
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--to-revisions "${ROLLBACK_REVISION}=100" \
|
||||
--quiet
|
||||
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
|
||||
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test "${serving}" = "${ROLLBACK_REVISION}"
|
||||
echo "TRAFFIC_ROLLED_BACK=true" >> "${GITHUB_ENV}"
|
||||
{
|
||||
echo
|
||||
echo '### Push gateway rolled back'
|
||||
echo
|
||||
echo "Traffic returned to \`${ROLLBACK_REVISION}\`; the candidate" \
|
||||
"\`${CANDIDATE_REVISION}\` no longer serves HTTP; deletion below must stop its workers."
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
# Deleting a revision does not restore the service template that Terraform reconciles.
|
||||
- name: Restore the known-good service template
|
||||
if: ${{ (failure() || cancelled()) && env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "${TRAFFIC_SHIFT_ATTEMPTED:-false}" != true || test "${TRAFFIC_ROLLED_BACK:-false}" = true
|
||||
test -n "${ROLLBACK_IMAGE}"
|
||||
# A partial activation may leave validation plus active; free one slot before recovery.
|
||||
latest="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format='value(status.latestCreatedRevisionName)')"
|
||||
test -n "${latest}"
|
||||
if test "${VALIDATION_RETIRED:-false}" != true && test "${latest}" != "${VALIDATION_REVISION}"; then
|
||||
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--filter "metadata.name=${VALIDATION_REVISION}" --format='value(metadata.name)')"
|
||||
if test -n "${existing}"; then
|
||||
test "${existing}" = "${VALIDATION_REVISION}"
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--remove-tags "${VALIDATION_TAG}" --quiet
|
||||
gcloud run revisions delete "${VALIDATION_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
||||
fi
|
||||
fi
|
||||
tag="r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
echo "RECOVERY_TAG=${tag}" >> "${GITHUB_ENV}"
|
||||
echo "TEMPLATE_RECOVERY_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
|
||||
echo "Template recovery attempted: ${SERVICE_NAME}-${tag}, image ${ROLLBACK_IMAGE}." \
|
||||
>> "${GITHUB_STEP_SUMMARY}"
|
||||
# Known-good schema/workers can run here even though HTTP stays on the old revision.
|
||||
gcloud run deploy "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--image "${ROLLBACK_IMAGE}" --revision-suffix "${tag}" --tag "${tag}" \
|
||||
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
|
||||
gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
> "${RUNNER_TEMP}/push-recovered-service.json"
|
||||
gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
> "${RUNNER_TEMP}/push-recovery-revision.json"
|
||||
jq -e --arg image "${ROLLBACK_IMAGE}" --arg serving "${ROLLBACK_REVISION}" \
|
||||
--arg floor "${PUSH_MIN_INSTANCES}" --arg ceiling "${PUSH_MAX_INSTANCES}" \
|
||||
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" \
|
||||
--slurpfile recovered "${RUNNER_TEMP}/push-recovery-revision.json" '
|
||||
def shape: del(.containers[0].image) |
|
||||
.containers[0].env = ((.containers[0].env // []) |
|
||||
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
|
||||
.spec.template.spec.containers[0].image == $image and
|
||||
all(.spec.template.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
|
||||
$recovered[0].spec.containers[0].image == $image and
|
||||
all($recovered[0].spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
|
||||
($recovered[0].spec | shape) == ($prior[0].spec | shape) and
|
||||
.spec.template.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
|
||||
(.spec.template.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber) and
|
||||
([.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
length == 1 and .[0].revisionName == $serving and .[0].percent == 100)
|
||||
' "${RUNNER_TEMP}/push-recovered-service.json" > /dev/null
|
||||
echo "TEMPLATE_RESTORED=true" >> "${GITHUB_ENV}"
|
||||
echo 'Known-good image and normal mode restored; previous revision still serves HTTP.' \
|
||||
>> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Promote and verify the known-good recovery revision
|
||||
if: ${{ always() && env.TEMPLATE_RESTORED == 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
url="$(jq -er --arg tag "${RECOVERY_TAG}" \
|
||||
'.status.traffic[] | select(.tag == $tag) | .url' "${RUNNER_TEMP}/push-recovered-service.json")"
|
||||
[[ "${url}" =~ ^https://[^/]+$ ]]
|
||||
curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
|
||||
curl --fail --silent --show-error --max-time 10 "${url}/health" \
|
||||
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--to-revisions "${TEMPLATE_RECOVERY_REVISION}=100" --quiet
|
||||
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test "${serving}" = "${TEMPLATE_RECOVERY_REVISION}"
|
||||
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/ready" | jq -e '.ok == true'
|
||||
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
|
||||
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
|
||||
echo "RECOVERY_VERIFIED=true" >> "${GITHUB_ENV}"
|
||||
echo "Recovery revision ${TEMPLATE_RECOVERY_REVISION} now serves the known-good image." \
|
||||
>> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Delete the rejected candidate revision
|
||||
if: ${{ always() && env.RECOVERY_VERIFIED == 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test -z "${CANDIDATE_REVISION:-}"; then
|
||||
echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
|
||||
exit 0
|
||||
fi
|
||||
if test -n "${CANDIDATE_TAG:-}"; then
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--remove-tags "${CANDIDATE_TAG}" \
|
||||
--quiet
|
||||
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--filter "metadata.name=${CANDIDATE_REVISION}" --format='value(metadata.name)')"
|
||||
if test -n "${existing}"; then
|
||||
test "${existing}" = "${CANDIDATE_REVISION}"
|
||||
gcloud run revisions delete "${CANDIDATE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
||||
fi
|
||||
echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
|
||||
echo "candidate revision ${CANDIDATE_REVISION} is absent"
|
||||
|
||||
# Public checks commit the serving revision; cleanup failures must not roll it back.
|
||||
- name: Retire previous consumers after public checks
|
||||
if: ${{ always() && (env.ROLLOUT_VERIFIED == 'true' || (env.RECOVERY_VERIFIED == 'true' && env.CANDIDATE_DELETED == 'true')) }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
serving="${CANDIDATE_REVISION}"
|
||||
if test "${RECOVERY_VERIFIED:-false}" = true; then
|
||||
serving="${TEMPLATE_RECOVERY_REVISION}"
|
||||
fi
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --clear-tags --quiet
|
||||
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
|
||||
while IFS= read -r revision; do
|
||||
test -n "${revision}" || continue
|
||||
test "${revision}" != "${serving}" || continue
|
||||
case "${revision}" in
|
||||
"${ROLLBACK_REVISION}"|"${VALIDATION_REVISION}"|"${CANDIDATE_REVISION}") ;;
|
||||
*) echo "Unexpected revision ${revision}; manual retirement required." >&2; exit 1 ;;
|
||||
esac
|
||||
gcloud run revisions delete "${revision}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
||||
done <<< "${revisions}"
|
||||
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
|
||||
echo 'Obsolete revision resources retired; verify actual SQL session drain operationally.' \
|
||||
>> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Drop the candidate traffic tag
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -n "${CANDIDATE_TAG:-}" || exit 0
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--remove-tags "${CANDIDATE_TAG}" \
|
||||
--quiet
|
||||
@@ -1,414 +0,0 @@
|
||||
name: Recover Relay Staging C4 Image
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [Prove Relay Staging Capacity]
|
||||
types: [completed]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
confirmation:
|
||||
description: Enter RECOVER_STAGING_ASIA_C4_IMAGE
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
gate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event_name == 'workflow_dispatch' || (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion != 'success')) }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
recover: ${{ steps.trigger.outputs.recover }}
|
||||
steps:
|
||||
- name: Bind recovery to the exact failed C4 job
|
||||
working-directory: .
|
||||
id: trigger
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
SOURCE_RUN_EVENT: ${{ github.event.workflow_run.event }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test "${GITHUB_EVENT_NAME}" = workflow_dispatch; then
|
||||
test "${CONFIRMATION}" = RECOVER_STAGING_ASIA_C4_IMAGE
|
||||
echo "recover=true" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
test "${SOURCE_RUN_EVENT}" = workflow_dispatch
|
||||
[[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
||||
jobs="$(gh api --paginate \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")"
|
||||
count="$(jq -s '[.[].jobs[] | select(.name == "refresh-asia-c4-image" and
|
||||
(.conclusion == "failure" or .conclusion == "cancelled" or
|
||||
.conclusion == "timed_out"))] | length' <<< "${jobs}")"
|
||||
if test "${count}" = 0; then
|
||||
echo "recover=false" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
test "${count}" = 1
|
||||
echo "recover=true" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
recover:
|
||||
needs: gate
|
||||
if: ${{ needs.gate.outputs.recover == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 90
|
||||
environment: staging
|
||||
concurrency:
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
CELL_ORIGIN: https://c4.relay-staging.onorca.dev
|
||||
TARGET_CELL_ID: staging-gce-c4
|
||||
TARGET_ZONE: asia-east2-a
|
||||
MIG_NAME: orca-cloud-staging-relay-gce-c4
|
||||
PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7
|
||||
TARGET_IMAGE_DIGEST: sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Initialize the exact staging backend
|
||||
run: node dev/scripts/infra.mjs init --env staging
|
||||
|
||||
- id: preflight-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Inspect the exact C4 recovery state
|
||||
id: preflight
|
||||
timeout-minutes: 3
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.preflight-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \
|
||||
--expected-generation '' --expected-membership-sha256 '' --attempt-id '' \
|
||||
--image-digest "${PREDECESSOR_IMAGE_DIGEST}")"
|
||||
generation="$(jq -er '.generation' <<< "${inspect}")"
|
||||
[[ "${generation}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \
|
||||
migration-only
|
||||
echo "selector_generation=${generation}" >> "${GITHUB_OUTPUT}"
|
||||
if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \
|
||||
"${CELL_ORIGIN}/v1/admin/runtime-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data '{"v":1}')"; then
|
||||
current_digest="$(jq -er '.imageDigest' <<< "${runtime}")"
|
||||
[[ "${current_digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
status="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
|
||||
jq -e '.draining | type == "boolean"' <<< "${runtime}" >/dev/null
|
||||
{
|
||||
echo "runtime_available=true"
|
||||
echo "current_digest=${current_digest}"
|
||||
echo "draining=$(jq -r '.draining' <<< "${runtime}")"
|
||||
echo "ready=$(jq -r '.status.runtime.ready == true' <<< "${status}")"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
else
|
||||
echo "runtime_available=false" >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
|
||||
- name: Classify both exact recovery end states
|
||||
id: recovery-plan
|
||||
timeout-minutes: 10
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay"
|
||||
predecessor_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}"
|
||||
target_image="${image_repository}@${TARGET_IMAGE_DIGEST}"
|
||||
served_digest="$(gcloud artifacts docker images describe "${predecessor_image}" \
|
||||
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${served_digest}" = "${PREDECESSOR_IMAGE_DIGEST}"
|
||||
cells="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
|
||||
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].image' <<< "${cells}")" = \
|
||||
"${target_image}"
|
||||
target_plan="${RUNNER_TEMP}/relay-c4-image-target.tfplan"
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars -lock-timeout=5m \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
|
||||
-out="${target_plan}"
|
||||
target_result="$(terraform -chdir=infra/terraform show -json "${target_plan}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
|
||||
--unobserved-bound 60 --image "${target_image}" \
|
||||
--rollback-image "${predecessor_image}")"
|
||||
target_state="$(jq -r '[.changes,.changeKind] | join(":")' <<< "${target_result}")"
|
||||
case "${target_state}" in
|
||||
0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
|
||||
--arg image "${predecessor_image}" '.[$cell].image = $image' <<< "${cells}")"
|
||||
jq -n --argjson cells "${recovery_cells}" \
|
||||
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-recovery.tfvars.json"
|
||||
plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan"
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-c4-recovery.tfvars.json" \
|
||||
-lock-timeout=5m \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
|
||||
-out="${plan}"
|
||||
result="$(terraform -chdir=infra/terraform show -json "${plan}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
|
||||
--unobserved-bound 60 --image "${predecessor_image}" \
|
||||
--rollback-image "${target_image}")"
|
||||
changes="$(jq -r '.changes' <<< "${result}")"
|
||||
change_kind="$(jq -r '.changeKind' <<< "${result}")"
|
||||
case "${changes}:${change_kind}" in
|
||||
0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
mig="$(gcloud compute instance-groups managed describe "${MIG_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)"
|
||||
mig_stable="$(jq -r '.status.isStable == true and .status.versionTarget.isReached == true' \
|
||||
<<< "${mig}")"
|
||||
instances="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)"
|
||||
instance_stable="$(jq -r 'length == 1 and .[0].instanceStatus == "RUNNING" and
|
||||
.[0].currentAction == "NONE"' <<< "${instances}")"
|
||||
action=rollback-predecessor
|
||||
recovery_digest="${PREDECESSOR_IMAGE_DIGEST}"
|
||||
if test "${{ steps.preflight.outputs.runtime_available }}" = true && \
|
||||
test "${{ steps.preflight.outputs.ready }}" = true && \
|
||||
test "${{ steps.preflight.outputs.draining }}" = false && \
|
||||
test "${mig_stable}" = true && test "${instance_stable}" = true; then
|
||||
if test "${{ steps.preflight.outputs.current_digest }}" = "${TARGET_IMAGE_DIGEST}" && \
|
||||
test "${target_state}" = 0:none; then
|
||||
action=verify-target
|
||||
recovery_digest="${TARGET_IMAGE_DIGEST}"
|
||||
elif test "${{ steps.preflight.outputs.current_digest }}" = \
|
||||
"${PREDECESSOR_IMAGE_DIGEST}" && test "${changes}:${change_kind}" = 0:none; then
|
||||
action=verify-predecessor
|
||||
fi
|
||||
fi
|
||||
{
|
||||
echo "changes=${changes}"
|
||||
echo "change_kind=${change_kind}"
|
||||
echo "action=${action}"
|
||||
echo "recovery_digest=${recovery_digest}"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- id: fence-auth
|
||||
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Fence C4 before predecessor recovery
|
||||
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
|
||||
timeout-minutes: 6
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
|
||||
current_digest="${{ steps.preflight.outputs.current_digest }}"
|
||||
if test -n "${current_digest}" && [[ ",${expected_digests}," != *",${current_digest},"* ]]; then
|
||||
expected_digests="${expected_digests},${current_digest}"
|
||||
fi
|
||||
if curl --silent --show-error --fail-with-body --max-time 30 --request POST \
|
||||
"${CELL_ORIGIN}/v1/admin/drain" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data '{"v":1,"graceMs":0}' \
|
||||
>/dev/null; then
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
|
||||
--heartbeat fresh --admission migration-only --draining required \
|
||||
--activity quiescent \
|
||||
--expected-image-digests "${expected_digests}" \
|
||||
--timeout-ms 240000
|
||||
else
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \
|
||||
--admission migration-only --draining either --activity restart-safe \
|
||||
--timeout-ms 240000
|
||||
fi
|
||||
|
||||
- name: Apply and stabilize the saved predecessor plan
|
||||
id: apply
|
||||
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
CHANGES: ${{ steps.recovery-plan.outputs.changes }}
|
||||
CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan"
|
||||
if test "${CHANGES}" != 0; then
|
||||
terraform -chdir=infra/terraform apply -auto-approve "${plan}"
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Restart only when the plan did not replace C4
|
||||
id: restore
|
||||
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
APPLY_STABLE_AT_MS: ${{ steps.apply.outputs.stable_at_ms }}
|
||||
CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ "${CHANGE_KIND}" =~ ^(replacement|replacement-with-obsolete-template|manager-convergence)$ ]]; then
|
||||
echo "stable_at_ms=${APPLY_STABLE_AT_MS}" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \
|
||||
| jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and
|
||||
.[0].currentAction == "NONE" then .[0].instance | split("/") | last
|
||||
else error("C4 is not one stable running instance") end')"
|
||||
gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \
|
||||
--instances "${instance}" --project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" --quiet
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Require an empty selected-image recovery readback
|
||||
timeout-minutes: 8
|
||||
env:
|
||||
RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay"
|
||||
recovery_image="${image_repository}@${RECOVERY_DIGEST}"
|
||||
other_image="${image_repository}@${TARGET_IMAGE_DIGEST}"
|
||||
if test "${RECOVERY_DIGEST}" = "${TARGET_IMAGE_DIGEST}"; then
|
||||
other_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}"
|
||||
fi
|
||||
cells="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
|
||||
recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" --arg image "${recovery_image}" \
|
||||
'.[$cell].image = $image' <<< "${cells}")"
|
||||
jq -n --argjson cells "${recovery_cells}" \
|
||||
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-readback.tfvars.json"
|
||||
readback="${RUNNER_TEMP}/relay-c4-image-recovery-readback.tfplan"
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-c4-readback.tfvars.json" \
|
||||
-lock-timeout=5m \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
|
||||
-out="${readback}"
|
||||
readback_result="$(terraform -chdir=infra/terraform show -json "${readback}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
|
||||
--unobserved-bound 60 --image "${recovery_image}" \
|
||||
--rollback-image "${other_image}")"
|
||||
test "$(jq -r '.changes' <<< "${readback_result}")" = 0
|
||||
|
||||
- id: verify-auth
|
||||
if: ${{ always() }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Verify the recovered image and unchanged isolation
|
||||
if: ${{ always() && steps.verify-auth.outcome == 'success' }}
|
||||
timeout-minutes: 8
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.verify-auth.outputs.id_token }}
|
||||
SELECTOR_GENERATION: ${{ steps.preflight.outputs.selector_generation }}
|
||||
STABLE_AT_MS: ${{ steps.restore.outputs.stable_at_ms }}
|
||||
RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
|
||||
--heartbeat fresh --admission migration-only --draining forbidden \
|
||||
--activity quiescent --expected-image-digests "${RECOVERY_DIGEST}" \
|
||||
--timeout-ms 240000
|
||||
stable_at_ms="${STABLE_AT_MS:-0}"
|
||||
for _ in $(seq 1 36); do
|
||||
status="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
|
||||
if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \
|
||||
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
|
||||
-ge "${stable_at_ms}"; then break; fi
|
||||
sleep 5
|
||||
done
|
||||
test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true
|
||||
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
|
||||
-ge "${stable_at_ms}"
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \
|
||||
--expected-generation "${SELECTOR_GENERATION}" \
|
||||
--expected-membership-sha256 '' --attempt-id '' \
|
||||
--image-digest "${RECOVERY_DIGEST}")"
|
||||
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \
|
||||
migration-only
|
||||
@@ -1,60 +0,0 @@
|
||||
name: Requeue Relay Staging C4 Recovery
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [Recover Relay Staging C4 Image]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: relay-staging-c4-recovery-requeue
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
requeue:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion == 'cancelled') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Requeue only a cancelled protected recovery job
|
||||
working-directory: .
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
||||
jobs="$(gh api --paginate \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")"
|
||||
count="$(jq -s '[.[].jobs[] | select(.name == "recover" and
|
||||
.conclusion == "cancelled" and .started_at == null)] | length' <<< "${jobs}")"
|
||||
if test "${count}" = 0; then exit 0; fi
|
||||
test "${count}" = 1
|
||||
runs="$(gh api \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/cloud-recover-relay-staging-c4-image.yml/runs?branch=main&per_page=100")"
|
||||
active=0
|
||||
while IFS= read -r run_id; do
|
||||
active_jobs="$(gh api --paginate \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?filter=latest")"
|
||||
if jq -se '
|
||||
([.[].jobs[] | select(.name == "gate" and .conclusion == "success")] | length) == 1 and
|
||||
([.[].jobs[] | select(.name == "recover" and .status != "completed")] | length) == 1
|
||||
' <<< "${active_jobs}" >/dev/null; then
|
||||
active=1
|
||||
break
|
||||
fi
|
||||
done < <(jq -r --arg source "${SOURCE_RUN_ID}" \
|
||||
'.workflow_runs[] | select((.id | tostring) != $source and
|
||||
.status != "completed") | .id' <<< "${runs}")
|
||||
if test "${active}" != 0; then exit 0; fi
|
||||
gh workflow run cloud-recover-relay-staging-c4-image.yml \
|
||||
--repo "${GITHUB_REPOSITORY}" --ref main \
|
||||
-f confirmation=RECOVER_STAGING_ASIA_C4_IMAGE
|
||||
@@ -116,7 +116,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
app: [relay, push, relay-fence-broker, cloud-sql-proxy]
|
||||
app: [relay, push]
|
||||
env:
|
||||
APP: ${{ matrix.app }}
|
||||
TRIVY_IMAGE: aquasec/trivy:0.76.1@sha256:790fd66d1697395d8693d1acaa1ea702406a20f109b6fc7bac8db623eca6c06c
|
||||
@@ -126,20 +126,12 @@ jobs:
|
||||
- name: Build or resolve the runtime image
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test "$APP" = cloud-sql-proxy; then
|
||||
IMAGE="$(sed -n '/variable "relay_gce_cloud_sql_proxy_image"/,/^}/s/.*default *= "\([^"]*\)"/\1/p' infra/terraform/variables.tf)"
|
||||
[[ "$IMAGE" =~ ^gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:[a-f0-9]{64}$ ]]
|
||||
docker pull --platform linux/amd64 "$IMAGE"
|
||||
else
|
||||
IMAGE="orca-cloud-security:$APP"
|
||||
docker build --platform linux/amd64 \
|
||||
--build-arg "ORCA_RELAY_FENCE_IMAGE_COMMIT=$GITHUB_SHA" \
|
||||
--file "apps/$APP/Dockerfile" --tag "$IMAGE" .
|
||||
fi
|
||||
IMAGE="orca-cloud-security:$APP"
|
||||
docker build --platform linux/amd64 \
|
||||
--file "apps/$APP/Dockerfile" --tag "$IMAGE" .
|
||||
echo "IMAGE=$IMAGE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Smoke test the nonroot Node runtime
|
||||
if: matrix.app != 'cloud-sql-proxy'
|
||||
run: |
|
||||
docker run --rm --platform linux/amd64 --entrypoint sh "$IMAGE" -ec '
|
||||
test "$(id -u)" -ne 0
|
||||
@@ -150,23 +142,6 @@ jobs:
|
||||
node --input-type=module -e "await import(\"./apps/$1/dist/$module.js\")"
|
||||
' sh "$APP"
|
||||
|
||||
- name: Smoke test broker tools and checksum fallback
|
||||
if: matrix.app == 'relay-fence-broker'
|
||||
run: |
|
||||
docker run --rm --platform linux/amd64 --entrypoint sh "$IMAGE" -ec '
|
||||
gcloud version
|
||||
printf relay-security > /tmp/hash-input
|
||||
test "$(gcloud storage hash --format="value(crc32c_hash)" /tmp/hash-input)" = "6oayYQ=="
|
||||
terraform version
|
||||
cd infra/terraform
|
||||
terraform init -backend=false -input=false -lockfile=readonly
|
||||
terraform validate
|
||||
'
|
||||
|
||||
- name: Smoke test the official SQL proxy binary
|
||||
if: matrix.app == 'cloud-sql-proxy'
|
||||
run: docker run --rm --platform linux/amd64 "$IMAGE" --version
|
||||
|
||||
- name: Scan every severity and retain the complete inventory
|
||||
run: |
|
||||
mkdir -p security-reports
|
||||
@@ -177,16 +152,12 @@ jobs:
|
||||
docker run --rm --volume "$PWD/security-reports:/reports" "$TRIVY_IMAGE" convert \
|
||||
--format cyclonedx --output /reports/sbom.cdx.json /reports/vulnerabilities.json
|
||||
|
||||
- name: Reject available fixes except dated upstream proxy rebuilds
|
||||
- name: Reject available fixes
|
||||
run: |
|
||||
jq '(.Results[]?.Vulnerabilities) |= ((. // []) | map(select(.FixedVersion != null and .FixedVersion != "")))' \
|
||||
security-reports/vulnerabilities.json > security-reports/fixable.json
|
||||
ignorefile=''
|
||||
if test "$APP" = cloud-sql-proxy; then
|
||||
ignorefile=/repo/security/cloud-sql-proxy.trivyignore.yaml
|
||||
fi
|
||||
docker run --rm --volume "$PWD:/repo:ro" "$TRIVY_IMAGE" convert \
|
||||
--ignorefile "$ignorefile" --show-suppressed --exit-code 1 \
|
||||
--ignorefile '' --show-suppressed --exit-code 1 \
|
||||
/repo/security-reports/fixable.json
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
@@ -234,38 +205,3 @@ jobs:
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: pnpm test
|
||||
|
||||
# Fork pull requests reach this job, so it never configures a backend, never plans, and never
|
||||
# holds a credential. Only the relay root ships here; foundation and apps stay private.
|
||||
terraform:
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
|
||||
- run: terraform -chdir=infra/terraform fmt -check -recursive
|
||||
- run: terraform -chdir=infra/terraform init -backend=false -input=false
|
||||
- run: terraform -chdir=infra/terraform validate
|
||||
|
||||
- name: Inventory provider binary vulnerabilities
|
||||
env:
|
||||
TRIVY_IMAGE: aquasec/trivy:0.76.1@sha256:790fd66d1697395d8693d1acaa1ea702406a20f109b6fc7bac8db623eca6c06c
|
||||
run: |
|
||||
mkdir -p security-reports
|
||||
docker run --rm --volume "$PWD:/repo" "$TRIVY_IMAGE" rootfs \
|
||||
--scanners vuln --ignorefile '' --format json \
|
||||
--output /repo/security-reports/providers.json /repo/infra/terraform/.terraform/providers
|
||||
docker run --rm --volume "$PWD/security-reports:/reports" "$TRIVY_IMAGE" convert \
|
||||
--format cyclonedx --output /reports/providers.cdx.json /reports/providers.json
|
||||
docker run --rm --volume "$PWD/security-reports:/reports" "$TRIVY_IMAGE" convert /reports/providers.json
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: cloud-terraform-provider-security
|
||||
path: cloud/security-reports/
|
||||
retention-days: 30
|
||||
|
||||
@@ -1386,7 +1386,7 @@ jobs:
|
||||
run: |
|
||||
$files = node config/scripts/pr-package-test-paths.mjs
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
pnpm exec vitest run --config config/vitest.config.ts $files
|
||||
node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts @files
|
||||
|
||||
# Why its own step: the CPU burners that make taskkill slow would starve
|
||||
# every other test file sharing the runner.
|
||||
|
||||
Reference in New Issue
Block a user