mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
Move Relay deployment configuration into private cloud operations (#31792)
* refactor(cloud): move deployment configuration into private operations repo * docs(cloud): clarify private operator ownership * fix(ci): install pnpm through npm-compatible bootstrap * fix(ci): avoid native downloader metadata timeout * test(push): move deployment preflight contract into private repo * fix(ci): bootstrap Node before installing pnpm in minimal containers * fix(ci): pass Windows boundary test paths through the Node runner * fix(ci): run Windows boundary Vitest with installed Node
This commit is contained in:
@@ -1,678 +0,0 @@
|
||||
name: Bootstrap Relay Staging Capacity
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
confirmation:
|
||||
description: Enter BOOTSTRAP_STAGING_CAPACITY
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
bootstrap:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
GCP_REGION: us-central1
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
LEGACY_C3_IMAGE_DIGEST: sha256:2d0f6e6db2b0eb9d6aba188698de8330f8c30b4e76badfcf0fac3f3eb9508a87
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: deploy-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: capacity-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: access_token
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Require explicit bootstrap confirmation
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: test "${CONFIRMATION}" = "BOOTSTRAP_STAGING_CAPACITY"
|
||||
|
||||
- name: Read and verify reviewed 600/60 topology
|
||||
shell: bash
|
||||
run: |
|
||||
node dev/scripts/infra.mjs init --env staging
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
|
||||
> "${RUNNER_TEMP}/relay-gce-state.json"
|
||||
DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'local.relay_director_cells_json' | jq -r '.')"
|
||||
DESIRED_IMAGES_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.image })' \
|
||||
| jq -r '.')"
|
||||
DESIRED_ZONES_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.zone })' \
|
||||
| jq -r '.')"
|
||||
DESIRED_TARGET_SIZES_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode(local.relay_gce_cell_target_sizes)' | jq -r '.')"
|
||||
jq -e \
|
||||
--argjson images "${DESIRED_IMAGES_JSON}" \
|
||||
--argjson target_sizes "${DESIRED_TARGET_SIZES_JSON}" \
|
||||
'([.[] | select(.id == "staging-gce-c2")] | length == 1) and
|
||||
([.[] | select(.id == "staging-gce-c3")] | length == 1) and
|
||||
(any(.[]; .id == "staging-gce-c2" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and
|
||||
(any(.[]; .id == "staging-gce-c3" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and
|
||||
($images["staging-gce-c2"] == $images["staging-gce-c3"]) and
|
||||
($target_sizes["staging-gce-c2"] == 1) and
|
||||
($target_sizes["staging-gce-c3"] == 1)' \
|
||||
<<< "${DESIRED_CELLS_JSON}" >/dev/null
|
||||
jq \
|
||||
--argjson desired "${DESIRED_CELLS_JSON}" \
|
||||
--argjson images "${DESIRED_IMAGES_JSON}" \
|
||||
--argjson zones "${DESIRED_ZONES_JSON}" \
|
||||
'($desired | map({key: .id, value: .}) | from_entries) as $cells |
|
||||
to_entries | map(. as $entry | {
|
||||
key: $entry.key,
|
||||
value: ($entry.value + {
|
||||
origin: $cells[$entry.key].url,
|
||||
image: $images[$entry.key],
|
||||
zone: $zones[$entry.key],
|
||||
connection_hard_cap: $cells[$entry.key].connectionHardCap,
|
||||
connection_unobserved_bound: $cells[$entry.key].connectionUnobservedBound
|
||||
})
|
||||
}) | from_entries' \
|
||||
"${RUNNER_TEMP}/relay-gce-state.json" \
|
||||
> "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
ACTIVE_REVISION="$(gcloud run services describe orca-cloud-relay-staging \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region us-central1 \
|
||||
--format=json \
|
||||
| jq -r '
|
||||
[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test -n "${ACTIVE_REVISION}"
|
||||
DESIRED_IMAGE="$(jq -r '.["staging-gce-c2"]' <<< "${DESIRED_IMAGES_JSON}")"
|
||||
gcloud run revisions describe "${ACTIVE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region us-central1 \
|
||||
--format=json \
|
||||
| jq -e \
|
||||
--arg image "${DESIRED_IMAGE}" \
|
||||
--arg capacity_service_account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
'(.spec.containers[0].image == $image) and
|
||||
any(.spec.containers[0].env[]?;
|
||||
.name == "ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT" and
|
||||
.value == $capacity_service_account)' >/dev/null
|
||||
CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format=json \
|
||||
| jq -cer '
|
||||
[.spec.containers[0].env[]? |
|
||||
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
|
||||
if length == 1 then .[0] | fromjson else error("missing director topology") end')"
|
||||
jq -e --argjson desired "${DESIRED_CELLS_JSON}" '
|
||||
def without_bootstrap_capacity:
|
||||
map(if .id == "staging-gce-c2" or .id == "staging-gce-c3"
|
||||
then del(.connectionHardCap, .connectionUnobservedBound)
|
||||
else . end);
|
||||
without_bootstrap_capacity == ($desired | without_bootstrap_capacity)
|
||||
' <<< "${CURRENT_CELLS_JSON}" >/dev/null
|
||||
|
||||
- name: Bootstrap C2 then C3
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
topology="${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
|
||||
fixed_one_instance_name() {
|
||||
local cell_id="$1"
|
||||
local mig_name zone
|
||||
mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")"
|
||||
zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
|
||||
gcloud compute instance-groups managed list-instances \
|
||||
"${mig_name}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${zone}" \
|
||||
--format=json \
|
||||
| jq -er '
|
||||
if length == 1 and .[0].instanceStatus == "RUNNING" and
|
||||
.[0].currentAction == "NONE"
|
||||
then .[0].instance | split("/") | last
|
||||
else error("legacy cell does not have one stable running instance") end'
|
||||
}
|
||||
|
||||
fixed_one_instance_id() {
|
||||
local cell_id="$1"
|
||||
local instance_name="$2"
|
||||
local zone
|
||||
zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
|
||||
gcloud compute instances describe "${instance_name}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${zone}" \
|
||||
--format='value(id)'
|
||||
}
|
||||
|
||||
write_legacy_metrics() {
|
||||
local cell_id="$1"
|
||||
local instance_id="$2"
|
||||
local after="$3"
|
||||
local output="$4"
|
||||
gcloud logging read \
|
||||
"resource.type=\"gce_instance\" AND
|
||||
resource.labels.instance_id=\"${instance_id}\" AND
|
||||
jsonPayload.event=\"orca_relay_runtime_metrics\" AND
|
||||
jsonPayload.cellId=\"${cell_id}\" AND
|
||||
timestamp>=\"${after}\"" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--limit 10 \
|
||||
--order desc \
|
||||
--format json \
|
||||
| jq '[.[] | {
|
||||
timestamp,
|
||||
cellId: .jsonPayload.cellId,
|
||||
metricVersion: .jsonPayload.metricVersion,
|
||||
totalConnections: .jsonPayload.totalConnections,
|
||||
preAuthConnections: .jsonPayload.preAuthConnections,
|
||||
controls: .jsonPayload.controls,
|
||||
splices: .jsonPayload.splices,
|
||||
pendingSplices: .jsonPayload.pendingSplices,
|
||||
queuedBytes: .jsonPayload.queuedBytes
|
||||
}]' > "${output}"
|
||||
}
|
||||
|
||||
verify_legacy_cell() {
|
||||
local cell_id="$1"
|
||||
local admission="$2"
|
||||
local after="$3"
|
||||
local expected_instance_id="$4"
|
||||
local runtime_started_after="${5:-}"
|
||||
local previous_incarnation_digest="${6:-}"
|
||||
local hard_cap="${7:-}"
|
||||
local unobserved_bound="${8:-}"
|
||||
local capacity_state="${9:-}"
|
||||
local current_instance current_instance_id metrics origin result
|
||||
local runtime_start_args=() incarnation_args=() capacity_args=()
|
||||
current_instance="$(fixed_one_instance_name "${cell_id}")"
|
||||
current_instance_id="$(fixed_one_instance_id "${cell_id}" "${current_instance}")"
|
||||
test "${current_instance_id}" = "${expected_instance_id}"
|
||||
metrics="${RUNNER_TEMP}/${cell_id}-legacy-runtime-metrics.json"
|
||||
origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
|
||||
if test -n "${runtime_started_after}"; then
|
||||
runtime_start_args=(--runtime-started-after "${runtime_started_after}")
|
||||
fi
|
||||
if test -n "${previous_incarnation_digest}"; then
|
||||
incarnation_args=(--previous-incarnation-digest "${previous_incarnation_digest}")
|
||||
fi
|
||||
if test -n "${hard_cap}"; then
|
||||
capacity_args=(--hard-cap "${hard_cap}" --unobserved-bound "${unobserved_bound}")
|
||||
fi
|
||||
if test -n "${capacity_state}"; then
|
||||
capacity_args+=(--capacity-state "${capacity_state}")
|
||||
fi
|
||||
for _attempt in $(seq 1 18); do
|
||||
write_legacy_metrics \
|
||||
"${cell_id}" "${current_instance_id}" "${after}" "${metrics}"
|
||||
if result="$(node dev/scripts/verify-relay-legacy-bootstrap.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--admission "${admission}" \
|
||||
--expected-image-digest "${LEGACY_C3_IMAGE_DIGEST}" \
|
||||
--metrics-after "${after}" \
|
||||
--metrics-file "${metrics}" \
|
||||
"${runtime_start_args[@]}" \
|
||||
"${incarnation_args[@]}" \
|
||||
"${capacity_args[@]}")"; then
|
||||
echo "${result}"
|
||||
return 0
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
post_admin() {
|
||||
local origin="$1"
|
||||
local path="$2"
|
||||
local body="$3"
|
||||
curl --fail --silent --show-error \
|
||||
--request POST \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "${body}" \
|
||||
"${origin}${path}"
|
||||
}
|
||||
|
||||
runtime_kind() {
|
||||
local cell_id="$1"
|
||||
local origin desired_digest digest
|
||||
origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
|
||||
desired_digest="$(jq -r --arg cell "${cell_id}" \
|
||||
'.[$cell].image | split("@") | last' "${topology}")"
|
||||
digest="$(post_admin "${origin}" /v1/admin/runtime-status '{"v":1}' \
|
||||
| jq -er '.imageDigest')"
|
||||
if test "${digest}" = "${LEGACY_C3_IMAGE_DIGEST}"; then
|
||||
echo legacy
|
||||
elif test "${digest}" = "${desired_digest}"; then
|
||||
echo modern
|
||||
else
|
||||
echo 'bootstrap cell image is neither legacy nor reviewed' >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
cell_admission() {
|
||||
local cell_id="$1"
|
||||
post_admin "${DIRECTOR_ORIGIN}" /v1/admin/cell-status \
|
||||
"$(jq -cn --arg cell "${cell_id}" '{v:1, cellId:$cell}')" \
|
||||
| jq -er '.status.admissionState |
|
||||
if . == "general" or . == "migration-only" then .
|
||||
else error("bootstrap admission is not recoverable") end'
|
||||
}
|
||||
|
||||
verify_modern_cell() {
|
||||
local cell_id="$1"
|
||||
local admission="$2"
|
||||
local origin
|
||||
origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission "${admission}" \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
}
|
||||
|
||||
ensure_modern_general() {
|
||||
local cell_id="$1"
|
||||
local admission="$2"
|
||||
verify_modern_cell "${cell_id}" "${admission}"
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--mode restore \
|
||||
--general-cell-ids "${cell_id}"
|
||||
verify_modern_cell "${cell_id}" general
|
||||
}
|
||||
|
||||
prepare_legacy_c3_fallback() {
|
||||
legacy_c3_metrics_boundary="$(node -e \
|
||||
'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')"
|
||||
legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
|
||||
legacy_c3_instance_id="$(fixed_one_instance_id \
|
||||
staging-gce-c3 "${legacy_c3_instance}")"
|
||||
verify_legacy_cell \
|
||||
staging-gce-c3 general \
|
||||
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}"
|
||||
node dev/scripts/probe-relay-legacy-admission.mjs \
|
||||
--cell-origin https://c3.relay-staging.onorca.dev
|
||||
legacy_c3_restart_started_after=
|
||||
legacy_c3_old_incarnation=
|
||||
}
|
||||
|
||||
normalize_legacy_c3() {
|
||||
legacy_pre_boundary="$(node -e \
|
||||
'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')"
|
||||
legacy_c2_instance="$(fixed_one_instance_name staging-gce-c2)"
|
||||
legacy_c2_instance_id="$(fixed_one_instance_id \
|
||||
staging-gce-c2 "${legacy_c2_instance}")"
|
||||
verify_legacy_cell \
|
||||
staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}"
|
||||
node dev/scripts/probe-relay-legacy-admission.mjs \
|
||||
--cell-origin https://c2.relay-staging.onorca.dev
|
||||
|
||||
legacy_c3_isolated=false
|
||||
restore_legacy_c3_fallback() {
|
||||
if test "${legacy_c3_isolated}" = true; then
|
||||
verify_legacy_cell \
|
||||
staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}"
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id staging-gce-c3 \
|
||||
--mode restore-fallback \
|
||||
--general-cell-ids staging-gce-c2
|
||||
fi
|
||||
}
|
||||
|
||||
trap restore_legacy_c3_fallback EXIT
|
||||
legacy_c3_isolated=true
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin https://c3.relay-staging.onorca.dev \
|
||||
--cell-id staging-gce-c3 \
|
||||
--mode isolate
|
||||
legacy_c3_drain_boundary="$(node -e \
|
||||
'process.stdout.write(new Date().toISOString())')"
|
||||
legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
|
||||
legacy_c3_instance_id="$(fixed_one_instance_id \
|
||||
staging-gce-c3 "${legacy_c3_instance}")"
|
||||
legacy_c3_drained="$(verify_legacy_cell \
|
||||
staging-gce-c3 migration-only \
|
||||
"${legacy_c3_drain_boundary}" "${legacy_c3_instance_id}")"
|
||||
echo "${legacy_c3_drained}"
|
||||
legacy_c3_old_incarnation="$(jq -er '.incarnationDigest' \
|
||||
<<< "${legacy_c3_drained}")"
|
||||
legacy_c3_restart_started_after="$(node -e \
|
||||
'process.stdout.write(new Date().toISOString())')"
|
||||
gcloud compute instance-groups managed recreate-instances \
|
||||
orca-cloud-staging-relay-gce-c3 \
|
||||
--instances "${legacy_c3_instance}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone us-central1-a \
|
||||
--quiet
|
||||
gcloud compute instance-groups managed wait-until \
|
||||
orca-cloud-staging-relay-gce-c3 \
|
||||
--stable \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone us-central1-a \
|
||||
--timeout 900
|
||||
legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
|
||||
legacy_c3_instance_id="$(fixed_one_instance_id \
|
||||
staging-gce-c3 "${legacy_c3_instance}")"
|
||||
legacy_c3_metrics_boundary="$(node -e \
|
||||
'process.stdout.write(new Date().toISOString())')"
|
||||
verify_legacy_cell \
|
||||
staging-gce-c3 migration-only \
|
||||
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
|
||||
"${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id staging-gce-c3 \
|
||||
--mode restore \
|
||||
--general-cell-ids staging-gce-c2,staging-gce-c3
|
||||
verify_legacy_cell \
|
||||
staging-gce-c3 general \
|
||||
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
|
||||
"${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
|
||||
legacy_c3_isolated=false
|
||||
trap - EXIT
|
||||
}
|
||||
|
||||
roll_cell() (
|
||||
local cell_id="$1"
|
||||
local fallback_cell_id="$2"
|
||||
local target_kind="$3"
|
||||
local fallback_kind="$4"
|
||||
local active_revision cell_origin current_cells_json desired_bound desired_cap
|
||||
local desired_cells_json director_result image mig_name plan
|
||||
local fallback_origin plan_changes plan_result restored target_drain_boundary
|
||||
local target_instance target_instance_id zone
|
||||
cell_origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
|
||||
fallback_origin="$(jq -r --arg cell "${fallback_cell_id}" \
|
||||
'.[$cell].origin' "${topology}")"
|
||||
zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
|
||||
mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")"
|
||||
image="$(jq -r --arg cell "${cell_id}" '.[$cell].image' "${topology}")"
|
||||
desired_cap="$(jq -r --arg cell "${cell_id}" \
|
||||
'.[$cell].connection_hard_cap' "${topology}")"
|
||||
desired_bound="$(jq -r --arg cell "${cell_id}" \
|
||||
'.[$cell].connection_unobserved_bound' "${topology}")"
|
||||
plan="${RUNNER_TEMP}/${cell_id}-capacity-bootstrap.tfplan"
|
||||
restored=false
|
||||
|
||||
restore_fallback() {
|
||||
if test "${restored}" = false; then
|
||||
verify_fallback
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--mode restore-fallback \
|
||||
--general-cell-ids "${fallback_cell_id}"
|
||||
fi
|
||||
}
|
||||
|
||||
verify_fallback() {
|
||||
if test "${fallback_kind}" = legacy; then
|
||||
verify_legacy_cell \
|
||||
"${fallback_cell_id}" general \
|
||||
"${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
|
||||
"${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
|
||||
return
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${fallback_origin}" \
|
||||
--cell-id "${fallback_cell_id}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
}
|
||||
|
||||
verify_fallback
|
||||
trap restore_fallback EXIT
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${cell_origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--mode isolate
|
||||
target_drain_boundary="$(node -e \
|
||||
'process.stdout.write(new Date().toISOString())')"
|
||||
if test "${target_kind}" = legacy; then
|
||||
target_instance="$(fixed_one_instance_name "${cell_id}")"
|
||||
target_instance_id="$(fixed_one_instance_id "${cell_id}" "${target_instance}")"
|
||||
verify_legacy_cell \
|
||||
"${cell_id}" migration-only \
|
||||
"${target_drain_boundary}" "${target_instance_id}" \
|
||||
'' '' "${desired_cap}" "${desired_bound}" absent-or-stale
|
||||
else
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${cell_origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity quiescent
|
||||
fi
|
||||
|
||||
active_revision="$(gcloud run services describe orca-cloud-relay-staging \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format=json \
|
||||
| jq -r '
|
||||
[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test -n "${active_revision}"
|
||||
current_cells_json="$(gcloud run revisions describe "${active_revision}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format=json \
|
||||
| jq -cer '
|
||||
[.spec.containers[0].env[]? |
|
||||
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
|
||||
if length == 1 then .[0] | fromjson else error("missing director topology") end')"
|
||||
desired_cells_json="$(jq -ce \
|
||||
--arg cell "${cell_id}" \
|
||||
--argjson cap "${desired_cap}" \
|
||||
--argjson bound "${desired_bound}" \
|
||||
'map(if .id == $cell then . + {
|
||||
connectionHardCap: $cap,
|
||||
connectionUnobservedBound: $bound
|
||||
} else . end)' <<< "${current_cells_json}")"
|
||||
director_result="$(node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service orca-cloud-relay-staging \
|
||||
--image "${image}" \
|
||||
--role director \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--capacity-cell-id "${cell_id}" \
|
||||
--director-cells-json "${desired_cells_json}" \
|
||||
--prune-revisions true \
|
||||
--release-id "bootstrap-${cell_id}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}")"
|
||||
echo "${director_result}"
|
||||
if test "${target_kind}" = legacy; then
|
||||
verify_legacy_cell \
|
||||
"${cell_id}" migration-only \
|
||||
"${target_drain_boundary}" "${target_instance_id}" \
|
||||
'' '' "${desired_cap}" "${desired_bound}"
|
||||
else
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${cell_origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--hard-cap "${desired_cap}" \
|
||||
--unobserved-bound "${desired_bound}" \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity quiescent
|
||||
fi
|
||||
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
"-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \
|
||||
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \
|
||||
-out="${plan}"
|
||||
plan_result="$(terraform -chdir=infra/terraform show -json "${plan}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode bootstrap-cell \
|
||||
--cell-id "${cell_id}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--image "${image}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
|
||||
echo "${plan_result}"
|
||||
plan_changes="$(jq -r '.changes' <<< "${plan_result}")"
|
||||
[[ "${plan_changes}" =~ ^(0|2)$ ]]
|
||||
if test "${plan_changes}" = 2; then
|
||||
terraform -chdir=infra/terraform apply -auto-approve "${plan}"
|
||||
else
|
||||
target_instance="$(fixed_one_instance_name "${cell_id}")"
|
||||
gcloud compute instance-groups managed recreate-instances "${mig_name}" \
|
||||
--instances "${target_instance}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${zone}" \
|
||||
--quiet
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until "${mig_name}" \
|
||||
--stable \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${zone}" \
|
||||
--timeout 900
|
||||
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${cell_origin}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${cell_id}" \
|
||||
--mode restore \
|
||||
--general-cell-ids staging-gce-c2,staging-gce-c3
|
||||
restored=true
|
||||
trap - EXIT
|
||||
)
|
||||
|
||||
c2_kind="$(runtime_kind staging-gce-c2)"
|
||||
c3_kind="$(runtime_kind staging-gce-c3)"
|
||||
c2_admission="$(cell_admission staging-gce-c2)"
|
||||
c3_admission="$(cell_admission staging-gce-c3)"
|
||||
bootstrap_phase="$(node dev/scripts/classify-relay-staging-bootstrap.mjs \
|
||||
--c2-kind "${c2_kind}" \
|
||||
--c2-admission "${c2_admission}" \
|
||||
--c3-kind "${c3_kind}" \
|
||||
--c3-admission "${c3_admission}")"
|
||||
jq -cn --arg phase "${bootstrap_phase}" \
|
||||
'{event:"relay_staging_bootstrap_phase", phase:$phase}'
|
||||
|
||||
case "${bootstrap_phase}" in
|
||||
normalize-and-roll-both)
|
||||
normalize_legacy_c3
|
||||
roll_cell staging-gce-c2 staging-gce-c3 legacy legacy
|
||||
roll_cell staging-gce-c3 staging-gce-c2 legacy modern
|
||||
;;
|
||||
resume-c2-then-c3)
|
||||
prepare_legacy_c3_fallback
|
||||
roll_cell staging-gce-c2 staging-gce-c3 legacy legacy
|
||||
roll_cell staging-gce-c3 staging-gce-c2 legacy modern
|
||||
;;
|
||||
roll-c2)
|
||||
ensure_modern_general staging-gce-c3 "${c3_admission}"
|
||||
roll_cell staging-gce-c2 staging-gce-c3 legacy modern
|
||||
;;
|
||||
roll-c3)
|
||||
ensure_modern_general staging-gce-c2 "${c2_admission}"
|
||||
roll_cell staging-gce-c3 staging-gce-c2 legacy modern
|
||||
;;
|
||||
complete)
|
||||
ensure_modern_general staging-gce-c2 "${c2_admission}"
|
||||
ensure_modern_general staging-gce-c3 "${c3_admission}"
|
||||
;;
|
||||
*)
|
||||
echo 'unsupported staging bootstrap phase' >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Verify both bootstrapped cells
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
for number in 2 3; do
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "https://c${number}.relay-staging.onorca.dev" \
|
||||
--cell-id "staging-gce-c${number}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
done
|
||||
@@ -1,288 +0,0 @@
|
||||
name: Deploy Relay Asia Topology
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
environment:
|
||||
description: Target Relay environment
|
||||
required: true
|
||||
type: choice
|
||||
options: [staging, production]
|
||||
mode:
|
||||
description: Validate a saved plan or apply that exact plan
|
||||
required: true
|
||||
default: plan
|
||||
type: choice
|
||||
options: [plan, apply]
|
||||
cell-ids:
|
||||
description: Exact reviewed comma-separated cell set (Asia, or the US cells at the Asia shape)
|
||||
required: true
|
||||
type: string
|
||||
image:
|
||||
description: Full environment Relay image pinned by sha256 digest
|
||||
required: true
|
||||
type: string
|
||||
confirmation:
|
||||
description: Enter APPLY_RELAY_ASIA_TOPOLOGY for apply mode
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
topology:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 30
|
||||
environment: ${{ inputs.environment }}
|
||||
env:
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
TARGET_ENVIRONMENT: ${{ inputs.environment }}
|
||||
TARGET_CELL_IDS: ${{ inputs.cell-ids }}
|
||||
TARGET_IMAGE: ${{ inputs.image }}
|
||||
GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }}
|
||||
CLOUD_SQL_INSTANCE: ${{ inputs.environment == 'production' && 'orca-cloud-auth-db' || 'orca-cloud-staging-auth-db' }}
|
||||
VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360
|
||||
VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17
|
||||
# SHOW max_connections on the live instance, 2026-09-16; no flag is set.
|
||||
VERIFIED_DEFAULT_MAX_CONNECTIONS: '500'
|
||||
TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }}
|
||||
TF_VARS: ${{ inputs.environment == 'production' && 'environments/production.tfvars' || 'environments/staging.tfvars' }}
|
||||
TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
TOPOLOGY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Validate the reviewed request before authentication
|
||||
shell: bash
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -n "${TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${TOPOLOGY_SERVICE_ACCOUNT}"
|
||||
target_region=asia-east2
|
||||
case "${TARGET_ENVIRONMENT}:${TARGET_CELL_IDS}" in
|
||||
staging:staging-gce-c4) ;;
|
||||
production:production-gce-c27,production-gce-c28,production-gce-c29) ;;
|
||||
production:production-gce-c30) ;;
|
||||
production:production-gce-c31) ;;
|
||||
production:production-gce-c32,production-gce-c33) target_region=us-central1 ;;
|
||||
production:production-gce-c34) ;;
|
||||
production:production-gce-c35) ;;
|
||||
*) echo "cell-ids do not match the reviewed environment topology" >&2; exit 1 ;;
|
||||
esac
|
||||
echo "TARGET_REGION=${target_region}" >> "${GITHUB_ENV}"
|
||||
[[ "${TARGET_IMAGE}" =~ ^us-central1-docker\.pkg\.dev/${GCP_PROJECT_ID}/orca-cloud/relay@sha256:[0-9a-f]{64}$ ]]
|
||||
if test "${DEPLOY_MODE}" = apply; then
|
||||
test "${CONFIRMATION}" = APPLY_RELAY_ASIA_TOPOLOGY
|
||||
else
|
||||
test "${DEPLOY_MODE}" = plan
|
||||
test -z "${CONFIRMATION}"
|
||||
fi
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ env.TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ env.TOPOLOGY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }}
|
||||
object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }}
|
||||
|
||||
- name: Require the checked Cloud SQL connection budget
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
budget="$(node dev/scripts/relay-cloud-sql-connection-budget.mjs)"
|
||||
checked_max="$(jq -er '.maxConnections' <<< "${budget}")"
|
||||
jq -e '.withinBudget == true' <<< "${budget}" >/dev/null
|
||||
if test "${TARGET_ENVIRONMENT}" = production; then
|
||||
instance="$(gcloud sql instances describe "${CLOUD_SQL_INSTANCE}" \
|
||||
--project "${GCP_PROJECT_ID}" --format=json)"
|
||||
live_flag="$(jq -er '[.settings.databaseFlags[]? |
|
||||
select(.name == "max_connections") | .value] |
|
||||
if length <= 1 then (.[0] // "") else error("duplicate max_connections flags") end' \
|
||||
<<< "${instance}")"
|
||||
if test -n "${live_flag}"; then
|
||||
live_max="${live_flag}"
|
||||
live_source=explicit-flag
|
||||
else
|
||||
# No flag: the ceiling is the tier default measured for this exact
|
||||
# shape; fail closed if the shape changes.
|
||||
test "$(jq -er '.settings.tier' <<< "${instance}")" = \
|
||||
"${VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER}"
|
||||
test "$(jq -er '.databaseVersion' <<< "${instance}")" = \
|
||||
"${VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION}"
|
||||
live_max="${VERIFIED_DEFAULT_MAX_CONNECTIONS}"
|
||||
live_source=verified-shape-default
|
||||
fi
|
||||
test "${live_max}" = "${checked_max}"
|
||||
else
|
||||
live_max="not-read-for-staging"
|
||||
live_source=not-read-for-staging
|
||||
fi
|
||||
{
|
||||
echo "### Relay Cloud SQL connection budget"
|
||||
echo "- Checked maximum: ${checked_max}"
|
||||
echo "- Configured maximum: $(jq -er '.configuredMaximum' <<< "${budget}")"
|
||||
echo "- Rollout operating maximum: $(jq -er '.operatingMaximum' <<< "${budget}")"
|
||||
echo "- Explicit reserve: $(jq -er '.explicitReserve' <<< "${budget}")"
|
||||
echo "- Production live max_connections: ${live_max}"
|
||||
echo "- Production live maximum source: ${live_source}"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Initialize the exact environment state
|
||||
run: terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}"
|
||||
|
||||
- id: targets
|
||||
name: Build the exact additive target set
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
file="${RUNNER_TEMP}/relay-asia-targets"
|
||||
: > "${file}"
|
||||
# The root region's network predates every cell; only asia-east2's is additive.
|
||||
if test "${TARGET_REGION}" = asia-east2; then
|
||||
printf '%s\n' \
|
||||
'-target=google_compute_subnetwork.relay_gce_additional["asia-east2"]' \
|
||||
'-target=google_compute_router.relay_gce_additional["asia-east2"]' \
|
||||
'-target=google_compute_router_nat.relay_gce_additional["asia-east2"]' >> "${file}"
|
||||
fi
|
||||
printf '%s\n' '-target=google_compute_url_map.relay_gce[0]' >> "${file}"
|
||||
IFS=, read -ra cells <<< "${TARGET_CELL_IDS}"
|
||||
for cell_id in "${cells[@]}"; do
|
||||
printf '%s\n' \
|
||||
"-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \
|
||||
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \
|
||||
"-target=google_compute_backend_service.relay_gce_cell[\"${cell_id}\"]" >> "${file}"
|
||||
done
|
||||
echo "file=${file}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- id: live-images
|
||||
name: Plan every non-target cell at the image it serves
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# The URL map target pulls every cell's template into the plan, and same-cap rolls
|
||||
# leave committed images behind the served ones; read only templates out of state.
|
||||
cells="${RUNNER_TEMP}/relay-committed-cells.json"
|
||||
live="${RUNNER_TEMP}/relay-live-cell-templates.json"
|
||||
overlay="${RUNNER_TEMP}/relay-asia-live-images.tfvars.json"
|
||||
committed_plan="${RUNNER_TEMP}/relay-committed-cells.tfplan"
|
||||
mapfile -t targets < "${{ steps.targets.outputs.file }}"
|
||||
# Not console: it evaluates every output against state, where a new cell has no MIG yet.
|
||||
terraform -chdir=infra/terraform plan -input=false -refresh=false -lock=false \
|
||||
-var-file="${TF_VARS}" "${targets[@]}" -out="${committed_plan}" > /dev/null
|
||||
terraform -chdir=infra/terraform show -json "${committed_plan}" \
|
||||
| jq -ce '.variables.relay_gce_cells.value | objects' > "${cells}"
|
||||
terraform -chdir=infra/terraform show -json | jq -ce '[
|
||||
.values.root_module.resources[]?
|
||||
| select(.mode == "managed" and
|
||||
.type == "google_compute_instance_template" and .name == "relay_gce_cell")
|
||||
| { index, metadata_startup_script: .values.metadata_startup_script }
|
||||
]' > "${live}"
|
||||
summary="$(node dev/scripts/relay-live-cell-image-overlay.mjs \
|
||||
--cells-json "${cells}" --live-templates-json "${live}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" --output "${overlay}")"
|
||||
echo "file=${overlay}" >> "${GITHUB_OUTPUT}"
|
||||
{
|
||||
echo "### Live images held for non-target cells"
|
||||
echo "- Cells whose committed image differs from the served one: $(jq -r '.drifted | join(", ")' <<< "${summary}")"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Create and validate the saved topology plan
|
||||
id: plan
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
plan="${RUNNER_TEMP}/relay-asia-topology.tfplan"
|
||||
plan_json="${RUNNER_TEMP}/relay-asia-topology.json"
|
||||
mapfile -t targets < "${{ steps.targets.outputs.file }}"
|
||||
terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \
|
||||
-var-file="${TF_VARS}" -var-file="${{ steps.live-images.outputs.file }}" \
|
||||
"${targets[@]}" -out="${plan}"
|
||||
terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}"
|
||||
committed="${RUNNER_TEMP}/relay-committed-asia-topology.json"
|
||||
jq -e '{
|
||||
relay_gce_cells: .variables.relay_gce_cells.value,
|
||||
relay_gce_additional_region_subnetwork_cidrs:
|
||||
.variables.relay_gce_additional_region_subnetwork_cidrs.value
|
||||
}' "${plan_json}" > "${committed}"
|
||||
node dev/scripts/prepare-relay-asia-topology-input.mjs \
|
||||
--existing-json "${committed}" \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--image "${TARGET_IMAGE}"
|
||||
result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \
|
||||
--plan-json "${plan_json}" \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--region "${TARGET_REGION}" \
|
||||
--image "${TARGET_IMAGE}")"
|
||||
changes="$(jq -er '.changes' <<< "${result}")"
|
||||
digest="$(sha256sum "${plan}" | awk '{print $1}')"
|
||||
echo "plan=${plan}" >> "${GITHUB_OUTPUT}"
|
||||
echo "changes=${changes}" >> "${GITHUB_OUTPUT}"
|
||||
{
|
||||
echo "### Relay Asia topology saved plan"
|
||||
echo "- Environment: ${TARGET_ENVIRONMENT}"
|
||||
echo "- Cells: ${TARGET_CELL_IDS}"
|
||||
echo "- Region: ${TARGET_REGION}"
|
||||
echo "- Mutating resources: ${changes}"
|
||||
echo "- Saved-plan SHA-256: ${digest}"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Apply only the validated saved plan
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
run: terraform -chdir=infra/terraform apply -input=false -auto-approve "${{ steps.plan.outputs.plan }}"
|
||||
|
||||
- name: Prove the exact topology targets converged
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mapfile -t targets < "${{ steps.targets.outputs.file }}"
|
||||
plan="${RUNNER_TEMP}/relay-asia-topology-readback.tfplan"
|
||||
plan_json="${RUNNER_TEMP}/relay-asia-topology-readback.json"
|
||||
terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \
|
||||
-var-file="${TF_VARS}" -var-file="${{ steps.live-images.outputs.file }}" \
|
||||
"${targets[@]}" -out="${plan}"
|
||||
terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}"
|
||||
result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \
|
||||
--plan-json "${plan_json}" \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--region "${TARGET_REGION}" \
|
||||
--image "${TARGET_IMAGE}")"
|
||||
test "$(jq -er '.changes' <<< "${result}")" = 0
|
||||
|
||||
- name: Record the required selector-safe next step
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
run: |
|
||||
{
|
||||
echo "### Required next step"
|
||||
echo "The VMs are not eligible for ordinary placement yet."
|
||||
echo "Register the exact new cells atomically as migration-only before any director configuration lists them."
|
||||
echo "Rollback is migration-only admission; do not destroy the Asia network on rollout day."
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,92 +0,0 @@
|
||||
name: Deploy Relay Fence Broker
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
image-digest:
|
||||
description: Immutable broker image digest built from this main commit
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: >-
|
||||
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
|
||||
github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
SERVICE_NAME: orca-cloud-relay-fence
|
||||
IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay-fence-broker
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
|
||||
- name: Resolve exact-commit broker image
|
||||
run: |
|
||||
[[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
|
||||
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \
|
||||
--format='value(image_summary.digest)')"
|
||||
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
|
||||
TAGS="$(gcloud artifacts docker tags list "${IMAGE_REPOSITORY}" \
|
||||
--filter="version:${IMAGE_DIGEST}" \
|
||||
--format=json)"
|
||||
jq -e --arg tag "/tags/sha-${GITHUB_SHA}" \
|
||||
'any(.[]; .tag | endswith($tag))' <<< "${TAGS}"
|
||||
echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Deploy broker image only
|
||||
run: |
|
||||
gcloud run services update "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--image "${IMAGE}" \
|
||||
--quiet
|
||||
|
||||
- name: Verify ready singleton revision
|
||||
run: |
|
||||
SERVICE="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format=json)"
|
||||
jq -e '.status.conditions[] | select(.type == "Ready" and .status == "True")' \
|
||||
<<< "${SERVICE}"
|
||||
REVISION="$(jq -r \
|
||||
'[.status.traffic[] | select((.percent // 0) == 100)] |
|
||||
if length == 1 then .[0].revisionName // empty else empty end' \
|
||||
<<< "${SERVICE}")"
|
||||
test -n "${REVISION}"
|
||||
SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format='value(spec.containers[0].image)')"
|
||||
test "${SERVED_IMAGE}" = "${IMAGE}"
|
||||
@@ -1,825 +0,0 @@
|
||||
name: Deploy Relay Production Capacity Job
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
mode:
|
||||
required: true
|
||||
type: string
|
||||
target-cell-id:
|
||||
required: true
|
||||
type: string
|
||||
confirmation:
|
||||
required: true
|
||||
type: string
|
||||
monitor-run-id:
|
||||
required: true
|
||||
type: string
|
||||
monitor-run-attempt:
|
||||
required: true
|
||||
type: string
|
||||
evidence-mode:
|
||||
required: true
|
||||
type: string
|
||||
wave-cell-ids:
|
||||
required: true
|
||||
type: string
|
||||
wave-index:
|
||||
required: true
|
||||
type: string
|
||||
source-wave-run-id:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
capacity:
|
||||
if: ${{ github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 75
|
||||
environment: production
|
||||
env:
|
||||
# These drains are unpaced, but restart-safe still waits out the production pace window
|
||||
# from the drain, the same one the same-cap drain uses.
|
||||
RESTART_SAFE_PACE_WINDOW_MS: '300000'
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
DIRECTOR_SERVICE_NAME: orca-cloud-relay
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
|
||||
CAPACITY_CELL_IDS: production-gce-c7,production-gce-c8,production-gce-c9,production-gce-c10,production-gce-c13,production-gce-c14,production-gce-c15,production-gce-c16,production-gce-c19,production-gce-c20,production-gce-c21,production-gce-c22,production-gce-c23,production-gce-c24,production-gce-c25,production-gce-c26
|
||||
PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d
|
||||
COMPATIBLE_DIRECTOR_IMAGE_DIGEST: sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73
|
||||
COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
EVIDENCE_MODE: ${{ inputs.evidence-mode }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
||||
WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }}
|
||||
WAVE_INDEX: ${{ inputs.wave-index }}
|
||||
SOURCE_WAVE_RUN_ID: ${{ inputs.source-wave-run-id }}
|
||||
steps:
|
||||
- name: Require exact reusable-workflow invocation
|
||||
working-directory: .
|
||||
run: |
|
||||
[[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]]
|
||||
if test "${EVIDENCE_MODE}" = continuation; then
|
||||
test "${DEPLOY_MODE}" = apply
|
||||
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
|
||||
test "${WAVE_CELL_IDS}" != none
|
||||
test "${SOURCE_WAVE_RUN_ID}" = none
|
||||
elif test "${EVIDENCE_MODE}" = resume; then
|
||||
test "${DEPLOY_MODE}" = apply
|
||||
test "${WAVE_INDEX}" = resume
|
||||
test "${WAVE_CELL_IDS}" != none
|
||||
[[ "${SOURCE_WAVE_RUN_ID}" =~ ^[0-9]+$ ]]
|
||||
else
|
||||
test "${EVIDENCE_MODE}" = single
|
||||
test "${WAVE_CELL_IDS}" = none
|
||||
test "${WAVE_INDEX}" = 0
|
||||
test "${SOURCE_WAVE_RUN_ID}" = none
|
||||
fi
|
||||
|
||||
- name: Require production workflow configuration
|
||||
working-directory: .
|
||||
env:
|
||||
DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
CAPACITY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
run: |
|
||||
test -n "${GCP_REGION}"
|
||||
test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${DEPLOY_SERVICE_ACCOUNT}"
|
||||
test -n "${CAPACITY_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${CAPACITY_SERVICE_ACCOUNT}"
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: resume-provenance
|
||||
if: ${{ inputs.evidence-mode == 'resume' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
test "${MONITOR_RUN_ATTEMPT}" = 1
|
||||
case "${MONITOR_RUN_ID}:${SOURCE_WAVE_RUN_ID}:${WAVE_CELL_IDS}:${TARGET_CELL_ID}" in
|
||||
31554591366:31555510376:production-gce-c16,production-gce-c15,production-gce-c14,production-gce-c13:production-gce-c13)
|
||||
EXPECTED_SHA=a917e8e1fc1a2654e8cb81ba39b57733ec56be9c
|
||||
EXPECTED_SOURCE_ATTEMPT=1
|
||||
;;
|
||||
31562760783:31563664692:production-gce-c10,production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c10)
|
||||
EXPECTED_SHA=6082e9ca89a918ca51f0c87db003f5e8805b64b7
|
||||
EXPECTED_SOURCE_ATTEMPT=1
|
||||
;;
|
||||
31571019947:31572080665:production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c8)
|
||||
EXPECTED_SHA=e59958130c9d9b7a6cd805df2678d08997842c7c
|
||||
EXPECTED_SOURCE_ATTEMPT=2
|
||||
;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
MONITOR_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${MONITOR_RUN_ID}" \
|
||||
--jq 'select(.name == "Monitor Relay Production" and
|
||||
.path == ".github/workflows/cloud-monitor-relay-production.yml" and
|
||||
.head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and
|
||||
.event == "workflow_dispatch" and .conclusion == "success" and .run_attempt == 1) |
|
||||
.head_sha')"
|
||||
SOURCE_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \
|
||||
--jq 'select(.name == "Deploy Relay Production Capacity" and
|
||||
.path == ".github/workflows/cloud-deploy-relay-production-capacity.yml" and
|
||||
.head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and
|
||||
.event == "workflow_dispatch" and .conclusion == "failure") |
|
||||
.head_sha')"
|
||||
SOURCE_ATTEMPT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \
|
||||
--jq '.run_attempt')"
|
||||
[[ "${MONITOR_SHA}" =~ ^[0-9a-f]{40}$ ]]
|
||||
test "${MONITOR_SHA}" = "${EXPECTED_SHA}"
|
||||
test "${SOURCE_SHA}" = "${MONITOR_SHA}"
|
||||
test "${SOURCE_ATTEMPT}" = "${EXPECTED_SOURCE_ATTEMPT}"
|
||||
echo "commit-sha=${MONITOR_SHA}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Require fresh dry-run evidence reference
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
run: |
|
||||
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
|
||||
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
|
||||
- name: Download private dry-run evidence
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Verify dry-run artifact before cloud authentication
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
run: |
|
||||
EVIDENCE_COMMIT_SHA="${GITHUB_SHA}"
|
||||
if test "${EVIDENCE_MODE:-single}" = resume; then
|
||||
EVIDENCE_COMMIT_SHA="${{ steps.resume-provenance.outputs.commit-sha }}"
|
||||
fi
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${EVIDENCE_COMMIT_SHA}" \
|
||||
--mode dry-run
|
||||
|
||||
- name: Reject previously consumed dry-run evidence
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
COUNT="$(gh api \
|
||||
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
||||
--jq '.total_count')"
|
||||
test "${COUNT}" = "0"
|
||||
|
||||
- name: Download this workflow's wave authority
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-wave-authority
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ github.run_id }}
|
||||
|
||||
- name: Download the failed wave authority for resume
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-wave-authority
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.source-wave-run-id }}
|
||||
|
||||
- name: Require wave evidence consumed by this workflow
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${GITHUB_RUN_ID}"
|
||||
|
||||
- name: Require wave evidence consumed by the failed source workflow
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${SOURCE_WAVE_RUN_ID}"
|
||||
|
||||
- id: deploy-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
release: 'false'
|
||||
|
||||
- name: Require exact mutation confirmation
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if test "${EVIDENCE_MODE:-single}" = resume; then
|
||||
test "${CONFIRMATION}" = "RESUME_SELECTED_CELL_TO_1000 ${TARGET_CELL_ID}"
|
||||
elif test "${DEPLOY_MODE}" = apply; then
|
||||
test "${CONFIRMATION}" = "RAISE_SELECTED_CELL_TO_1000"
|
||||
else
|
||||
test "${CONFIRMATION}" = "ROLL_BACK_SELECTED_CELL_TO_600 ${TARGET_CELL_ID}"
|
||||
fi
|
||||
|
||||
- name: Initialize the exact production backend
|
||||
run: node dev/scripts/infra.mjs init --env production
|
||||
|
||||
- name: Build the exact selected-cell configuration
|
||||
shell: bash
|
||||
run: |
|
||||
if test "${DEPLOY_MODE}" = rollback; then
|
||||
TARGET_HARD_CAP=600
|
||||
else
|
||||
TARGET_HARD_CAP=1000
|
||||
fi
|
||||
TARGET_UNOBSERVED_BOUND=60
|
||||
TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"
|
||||
[[ "${TARGET_HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]]
|
||||
CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev"
|
||||
CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
|
||||
OVERRIDE_CELLS_JSON="$(jq -ce \
|
||||
--arg cell "${TARGET_CELL_ID}" \
|
||||
--argjson cap "${TARGET_HARD_CAP}" \
|
||||
--argjson bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
'.[$cell].connection_hard_cap = $cap |
|
||||
.[$cell].connection_unobserved_bound = $bound' \
|
||||
<<< "${CELLS_JSON}")"
|
||||
jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \
|
||||
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-capacity.tfvars.json"
|
||||
BASE_CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
<<< 'local.relay_director_cells_json' | jq -er '.')"
|
||||
IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image"
|
||||
ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone"
|
||||
DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
|
||||
<<< "${IMAGE_EXPRESSION}" | jq -r '.')"
|
||||
TARGET_ZONE="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
|
||||
<<< "${ZONE_EXPRESSION}" | jq -r '.')"
|
||||
MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
|
||||
| jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
|
||||
jq -e --arg cell "${TARGET_CELL_ID}" \
|
||||
'any(.[]; .id == $cell and .connectionHardCap == 1000 and
|
||||
.connectionUnobservedBound == 60)' \
|
||||
<<< "${BASE_CELLS_JSON}" >/dev/null
|
||||
[[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]]
|
||||
DESIRED_IMAGE_DIGEST="${DESIRED_IMAGE##*@}"
|
||||
[[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]]
|
||||
test "${MIG_NAME}" = "orca-cloud-relay-gce-${TARGET_HOSTNAME}"
|
||||
{
|
||||
echo "CELL_ORIGIN=${CELL_ORIGIN}"
|
||||
echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}"
|
||||
echo "TARGET_HARD_CAP=${TARGET_HARD_CAP}"
|
||||
echo "TARGET_UNOBSERVED_BOUND=${TARGET_UNOBSERVED_BOUND}"
|
||||
echo "DESIRED_IMAGE=${DESIRED_IMAGE}"
|
||||
echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}"
|
||||
echo "TARGET_ZONE=${TARGET_ZONE}"
|
||||
echo "MIG_NAME=${MIG_NAME}"
|
||||
echo "BASE_CELLS_JSON=${BASE_CELLS_JSON}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Require the exact compatible production image and topology
|
||||
shell: bash
|
||||
run: |
|
||||
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
ACTIVE_REVISION="$(jq -r \
|
||||
'[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \
|
||||
<<< "${SERVICE_JSON}")"
|
||||
test -n "${ACTIVE_REVISION}"
|
||||
ACTIVE_REVISION_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
ACTIVE_IMAGE="$(jq -er '.spec.containers[0].image' <<< "${ACTIVE_REVISION_JSON}")"
|
||||
ACTIVE_IMAGE_DIGEST="${ACTIVE_IMAGE##*@}"
|
||||
if test "${ACTIVE_IMAGE}" != "${DESIRED_IMAGE}"; then
|
||||
test "${ACTIVE_IMAGE_DIGEST}" = "${COMPATIBLE_DIRECTOR_IMAGE_DIGEST}"
|
||||
test "${DESIRED_IMAGE_DIGEST}" = "${COMPATIBLE_CELL_IMAGE_DIGEST}"
|
||||
fi
|
||||
CURRENT_CELLS_JSON="$(jq -cer '[.spec.containers[0].env[]? |
|
||||
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
|
||||
if length == 1 then .[0] | fromjson else error("missing director topology") end' \
|
||||
<<< "${ACTIVE_REVISION_JSON}")"
|
||||
CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON="$(
|
||||
node dev/scripts/read-relay-production-capacity-identity.mjs \
|
||||
<<< "${ACTIVE_REVISION_JSON}"
|
||||
)"
|
||||
CLASSIFICATION="$(jq -nc \
|
||||
--argjson baseCells "${BASE_CELLS_JSON}" \
|
||||
--argjson currentCells "${CURRENT_CELLS_JSON}" \
|
||||
--arg capacityCellIds "${CAPACITY_CELL_IDS}" \
|
||||
--arg targetCellId "${TARGET_CELL_ID}" \
|
||||
--argjson targetHardCap "${TARGET_HARD_CAP}" \
|
||||
--argjson currentCapacityServiceAccount \
|
||||
"${CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON}" \
|
||||
'{baseCells:$baseCells, currentCells:$currentCells,
|
||||
capacityCellIds:($capacityCellIds | split(",")),
|
||||
targetCellId:$targetCellId, targetHardCap:$targetHardCap,
|
||||
currentCapacityServiceAccount:$currentCapacityServiceAccount}' \
|
||||
| node dev/scripts/classify-relay-production-capacity-director.mjs \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
|
||||
TOPOLOGY_PHASE="$(jq -er '.topologyPhase' <<< "${CLASSIFICATION}")"
|
||||
DESIRED_CELLS_JSON="$(jq -cer '.desiredCells' <<< "${CLASSIFICATION}")"
|
||||
DIRECTOR_READY="$(jq -er \
|
||||
'if (.directorReady | type) == "boolean" then
|
||||
(.directorReady | tostring)
|
||||
else error("invalid directorReady classification") end' \
|
||||
<<< "${CLASSIFICATION}")"
|
||||
{
|
||||
echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}"
|
||||
echo "TOPOLOGY_PHASE=${TOPOLOGY_PHASE}"
|
||||
echo "DIRECTOR_READY=${DIRECTOR_READY}"
|
||||
echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Require the exact wave predecessor topology
|
||||
if: ${{ inputs.mode == 'apply' && (inputs.evidence-mode == 'continuation' || inputs.evidence-mode == 'resume') }}
|
||||
run: test "${TOPOLOGY_PHASE}" = predecessor
|
||||
|
||||
- name: Verify fresh dry-run evidence against the live selector
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run \
|
||||
--mutation-mode capacity-transition \
|
||||
--source-cell-id "${TARGET_CELL_ID}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}"
|
||||
|
||||
- name: Recheck every live safety signal
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
|
||||
|
||||
- name: Recheck exact wave state and every live safety signal
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/relay-production-capacity-wave.mjs build-preflight \
|
||||
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
|
||||
--wave-cell-ids "${WAVE_CELL_IDS}" \
|
||||
--wave-index "${WAVE_INDEX}" \
|
||||
--target-cell-id "${TARGET_CELL_ID}" \
|
||||
--output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json"
|
||||
RETRY_ARGS=()
|
||||
if test "${WAVE_INDEX}" != 0; then RETRY_ARGS=(--retry-freshness); fi
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \
|
||||
"${RETRY_ARGS[@]}"
|
||||
|
||||
- name: Recheck exact isolated resume state and every live safety signal
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/relay-production-capacity-wave.mjs build-resume-preflight \
|
||||
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
|
||||
--wave-cell-ids "${WAVE_CELL_IDS}" \
|
||||
--target-cell-id "${TARGET_CELL_ID}" \
|
||||
--output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json"
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \
|
||||
--retry-freshness
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity allowed \
|
||||
--runtime required \
|
||||
--expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}"
|
||||
|
||||
- name: Consume the single-use dry-run evidence
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
|
||||
printf '%s\n' "${GITHUB_RUN_ID}" \
|
||||
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
|
||||
|
||||
- name: Publish the consumed-evidence marker
|
||||
if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Verify current selected-cell capacity
|
||||
if: ${{ inputs.mode == 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
CURRENT_CAP="${TARGET_HARD_CAP}"
|
||||
CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
|
||||
if test "${TOPOLOGY_PHASE}" = predecessor; then
|
||||
CURRENT_CAP=600
|
||||
CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${CURRENT_CAP}" \
|
||||
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed \
|
||||
--expected-image-digests "${CURRENT_IMAGE_DIGEST}"
|
||||
|
||||
- name: Arm fail-closed mutation cleanup
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
run: echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Reversibly isolate only the selected cell
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
test "${MUTATION_STARTED:-false}" = true || exit 0
|
||||
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode isolate
|
||||
|
||||
- name: Drain the selected cell or prove an offline rollback
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
if node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode drain; then
|
||||
echo "OFFLINE_ROLLBACK=false" >> "${GITHUB_ENV}"
|
||||
elif test "${DEPLOY_MODE}" = rollback; then
|
||||
echo "OFFLINE_ROLLBACK=true" >> "${GITHUB_ENV}"
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- id: restart-auth-one
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: restart-gate-one
|
||||
name: Require restart-safe selected-cell activity
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-one.outputs.id_token }}
|
||||
run: |
|
||||
if test "${OFFLINE_ROLLBACK:-false}" = true; then
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--heartbeat stale \
|
||||
--admission migration-only \
|
||||
--draining either \
|
||||
--activity restart-safe \
|
||||
--runtime unavailable
|
||||
echo "settled=true" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
CURRENT_CAP=600
|
||||
if test "${TOPOLOGY_PHASE}" = desired; then
|
||||
CURRENT_CAP="${TARGET_HARD_CAP}"
|
||||
elif test "${TARGET_HARD_CAP}" = 600; then
|
||||
CURRENT_CAP=1000
|
||||
fi
|
||||
GATE_LOG="${RUNNER_TEMP}/relay-capacity-restart-gate-one.log"
|
||||
set +e
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${CURRENT_CAP}" \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--runtime required \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
|
||||
--timeout-ms 450000 \
|
||||
--expected-image-digests \
|
||||
"${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
2> "${GATE_LOG}"
|
||||
GATE_EXIT=$?
|
||||
set -e
|
||||
cat "${GATE_LOG}" >&2
|
||||
if test "${GATE_EXIT}" = 0; then
|
||||
echo "settled=true" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
if test "$(wc -l < "${GATE_LOG}" | tr -d ' ')" = 1 &&
|
||||
grep -Eq '^capacity transition verification timed out: \{.*\}$' "${GATE_LOG}"; then
|
||||
echo "settled=false" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
exit "${GATE_EXIT}"
|
||||
|
||||
- id: restart-auth-two
|
||||
if: ${{ steps.restart-gate-one.outputs.settled == 'false' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Require extended restart-safe selected-cell activity
|
||||
if: ${{ steps.restart-gate-one.outputs.settled == 'false' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-two.outputs.id_token }}
|
||||
run: |
|
||||
CURRENT_CAP=600
|
||||
if test "${TOPOLOGY_PHASE}" = desired; then
|
||||
CURRENT_CAP="${TARGET_HARD_CAP}"
|
||||
elif test "${TARGET_HARD_CAP}" = 600; then
|
||||
CURRENT_CAP=1000
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${CURRENT_CAP}" \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--runtime required \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
|
||||
--timeout-ms 450000 \
|
||||
--expected-image-digests \
|
||||
"${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
|
||||
|
||||
- name: Deploy only the reviewed director topology
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
run: |
|
||||
if test "${DIRECTOR_READY}" = true; then exit 0; fi
|
||||
RELEASE_ID="capacity-${TARGET_HOSTNAME}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
|
||||
node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service "${DIRECTOR_SERVICE_NAME}" \
|
||||
--image "${ACTIVE_IMAGE}" \
|
||||
--role director \
|
||||
--max-instances 5 \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--capacity-cell-id "${TARGET_CELL_ID}" \
|
||||
--director-cells-json "${DESIRED_CELLS_JSON}" \
|
||||
--min-instances 5 \
|
||||
--prune-revisions false \
|
||||
--release-id "${RELEASE_ID}"
|
||||
|
||||
- id: director-transition-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Require fail-closed director transition
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.director-transition-auth.outputs.id_token }}
|
||||
run: |
|
||||
if test "${OFFLINE_ROLLBACK:-false}" = true; then
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--heartbeat stale \
|
||||
--admission migration-only \
|
||||
--draining either \
|
||||
--activity restart-safe \
|
||||
--runtime unavailable
|
||||
exit 0
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${TARGET_HARD_CAP}" \
|
||||
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--runtime required \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
|
||||
--timeout-ms 480000 \
|
||||
--expected-image-digests \
|
||||
"${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
|
||||
|
||||
- id: capacity-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Plan and apply only the empty selected cell
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
shell: bash
|
||||
run: |
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/production.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
|
||||
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
-out="${RUNNER_TEMP}/relay-capacity-cell.tfplan"
|
||||
PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-capacity-cell.tfplan" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode bootstrap-cell \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${TARGET_HARD_CAP}" \
|
||||
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
--image "${DESIRED_IMAGE}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
|
||||
echo "${PLAN_RESULT}"
|
||||
PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")"
|
||||
[[ "${PLAN_CHANGES}" =~ ^(0|1|2)$ ]]
|
||||
if test "${PLAN_CHANGES}" != 0; then
|
||||
terraform -chdir=infra/terraform apply \
|
||||
-auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan"
|
||||
else
|
||||
INSTANCE="$(gcloud compute instance-groups managed list-instances \
|
||||
"${MIG_NAME}" --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
|
||||
--format=json | jq -er 'if length == 1 and
|
||||
.[0].instanceStatus == "RUNNING" and .[0].currentAction == "NONE"
|
||||
then .[0].instance | split("/") | last
|
||||
else error("selected cell is not one stable running instance") end')"
|
||||
gcloud compute instance-groups managed recreate-instances \
|
||||
"${MIG_NAME}" --instances "${INSTANCE}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --quiet
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until \
|
||||
"${MIG_NAME}" --stable --project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" --timeout 900
|
||||
|
||||
- id: capacity-transition-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Verify fresh exact selected-cell heartbeat before admission
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${TARGET_HARD_CAP}" \
|
||||
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining forbidden \
|
||||
--activity allowed \
|
||||
--expected-image-digests "${DESIRED_IMAGE_DIGEST}"
|
||||
|
||||
- name: Restore only the selected cell to general admission
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode activate
|
||||
|
||||
- name: Verify the live general selected cell
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${TARGET_HARD_CAP}" \
|
||||
--unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed \
|
||||
--expected-image-digests "${DESIRED_IMAGE_DIGEST}"
|
||||
|
||||
- id: cleanup-auth
|
||||
if: ${{ failure() && inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Keep the selected cell isolated after a failed mutation
|
||||
if: ${{ failure() && inputs.mode != 'verify' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }}
|
||||
run: |
|
||||
test "${MUTATION_STARTED:-false}" = true || exit 0
|
||||
CLEANUP_STATUS=0
|
||||
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode isolate || CLEANUP_STATUS=$?
|
||||
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode drain || CLEANUP_STATUS=$?
|
||||
exit "${CLEANUP_STATUS}"
|
||||
@@ -1,356 +0,0 @@
|
||||
name: Deploy Relay Production Capacity
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Verify, change one cell, or raise a sequential wave
|
||||
required: true
|
||||
default: verify
|
||||
type: choice
|
||||
options:
|
||||
- verify
|
||||
- apply
|
||||
- rollback
|
||||
- wave-apply
|
||||
- wave-resume
|
||||
target-cell-id:
|
||||
description: Exact serving cell for verify, apply, or rollback
|
||||
required: true
|
||||
default: production-gce-c26
|
||||
type: choice
|
||||
options:
|
||||
- production-gce-c7
|
||||
- production-gce-c8
|
||||
- production-gce-c9
|
||||
- production-gce-c10
|
||||
- production-gce-c13
|
||||
- production-gce-c14
|
||||
- production-gce-c15
|
||||
- production-gce-c16
|
||||
- production-gce-c19
|
||||
- production-gce-c20
|
||||
- production-gce-c21
|
||||
- production-gce-c22
|
||||
- production-gce-c23
|
||||
- production-gce-c24
|
||||
- production-gce-c25
|
||||
- production-gce-c26
|
||||
wave-cell-ids:
|
||||
description: Ordered comma-separated wave of two to four serving cells
|
||||
required: false
|
||||
default: none
|
||||
type: string
|
||||
confirmation:
|
||||
description: Enter the exact single-cell or wave confirmation
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-id:
|
||||
description: Successful fresh dry-run monitor workflow run ID for apply
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-attempt:
|
||||
description: Exact dry-run monitor workflow attempt for apply
|
||||
required: false
|
||||
type: string
|
||||
source-wave-run-id:
|
||||
description: Failed wave run that isolated the resume target
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
single_cell:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode != 'wave-apply' && inputs.mode != 'wave-resume') }}
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: ${{ inputs.mode }}
|
||||
target-cell-id: ${{ inputs.target-cell-id }}
|
||||
confirmation: ${{ inputs.confirmation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: single
|
||||
wave-cell-ids: none
|
||||
wave-index: '0'
|
||||
source-wave-run-id: none
|
||||
secrets: inherit
|
||||
|
||||
resume_cell:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-resume' && github.ref == 'refs/heads/main') }}
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: apply
|
||||
target-cell-id: ${{ inputs.target-cell-id }}
|
||||
confirmation: ${{ inputs.confirmation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: resume
|
||||
wave-cell-ids: ${{ inputs.wave-cell-ids }}
|
||||
wave-index: resume
|
||||
source-wave-run-id: ${{ inputs.source-wave-run-id }}
|
||||
secrets: inherit
|
||||
|
||||
wave_gate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-apply' && github.ref == 'refs/heads/main') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 30
|
||||
environment: production
|
||||
outputs:
|
||||
cells: ${{ steps.wave.outputs.cells }}
|
||||
env:
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
||||
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
|
||||
PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d
|
||||
COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f
|
||||
WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }}
|
||||
steps:
|
||||
- name: Require production workflow configuration
|
||||
working-directory: .
|
||||
env:
|
||||
DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${DEPLOY_SERVICE_ACCOUNT}"
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- id: wave
|
||||
name: Validate the exact wave request
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
CELLS="$(node dev/scripts/relay-production-capacity-wave.mjs validate \
|
||||
--wave-cell-ids "${WAVE_CELL_IDS}" \
|
||||
--confirmation "${CONFIRMATION}")"
|
||||
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Require fresh dry-run evidence reference
|
||||
run: |
|
||||
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
|
||||
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
|
||||
- name: Download private dry-run evidence
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ github.workspace }}/relay-monitor-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
- name: Verify dry-run artifact before cloud authentication
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
|
||||
--directory "${OUTPUT_DIRECTORY}" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run
|
||||
|
||||
- name: Reject previously consumed dry-run evidence
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
COUNT="$(gh api \
|
||||
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
||||
--jq '.total_count')"
|
||||
test "${COUNT}" = "0"
|
||||
|
||||
- id: deploy-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
release: 'false'
|
||||
|
||||
- name: Verify wave evidence against the live selector
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
FIRST_CELL="$(jq -er '.[0]' <<< '${{ steps.wave.outputs.cells }}')"
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
|
||||
--directory "${OUTPUT_DIRECTORY}" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run \
|
||||
--mutation-mode capacity-transition \
|
||||
--source-cell-id "${FIRST_CELL}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}"
|
||||
|
||||
- name: Recheck every live safety signal
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json"
|
||||
|
||||
- name: Require exact 600/60 predecessor wave cells
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
while read -r CELL_ID; do
|
||||
HOSTNAME="${CELL_ID#production-gce-}"
|
||||
[[ "${HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]]
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "https://${HOSTNAME}.relay.onorca.dev" \
|
||||
--cell-id "${CELL_ID}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed \
|
||||
--expected-image-digests \
|
||||
"${PREDECESSOR_IMAGE_DIGEST},${COMPATIBLE_CELL_IMAGE_DIGEST}"
|
||||
done < <(jq -r '.[]' <<< '${{ steps.wave.outputs.cells }}')
|
||||
|
||||
- name: Consume the single-use dry-run evidence
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
|
||||
printf '%s\n' "${GITHUB_RUN_ID}" \
|
||||
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
|
||||
|
||||
- name: Publish the consumed-evidence marker
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
wave_cell_1:
|
||||
needs: wave_gate
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: apply
|
||||
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[0] }}
|
||||
confirmation: RAISE_SELECTED_CELL_TO_1000
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: continuation
|
||||
wave-cell-ids: ${{ inputs.wave-cell-ids }}
|
||||
wave-index: '0'
|
||||
source-wave-run-id: none
|
||||
secrets: inherit
|
||||
|
||||
wave_cell_2:
|
||||
needs: [wave_gate, wave_cell_1]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: apply
|
||||
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[1] }}
|
||||
confirmation: RAISE_SELECTED_CELL_TO_1000
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: continuation
|
||||
wave-cell-ids: ${{ inputs.wave-cell-ids }}
|
||||
wave-index: '1'
|
||||
source-wave-run-id: none
|
||||
secrets: inherit
|
||||
|
||||
wave_cell_3:
|
||||
if: ${{ needs.wave_cell_2.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[2] != null }}
|
||||
needs: [wave_gate, wave_cell_2]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: apply
|
||||
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[2] }}
|
||||
confirmation: RAISE_SELECTED_CELL_TO_1000
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: continuation
|
||||
wave-cell-ids: ${{ inputs.wave-cell-ids }}
|
||||
wave-index: '2'
|
||||
source-wave-run-id: none
|
||||
secrets: inherit
|
||||
|
||||
wave_cell_4:
|
||||
if: ${{ needs.wave_cell_3.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[3] != null }}
|
||||
needs: [wave_gate, wave_cell_3]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
|
||||
with:
|
||||
mode: apply
|
||||
target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[3] }}
|
||||
confirmation: RAISE_SELECTED_CELL_TO_1000
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
evidence-mode: continuation
|
||||
wave-cell-ids: ${{ inputs.wave-cell-ids }}
|
||||
wave-index: '3'
|
||||
source-wave-run-id: none
|
||||
secrets: inherit
|
||||
|
||||
# Every wave job re-enters the run's lease with release: 'false'; only this job frees it.
|
||||
release_lease:
|
||||
if: always()
|
||||
needs:
|
||||
- single_cell
|
||||
- resume_cell
|
||||
- wave_gate
|
||||
- wave_cell_1
|
||||
- wave_cell_2
|
||||
- wave_cell_3
|
||||
- wave_cell_4
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 10
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
release: 'true'
|
||||
@@ -1,311 +0,0 @@
|
||||
name: Deploy Relay Production Director
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
image-digest:
|
||||
description: 'Immutable relay image digest (sha256: plus 64 lowercase hex characters)'
|
||||
required: true
|
||||
type: string
|
||||
regional-placement-mode:
|
||||
description: Preserve the live switch, explicitly enable Asia preference, or force US-first
|
||||
required: true
|
||||
default: preserve
|
||||
type: choice
|
||||
options: [preserve, enable, disable]
|
||||
region-correction-cohort-percent:
|
||||
description: 'Preserve the measured-correction cohort, or set an integer 0–100; durable rehome stays disabled'
|
||||
required: true
|
||||
default: preserve
|
||||
type: string
|
||||
reserve-placement:
|
||||
description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve
|
||||
required: true
|
||||
default: preserve
|
||||
type: choice
|
||||
options: [preserve, 'off', dry-run, 'on']
|
||||
shadow-seat-feed-cells:
|
||||
description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused
|
||||
required: true
|
||||
default: preserve
|
||||
type: string
|
||||
prune-incompatible-revisions:
|
||||
description: Retain only the newly verified serving and rollback revisions
|
||||
required: true
|
||||
default: false
|
||||
type: boolean
|
||||
confirmation:
|
||||
description: Enter the exact confirmation required by a destructive option
|
||||
required: false
|
||||
type: string
|
||||
expected-rehome-generation:
|
||||
description: Exact durable regional-rehome generation; it must remain disabled
|
||||
required: true
|
||||
type: string
|
||||
bootstrap-runtime-identity:
|
||||
description: One-time move from the stamped-cell identity to the director identity
|
||||
required: true
|
||||
default: false
|
||||
type: boolean
|
||||
predecessor-image-digest:
|
||||
description: Exact immutable serving predecessor digest for the one-time identity bootstrap
|
||||
required: true
|
||||
type: string
|
||||
shape-baseline-revision:
|
||||
description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# Director updates and candidate operations both mutate production relay control state.
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
DIRECTOR_SERVICE_NAME: orca-cloud-relay
|
||||
IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay
|
||||
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }}
|
||||
REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }}
|
||||
RESERVE_PLACEMENT: ${{ inputs.reserve-placement }}
|
||||
SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }}
|
||||
PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }}
|
||||
# Floor the served revision must keep, matching relay_min_instances in
|
||||
# environments/production.tfvars. This gate only fails a bad deploy; Terraform
|
||||
# still owns the value, and the candidate inherits it from the serving revision.
|
||||
DIRECTOR_MIN_INSTANCES: 5
|
||||
DIRECTOR_MAX_INSTANCES: 5
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
PREDECESSOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_RUNTIME_SERVICE_ACCOUNT }}
|
||||
REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain
|
||||
EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
||||
BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }}
|
||||
PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }}
|
||||
SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Resolve immutable production image
|
||||
shell: bash
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if [[ ! "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
|
||||
echo "image-digest must be an immutable lowercase sha256 digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
if test "${REGION_CORRECTION_COHORT_PERCENT}" != preserve; then
|
||||
[[ "${REGION_CORRECTION_COHORT_PERCENT}" =~ ^([0-9]|[1-9][0-9]|100)$ ]]
|
||||
fi
|
||||
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
|
||||
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
|
||||
[[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]]
|
||||
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]]
|
||||
[[ "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]]
|
||||
[[ "${BOOTSTRAP_RUNTIME_IDENTITY}" =~ ^(true|false)$ ]]
|
||||
if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then
|
||||
[[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false
|
||||
test "${REGIONAL_PLACEMENT_MODE}" = preserve
|
||||
test -z "${SHAPE_BASELINE_REVISION}"
|
||||
test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY
|
||||
elif test -n "${SHAPE_BASELINE_REVISION}"; then
|
||||
[[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-[0-9]{5}-[a-z0-9]{3}$ ]]
|
||||
test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false
|
||||
test "${REGIONAL_PLACEMENT_MODE}" = preserve
|
||||
test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE
|
||||
elif test "${PRUNE_INCOMPATIBLE_REVISIONS}" = true; then
|
||||
test "${REGIONAL_PLACEMENT_MODE}" = preserve
|
||||
test "${CONFIRMATION}" = PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS
|
||||
elif test "${REGIONAL_PLACEMENT_MODE}" = disable; then
|
||||
test "${CONFIRMATION}" = FORCE_RELAY_US_FIRST
|
||||
else
|
||||
test -z "${CONFIRMATION}"
|
||||
fi
|
||||
echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
|
||||
|
||||
# Why: the deploy INHERITS the serving revision's floor, so when that revision has
|
||||
# already lost it the candidate inherits zero, the in-script gate compares zero against
|
||||
# zero and passes, and the post-deploy check below only notices after traffic moved.
|
||||
# The documented rollback target is created at minimum instances zero, so promoting it
|
||||
# arms exactly that. Refuse to inherit a degraded floor rather than latch it.
|
||||
- name: Require a healthy serving floor before deploying
|
||||
shell: bash
|
||||
run: |
|
||||
SERVING="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
|
||||
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test -n "${SERVING}"
|
||||
FLOOR="$(gcloud run revisions describe "${SERVING}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
|
||||
if [[ "${FLOOR:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then
|
||||
echo "serving revision ${SERVING} holds ${FLOOR:-0} minimum instances," \
|
||||
"below ${DIRECTOR_MIN_INSTANCES}; deploying would inherit and latch it." >&2
|
||||
echo "Restore the floor first: gcloud run services update ${DIRECTOR_SERVICE_NAME}" \
|
||||
"--min-instances=${DIRECTOR_MIN_INSTANCES}" >&2
|
||||
exit 1
|
||||
fi
|
||||
CEILING="$(gcloud run revisions describe "${SERVING}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
|
||||
test "${CEILING}" = "${DIRECTOR_MAX_INSTANCES}"
|
||||
echo "serving revision ${SERVING} holds ${FLOOR} minimum instances"
|
||||
echo "SERVING_REVISION=${SERVING}" >> "${GITHUB_ENV}"
|
||||
|
||||
# Why: no --min-instances here. The candidate inherits the Terraform-owned
|
||||
# scaling, and this step ends with 100% traffic on it. Pinning 1 rebuilt the
|
||||
# per-instance admission shortage that took placement failures to ~70%.
|
||||
- name: Deploy director blue/green
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
served_version="$(gcloud run revisions describe "${SERVING_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '[.spec.containers[0].env[]? |
|
||||
select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
|
||||
(.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
|
||||
(.version // .key // empty)] |
|
||||
if length == 1 then .[0] else empty end')"
|
||||
if [[ "${served_version}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
current_version="${served_version}"
|
||||
else
|
||||
test "${REGIONAL_PLACEMENT_MODE}" = preserve
|
||||
current_version="$(gcloud secrets versions describe latest \
|
||||
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \
|
||||
--format='value(name)' | awk -F/ '{print $NF}')"
|
||||
[[ "${current_version}" =~ ^[1-9][0-9]*$ ]]
|
||||
fi
|
||||
current="$(gcloud secrets versions access "${current_version}" \
|
||||
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")"
|
||||
[[ "${current}" =~ ^(true|false)$ ]]
|
||||
case "${REGIONAL_PLACEMENT_MODE}" in
|
||||
preserve) desired="${current}" ;;
|
||||
enable) desired=true ;;
|
||||
disable) desired=false ;;
|
||||
*) echo "regional-placement-mode is invalid" >&2; exit 1 ;;
|
||||
esac
|
||||
if test "${current}" != "${desired}"; then
|
||||
target_version="$(printf '%s' "${desired}" | gcloud secrets versions add \
|
||||
"${REGIONAL_PLACEMENT_SECRET}" --project "${GCP_PROJECT_ID}" --data-file=- \
|
||||
--format='value(name)' --quiet | awk -F/ '{print $NF}')"
|
||||
else
|
||||
target_version="${current_version}"
|
||||
fi
|
||||
[[ "${target_version}" =~ ^[1-9][0-9]*$ ]]
|
||||
RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
|
||||
shape_args=()
|
||||
if test -n "${SHAPE_BASELINE_REVISION}"; then
|
||||
shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}")
|
||||
fi
|
||||
node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service "${DIRECTOR_SERVICE_NAME}" \
|
||||
--image "${IMAGE}" \
|
||||
--role director \
|
||||
"${shape_args[@]}" \
|
||||
--runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--bootstrap-runtime-identity "${BOOTSTRAP_RUNTIME_IDENTITY}" \
|
||||
--predecessor-image-digest "${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience "${REHOME_AUDIENCE}" \
|
||||
--rehome-control-origin https://relay.onorca.dev \
|
||||
--admin-audience https://relay.onorca.dev/v1/admin/drain \
|
||||
--expected-rehome-generation "${EXPECTED_REHOME_GENERATION}" \
|
||||
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
|
||||
--prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \
|
||||
--release-id "${RELEASE_ID}" \
|
||||
--regional-placement-secret-version "${target_version}" \
|
||||
--region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}" \
|
||||
--reserve-placement "${RESERVE_PLACEMENT}" \
|
||||
--shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}"
|
||||
echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}"
|
||||
echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify served revision and native health
|
||||
shell: bash
|
||||
run: |
|
||||
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format=json)"
|
||||
REVISION="$(jq -r '[.status.traffic[] | select((.percent // 0) > 0)] | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' <<< "${SERVICE_JSON}")"
|
||||
test -n "${REVISION}"
|
||||
SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format='value(spec.containers[0].image)')"
|
||||
test "${SERVED_IMAGE}" = "${IMAGE}"
|
||||
SERVED_REGIONAL_PLACEMENT_SECRET="$(gcloud run revisions describe "${REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -cer '[.spec.containers[0].env[] |
|
||||
select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
|
||||
(.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
|
||||
{secret: (.secret // .name), version: (.version // .key)}] |
|
||||
if length == 1 then .[0] else error("regional placement secret missing") end')"
|
||||
test "$(jq -r '.secret' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \
|
||||
"${REGIONAL_PLACEMENT_SECRET}"
|
||||
test "$(jq -r '.version' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \
|
||||
"${REGIONAL_PLACEMENT_VERSION}"
|
||||
test "$(gcloud secrets versions access "${REGIONAL_PLACEMENT_VERSION}" --project "${GCP_PROJECT_ID}" \
|
||||
--secret "${REGIONAL_PLACEMENT_SECRET}")" = "${REGIONAL_PLACEMENT_ENABLED}"
|
||||
# Why: a served revision with no warm-instance floor still passes health and digest
|
||||
# checks while quietly shrinking per-instance admission capacity.
|
||||
SERVED_MIN_INSTANCES="$(gcloud run revisions describe "${REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
|
||||
if [[ "${SERVED_MIN_INSTANCES:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then
|
||||
echo "served revision ${REVISION} holds ${SERVED_MIN_INSTANCES:-0} minimum instances, expected at least ${DIRECTOR_MIN_INSTANCES}" >&2
|
||||
exit 1
|
||||
fi
|
||||
SERVED_MAX_INSTANCES="$(gcloud run revisions describe "${REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
|
||||
test "${SERVED_MAX_INSTANCES}" = "${DIRECTOR_MAX_INSTANCES}"
|
||||
SERVICE_URL="$(jq -r '.status.url' <<< "${SERVICE_JSON}")"
|
||||
node dev/scripts/smoke-relay.mjs "${SERVICE_URL}"
|
||||
@@ -1,500 +0,0 @@
|
||||
name: Deploy Relay Production Multi-Target
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source-cell-id:
|
||||
description: Existing Terraform source cell ID
|
||||
required: true
|
||||
type: string
|
||||
target-cell-ids:
|
||||
description: Comma-separated distinct Terraform target cell IDs
|
||||
required: true
|
||||
type: string
|
||||
general-cell-ids:
|
||||
description: Comma-separated proven cells that remain eligible for ordinary placement
|
||||
required: false
|
||||
type: string
|
||||
unobserved-connection-bound:
|
||||
description: Exact worst-case unobserved connection bound proven by the passing load gate
|
||||
required: false
|
||||
type: string
|
||||
failed-target-cell-id:
|
||||
description: Registered failed target to fence and supersede
|
||||
required: false
|
||||
type: string
|
||||
replacement-target-cell-id:
|
||||
description: Healthy replacement for registered failed target
|
||||
required: false
|
||||
type: string
|
||||
mode:
|
||||
description: Preflight/audit are read-only; other modes mutate production
|
||||
required: true
|
||||
default: preflight
|
||||
type: choice
|
||||
options:
|
||||
- audit
|
||||
- preflight
|
||||
- cutover-admission
|
||||
- add-migration-cells
|
||||
- promote-general-cell
|
||||
- retire-migration-cell
|
||||
- execute
|
||||
- recover-forward
|
||||
- fence-source
|
||||
- supersede-target
|
||||
confirmation:
|
||||
description: Enter CUTOVER_SELECTOR, ADD_MIGRATION_CELLS, PROMOTE_GENERAL_CELL, RETIRE_MIGRATION_CELL, EVACUATE_MULTI, RECOVER_FORWARD, or FENCE_SOURCE
|
||||
required: false
|
||||
type: string
|
||||
selector-attempt-id:
|
||||
description: Exact durable selector attempt ID for admission mutations
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-id:
|
||||
description: Successful fresh dry-run monitor workflow run ID
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-attempt:
|
||||
description: Exact dry-run monitor workflow attempt
|
||||
required: false
|
||||
type: string
|
||||
broker-operation-id:
|
||||
description: Stable durable broker operation ID for target supersession
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-attempt-id:
|
||||
description: Exact older completed fence attempt to recover without replay
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-commit:
|
||||
description: Exact older fence commit bound to the completed attempt
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-operation:
|
||||
description: Exact DONE Compute resize operation to adopt
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-state-serial:
|
||||
description: Exact Terraform serial before the completed fence
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-plan-generation:
|
||||
description: Exact saved-plan object generation
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-state-generation:
|
||||
description: Exact current Terraform state object generation
|
||||
required: false
|
||||
type: string
|
||||
completed-fence-state-sha256:
|
||||
description: Exact current Terraform state object SHA-256
|
||||
required: false
|
||||
type: string
|
||||
expected-lease-generation:
|
||||
description: Exact live lease generation authorized for conditional takeover
|
||||
required: false
|
||||
type: string
|
||||
expected-lease-operation-id:
|
||||
description: Exact live lease operation ID authorized for takeover
|
||||
required: false
|
||||
type: string
|
||||
expected-lease-request-digest:
|
||||
description: Exact live lease request digest authorized for takeover
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: >-
|
||||
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
|
||||
github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain
|
||||
SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
|
||||
TARGET_CELL_IDS: ${{ inputs.target-cell-ids }}
|
||||
GENERAL_CELL_IDS: ${{ inputs.general-cell-ids }}
|
||||
UNOBSERVED_CONNECTION_BOUND: ${{ inputs.unobserved-connection-bound }}
|
||||
FAILED_TARGET_CELL_ID: ${{ inputs.failed-target-cell-id }}
|
||||
REPLACEMENT_TARGET_CELL_ID: ${{ inputs.replacement-target-cell-id }}
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
||||
SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }}
|
||||
BROKER_OPERATION_ID: ${{ inputs.broker-operation-id }}
|
||||
COMPLETED_FENCE_ATTEMPT_ID: ${{ inputs.completed-fence-attempt-id }}
|
||||
COMPLETED_FENCE_COMMIT: ${{ inputs.completed-fence-commit }}
|
||||
COMPLETED_FENCE_OPERATION: ${{ inputs.completed-fence-operation }}
|
||||
COMPLETED_FENCE_STATE_SERIAL: ${{ inputs.completed-fence-state-serial }}
|
||||
COMPLETED_FENCE_PLAN_GENERATION: ${{ inputs.completed-fence-plan-generation }}
|
||||
COMPLETED_FENCE_STATE_GENERATION: ${{ inputs.completed-fence-state-generation }}
|
||||
COMPLETED_FENCE_STATE_SHA256: ${{ inputs.completed-fence-state-sha256 }}
|
||||
EXPECTED_LEASE_GENERATION: ${{ inputs.expected-lease-generation }}
|
||||
EXPECTED_LEASE_OPERATION_ID: ${{ inputs.expected-lease-operation-id }}
|
||||
EXPECTED_LEASE_REQUEST_DIGEST: ${{ inputs.expected-lease-request-digest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Require private fence-broker environment
|
||||
if: >-
|
||||
${{ inputs.mode == 'fence-source' ||
|
||||
inputs.mode == 'supersede-target' }}
|
||||
env:
|
||||
FENCE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
FENCE_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }}
|
||||
FENCE_BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
|
||||
run: |
|
||||
test -n "${FENCE_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${FENCE_SERVICE_ACCOUNT}"
|
||||
test -n "${FENCE_BROKER_URI}"
|
||||
|
||||
- name: Reject direct-runner Terraform fence aborts
|
||||
if: ${{ inputs.mode == 'abort-fence-source' }}
|
||||
run: |
|
||||
echo "Terraform fence aborts require a reviewed private-broker recovery path." >&2
|
||||
exit 1
|
||||
|
||||
- name: Require fresh dry-run evidence reference
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
run: |
|
||||
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
|
||||
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
|
||||
- name: Download private dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Verify dry-run artifact before cloud authentication
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run
|
||||
|
||||
- name: Reject previously consumed dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
COUNT="$(gh api \
|
||||
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
||||
--jq '.total_count')"
|
||||
test "${COUNT}" = "0"
|
||||
|
||||
- id: google-auth
|
||||
if: ${{ inputs.mode != 'supersede-target' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
|
||||
- name: Require explicit mutation confirmation
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if [[ "${DEPLOY_MODE}" = "cutover-admission" ]]; then
|
||||
test "${CONFIRMATION}" = "CUTOVER_SELECTOR"
|
||||
elif [[ "${DEPLOY_MODE}" = "add-migration-cells" ]]; then
|
||||
test "${CONFIRMATION}" = "ADD_MIGRATION_CELLS"
|
||||
elif [[ "${DEPLOY_MODE}" = "promote-general-cell" ]]; then
|
||||
test "${CONFIRMATION}" = "PROMOTE_GENERAL_CELL"
|
||||
elif [[ "${DEPLOY_MODE}" = "retire-migration-cell" ]]; then
|
||||
test "${CONFIRMATION}" = "RETIRE_MIGRATION_CELL"
|
||||
elif [[ "${DEPLOY_MODE}" = "execute" ]]; then
|
||||
test "${CONFIRMATION}" = "EVACUATE_MULTI"
|
||||
elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then
|
||||
test "${CONFIRMATION}" = "RECOVER_FORWARD"
|
||||
elif [[ "${DEPLOY_MODE}" = "fence-source" ]]; then
|
||||
test "${CONFIRMATION}" = "FENCE_SOURCE"
|
||||
elif [[ "${DEPLOY_MODE}" = "supersede-target" ]]; then
|
||||
test "${CONFIRMATION}" = "SUPERSEDE_TARGET"
|
||||
elif [[ "${DEPLOY_MODE}" = "abort-fence-source" ]]; then
|
||||
test "${CONFIRMATION}" = "ABORT_FENCE"
|
||||
else
|
||||
test "${CONFIRMATION}" = "FENCE_SOURCE"
|
||||
fi
|
||||
|
||||
- name: Require exact source-fence broker contract
|
||||
if: ${{ inputs.mode == 'fence-source' }}
|
||||
run: |
|
||||
test "${SOURCE_CELL_ID}" = "production-gce-c3"
|
||||
test "${TARGET_CELL_IDS}" = "production-gce-c7,production-gce-c8,production-gce-c10,production-gce-c13,production-gce-c17,production-gce-c18"
|
||||
[[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
|
||||
[[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
|
||||
test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}"
|
||||
[[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]]
|
||||
else
|
||||
test -z "${EXPECTED_LEASE_OPERATION_ID}"
|
||||
test -z "${EXPECTED_LEASE_REQUEST_DIGEST}"
|
||||
fi
|
||||
|
||||
- name: Require exact broker cell contract
|
||||
if: ${{ inputs.mode == 'supersede-target' }}
|
||||
run: |
|
||||
test "${SOURCE_CELL_ID}" = "production-gce-c3"
|
||||
test "${FAILED_TARGET_CELL_ID}" = "production-gce-c12"
|
||||
test "${REPLACEMENT_TARGET_CELL_ID}" = "production-gce-c13"
|
||||
test "${TARGET_CELL_IDS}" = "production-gce-c12,production-gce-c13"
|
||||
if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then
|
||||
[[ "${COMPLETED_FENCE_ATTEMPT_ID}" =~ ^[0-9a-f-]{36}$ ]]
|
||||
[[ "${COMPLETED_FENCE_COMMIT}" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "${COMPLETED_FENCE_OPERATION}" =~ ^[A-Za-z0-9._-]{1,256}$ ]]
|
||||
[[ "${COMPLETED_FENCE_STATE_SERIAL}" =~ ^[0-9]+$ ]]
|
||||
[[ "${COMPLETED_FENCE_PLAN_GENERATION}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${COMPLETED_FENCE_STATE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${COMPLETED_FENCE_STATE_SHA256}" =~ ^[0-9a-f]{64}$ ]]
|
||||
test -n "${EXPECTED_LEASE_GENERATION}"
|
||||
fi
|
||||
if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
|
||||
[[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
|
||||
test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}"
|
||||
[[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]]
|
||||
else
|
||||
test -z "${EXPECTED_LEASE_OPERATION_ID}"
|
||||
test -z "${EXPECTED_LEASE_REQUEST_DIGEST}"
|
||||
fi
|
||||
|
||||
- name: Read reviewed Terraform topology
|
||||
if: ${{ inputs.mode != 'supersede-target' }}
|
||||
run: |
|
||||
node dev/scripts/infra.mjs init --env production
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
|
||||
DIRECTOR_MIN_INSTANCES="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars <<< 'var.relay_min_instances')"
|
||||
[[ "${DIRECTOR_MIN_INSTANCES}" =~ ^[1-9][0-9]*$ ]]
|
||||
echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
|
||||
echo "DIRECTOR_MIN_INSTANCES=${DIRECTOR_MIN_INSTANCES}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify fresh dry-run evidence against live selector
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
SCOPED_RECOVERY_ARGS=()
|
||||
if [[ ("${DEPLOY_MODE}" = "execute" ||
|
||||
"${DEPLOY_MODE}" = "recover-forward") &&
|
||||
"${SOURCE_CELL_ID}" = "production-gce-c12" ]]; then
|
||||
SCOPED_RECOVERY_ARGS=(
|
||||
--scoped-recovery-source-cell-id
|
||||
production-gce-c3
|
||||
)
|
||||
fi
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run \
|
||||
--mutation-mode "${DEPLOY_MODE}" \
|
||||
--source-cell-id "${SOURCE_CELL_ID}" \
|
||||
"${SCOPED_RECOVERY_ARGS[@]}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}"
|
||||
|
||||
- name: Recheck all live safety signals
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
|
||||
|
||||
- name: Create single-use dry-run marker
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
|
||||
printf '%s\n' "${GITHUB_RUN_ID}" \
|
||||
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
|
||||
|
||||
- name: Consume dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
- id: google-fence-broker-auth
|
||||
if: >-
|
||||
${{ inputs.mode == 'fence-source' ||
|
||||
inputs.mode == 'supersede-target' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Invoke private target-supersession broker
|
||||
if: ${{ inputs.mode == 'supersede-target' }}
|
||||
env:
|
||||
BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }}
|
||||
BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
|
||||
run: |
|
||||
[[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then
|
||||
REQUEST="$(jq -cn \
|
||||
--arg operationId "${BROKER_OPERATION_ID}" \
|
||||
--arg fenceCommit "${GITHUB_SHA}" \
|
||||
--arg attemptId "${COMPLETED_FENCE_ATTEMPT_ID}" \
|
||||
--arg completedCommit "${COMPLETED_FENCE_COMMIT}" \
|
||||
--arg gceOperation "${COMPLETED_FENCE_OPERATION}" \
|
||||
--arg stateSerial "${COMPLETED_FENCE_STATE_SERIAL}" \
|
||||
--arg planGeneration "${COMPLETED_FENCE_PLAN_GENERATION}" \
|
||||
--arg stateGeneration "${COMPLETED_FENCE_STATE_GENERATION}" \
|
||||
--arg stateSha256 "${COMPLETED_FENCE_STATE_SHA256}" \
|
||||
--arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
|
||||
--arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
|
||||
--arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
|
||||
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
|
||||
completedFenceRecovery:{attemptId:$attemptId,fenceCommit:$completedCommit,
|
||||
gceOperation:$gceOperation,terraformStateSerial:($stateSerial|tonumber),
|
||||
planObjectGeneration:$planGeneration,
|
||||
terraformStateObjectGeneration:$stateGeneration,
|
||||
terraformStateObjectSha256:$stateSha256},
|
||||
expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
|
||||
requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')"
|
||||
elif [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
|
||||
REQUEST="$(jq -cn \
|
||||
--arg operationId "${BROKER_OPERATION_ID}" \
|
||||
--arg fenceCommit "${GITHUB_SHA}" \
|
||||
--arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
|
||||
--arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
|
||||
--arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
|
||||
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
|
||||
expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
|
||||
requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')"
|
||||
else
|
||||
REQUEST="$(jq -cn \
|
||||
--arg operationId "${BROKER_OPERATION_ID}" \
|
||||
--arg fenceCommit "${GITHUB_SHA}" \
|
||||
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,confirmation:"SUPERSEDE_TARGET"}')"
|
||||
fi
|
||||
curl --fail-with-body --max-time 1790 \
|
||||
--request POST "${BROKER_URI}/v1/supersede-target" \
|
||||
--header "Authorization: Bearer ${BROKER_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "${REQUEST}"
|
||||
|
||||
- name: Invoke private source-fence broker
|
||||
if: ${{ inputs.mode == 'fence-source' }}
|
||||
env:
|
||||
BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }}
|
||||
BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
|
||||
run: |
|
||||
if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
|
||||
REQUEST="$(jq -cn \
|
||||
--arg operationId "${BROKER_OPERATION_ID}" \
|
||||
--arg fenceCommit "${GITHUB_SHA}" \
|
||||
--arg targetCellIds "${TARGET_CELL_IDS}" \
|
||||
--arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
|
||||
--arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
|
||||
--arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
|
||||
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
|
||||
targetCellIds:($targetCellIds|split(",")),
|
||||
expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
|
||||
requestDigest:$leaseRequestDigest},confirmation:"FENCE_SOURCE"}')"
|
||||
else
|
||||
REQUEST="$(jq -cn \
|
||||
--arg operationId "${BROKER_OPERATION_ID}" \
|
||||
--arg fenceCommit "${GITHUB_SHA}" \
|
||||
--arg targetCellIds "${TARGET_CELL_IDS}" \
|
||||
'{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
|
||||
targetCellIds:($targetCellIds|split(",")),confirmation:"FENCE_SOURCE"}')"
|
||||
fi
|
||||
curl --fail-with-body --max-time 1790 \
|
||||
--request POST "${BROKER_URI}/v1/fence-source" \
|
||||
--header "Authorization: Bearer ${BROKER_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "${REQUEST}"
|
||||
|
||||
- name: Preflight or run multi-target evacuation
|
||||
if: >-
|
||||
${{ inputs.mode != 'fence-source' &&
|
||||
inputs.mode != 'supersede-target' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/deploy-relay-gce-multi-target.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--admin-audience "${ADMIN_AUDIENCE}" \
|
||||
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
|
||||
--source-cell-id "${SOURCE_CELL_ID}" \
|
||||
--target-cell-ids "${TARGET_CELL_IDS}" \
|
||||
--general-cell-ids "${GENERAL_CELL_IDS}" \
|
||||
--unobserved-connection-bound "${UNOBSERVED_CONNECTION_BOUND}" \
|
||||
--director-region "${{ vars.PRODUCTION_GCP_REGION }}" \
|
||||
--director-service "orca-cloud-relay" \
|
||||
--director-min-instances "${DIRECTOR_MIN_INSTANCES}" \
|
||||
--selector-attempt-id "${SELECTOR_ATTEMPT_ID}" \
|
||||
--failed-target-cell-id "${FAILED_TARGET_CELL_ID}" \
|
||||
--replacement-target-cell-id "${REPLACEMENT_TARGET_CELL_ID}" \
|
||||
--runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--environment production \
|
||||
--fence-commit "${GITHUB_SHA}" \
|
||||
--terraform-dir infra/terraform \
|
||||
--terraform-var-file environments/production.tfvars \
|
||||
--mode "${DEPLOY_MODE}" \
|
||||
--connection-ceiling 1000 \
|
||||
--minimum-lease-remaining-ms 600000
|
||||
@@ -1,930 +0,0 @@
|
||||
name: Deploy Relay Production Same-Cap Job
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
mode: { required: true, type: string }
|
||||
target-cell-id: { required: true, type: string }
|
||||
target-image-digest: { required: true, type: string }
|
||||
rollback-image-digest: { required: true, type: string }
|
||||
target-rehome-protocol: { required: true, type: string }
|
||||
rollback-rehome-protocol: { required: true, type: string }
|
||||
expected-selector-generation: { required: true, type: string }
|
||||
expected-existing-only-cells: { required: true, type: string }
|
||||
expected-migration-only-cells: { required: true, type: string }
|
||||
expected-general-cells: { required: true, type: string }
|
||||
expected-rehome-generation: { required: true, type: string }
|
||||
wave-index: { required: true, type: string }
|
||||
drain-pace-window-ms: { required: true, type: string }
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
rollout:
|
||||
if: ${{ github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
# The pre-drain sample adds 3-8 min (20 at most); a timeout cancels the job and skips the failsafe.
|
||||
# A 1200000 drain step alone may take 50 min.
|
||||
timeout-minutes: 120
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay
|
||||
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }}
|
||||
ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }}
|
||||
TARGET_REHOME_PROTOCOL: ${{ inputs.target-rehome-protocol }}
|
||||
ROLLBACK_REHOME_PROTOCOL: ${{ inputs.rollback-rehome-protocol }}
|
||||
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
||||
EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
|
||||
EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
|
||||
EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
|
||||
EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
||||
WAVE_INDEX: ${{ inputs.wave-index }}
|
||||
# The cell spreads its drain sends over this window, so each drained host re-dials
|
||||
# /v1/assign at hosts / window and is re-placed through the director's drain-return
|
||||
# lane. The wave validator admits only its closed set, and anything below the 300000
|
||||
# default only on US general cells: Asia drains are bound by their targets' accept
|
||||
# rate. The transition wait below is the 15-min migration lease plus this window. A
|
||||
# cell on an older image drains at once, or refuses a window above 300000.
|
||||
DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }}
|
||||
steps:
|
||||
- name: Require exact reusable-workflow configuration
|
||||
working-directory: .
|
||||
env:
|
||||
DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
CAPACITY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
[[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]]
|
||||
[[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
[[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}"
|
||||
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
|
||||
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
|
||||
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
# cell_1..cell_10 in the calling wave; the chain is static, so this range is too.
|
||||
[[ "${WAVE_INDEX}" =~ ^[0-9]$ ]]
|
||||
if test "${DEPLOY_MODE}" != verify && test "${GITHUB_RUN_ATTEMPT}" != 1; then
|
||||
echo "mutations are single-dispatch: recover each remaining cell" >&2
|
||||
echo "with its own fresh dispatch instead of a re-run" >&2
|
||||
exit 1
|
||||
fi
|
||||
test -n "${GCP_REGION}"
|
||||
test -n "${DEPLOY_WIF}"
|
||||
test -n "${DEPLOY_SERVICE_ACCOUNT}"
|
||||
test -n "${CAPACITY_WIF}"
|
||||
test -n "${CAPACITY_SERVICE_ACCOUNT}"
|
||||
test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with: { package-json-file: cloud/package.json }
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with: { terraform_wrapper: false }
|
||||
|
||||
# One approved-cell table, in the wave validator the dispatch gate already uses, so
|
||||
# a cell's class and its wave's selector delta cannot drift apart between the two.
|
||||
# Also re-checks the drain pace against this cell, before anything reads production.
|
||||
- name: Resolve this cell's admission class and wave selector delta
|
||||
run: |
|
||||
CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \
|
||||
--cell-id "${TARGET_CELL_ID}" --drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}")"
|
||||
ENTRY_ADMISSION="$(jq -er '.entryAdmission' <<< "${CELL_CLASS}")"
|
||||
SELECTOR_WAVE_DELTA="$(jq -er '.selectorWaveDelta' <<< "${CELL_CLASS}")"
|
||||
if test "${DEPLOY_MODE}" = verify; then
|
||||
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
|
||||
else
|
||||
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION \
|
||||
+ (SELECTOR_WAVE_DELTA * WAVE_INDEX)))"
|
||||
fi
|
||||
{
|
||||
echo "ENTRY_ADMISSION=${ENTRY_ADMISSION}"
|
||||
echo "SELECTOR_WAVE_DELTA=${SELECTOR_WAVE_DELTA}"
|
||||
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- id: deploy-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
release: 'false'
|
||||
|
||||
- name: Recheck aggregate SQL, pool, reconnect, migration, and selector safety
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
# Freshness-only failures are publish lag, not health, so the CLI retries
|
||||
# them a bounded number of times. The expected selector comes from the
|
||||
# dispatch inputs, which the rehome inspect below verifies against the
|
||||
# live director; the pre-drain sample later reads the same file.
|
||||
jq -n \
|
||||
--arg existingOnly "${EXPECTED_EXISTING_ONLY_CELLS/none/}" \
|
||||
--arg migrationOnly "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
|
||||
--arg general "${EXPECTED_GENERAL_CELLS/none/}" \
|
||||
'{existingOnly:$existingOnly,migrationOnly:$migrationOnly,general:$general}
|
||||
| map_values(split(",") | map(select(length > 0)))' \
|
||||
> "${RUNNER_TEMP}/relay-same-cap-selector.json"
|
||||
pnpm incident:relay-preflight -- \
|
||||
--no-monitor-state \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \
|
||||
--wave-index "${WAVE_INDEX}" \
|
||||
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
|
||||
|
||||
- name: Require durable rehome disabled and exact selector
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode inspect \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${EFFECTIVE_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
|
||||
| jq -e '.control.enabled == false' >/dev/null
|
||||
|
||||
- name: Initialize the exact production backend
|
||||
run: node dev/scripts/infra.mjs init --env production
|
||||
|
||||
- name: Resolve immutable same-cap cell configuration
|
||||
shell: bash
|
||||
run: |
|
||||
TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"
|
||||
case "${TARGET_HOSTNAME}" in
|
||||
c7|c8|c9|c10|c13|c14|c15|c16|c19|c20|c21|c22|c23|c24|c25|c26)
|
||||
EXPECTED_HARD_CAP=1000
|
||||
EXPECTED_REGION=us-central1
|
||||
EXPECTED_DATABASE_POOL_MAX=
|
||||
;;
|
||||
c17|c18)
|
||||
EXPECTED_HARD_CAP=600
|
||||
EXPECTED_REGION=us-central1
|
||||
EXPECTED_DATABASE_POOL_MAX=
|
||||
;;
|
||||
c27|c28|c29|c30|c31|c34|c35)
|
||||
EXPECTED_HARD_CAP=3000
|
||||
EXPECTED_REGION=asia-east2
|
||||
EXPECTED_DATABASE_POOL_MAX=16
|
||||
;;
|
||||
c32|c33)
|
||||
EXPECTED_HARD_CAP=3000
|
||||
EXPECTED_REGION=us-central1
|
||||
EXPECTED_DATABASE_POOL_MAX=
|
||||
;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
EXPECTED_UNOBSERVED_BOUND=60
|
||||
CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev"
|
||||
CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
|
||||
SOURCE_CELLS="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
<<< 'jsonencode(var.relay_region_rehome_source_cell_ids)' | jq -er '.')"
|
||||
CURRENT_SHAPE="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${CELLS_JSON}")"
|
||||
test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}"
|
||||
test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \
|
||||
"${EXPECTED_UNOBSERVED_BOUND}"
|
||||
# The startup script emits a pool line only off the root default, so an unpinned cell
|
||||
# must still be on that default or its plan would carry a line nothing reviews.
|
||||
test "$(jq -r '.database_pool_max' <<< "${CURRENT_SHAPE}")" = \
|
||||
"${EXPECTED_DATABASE_POOL_MAX:-10}"
|
||||
TARGET_ZONE="$(jq -r '.zone' <<< "${CURRENT_SHAPE}")"
|
||||
MIG_NAME="orca-cloud-relay-gce-${TARGET_HOSTNAME}"
|
||||
if test "${DEPLOY_MODE}" = rollback; then
|
||||
DESIRED_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}"
|
||||
CURRENT_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}"
|
||||
DESIRED_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
|
||||
CURRENT_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
|
||||
else
|
||||
DESIRED_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}"
|
||||
CURRENT_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}"
|
||||
DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
|
||||
CURRENT_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
|
||||
fi
|
||||
# The startup template emits rehome trust lines only for a declared source cell, so
|
||||
# require membership exactly when either side of this roll expects those lines.
|
||||
if test "${EXPECTED_REGION}" = us-central1 && {
|
||||
test "${DESIRED_REHOME_PROTOCOL}" != 0 || test "${CURRENT_REHOME_PROTOCOL}" != 0
|
||||
}; then
|
||||
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
|
||||
<<< "${SOURCE_CELLS}" >/dev/null
|
||||
fi
|
||||
DESIRED_IMAGE="${IMAGE_REPOSITORY}@${DESIRED_IMAGE_DIGEST}"
|
||||
OVERRIDE_CELLS_JSON="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
|
||||
--arg image "${DESIRED_IMAGE}" '.[$cell].image = $image' <<< "${CELLS_JSON}")"
|
||||
jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \
|
||||
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-same-cap.tfvars.json"
|
||||
SERVED_DIGEST="$(gcloud artifacts docker images describe "${DESIRED_IMAGE}" \
|
||||
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${SERVED_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"
|
||||
{
|
||||
echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}"
|
||||
echo "CELL_ORIGIN=${CELL_ORIGIN}"
|
||||
echo "TARGET_ZONE=${TARGET_ZONE}"
|
||||
echo "MIG_NAME=${MIG_NAME}"
|
||||
echo "EXPECTED_HARD_CAP=${EXPECTED_HARD_CAP}"
|
||||
echo "EXPECTED_UNOBSERVED_BOUND=${EXPECTED_UNOBSERVED_BOUND}"
|
||||
echo "EXPECTED_DATABASE_POOL_MAX=${EXPECTED_DATABASE_POOL_MAX}"
|
||||
echo "EXPECTED_REGION=${EXPECTED_REGION}"
|
||||
echo "DESIRED_IMAGE=${DESIRED_IMAGE}"
|
||||
echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}"
|
||||
echo "CURRENT_IMAGE_DIGEST=${CURRENT_IMAGE_DIGEST}"
|
||||
echo "DESIRED_REHOME_PROTOCOL=${DESIRED_REHOME_PROTOCOL}"
|
||||
echo "CURRENT_REHOME_PROTOCOL=${CURRENT_REHOME_PROTOCOL}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify exact current generation, digest, cap, and rollback point
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
# A single transient 5xx (LB warm-up behind a fresh instance) must not
|
||||
# fail a canary; 4xx (auth, generation mismatch) still fails fast.
|
||||
admin_post() {
|
||||
local out="${RUNNER_TEMP}/$1.json"
|
||||
if ! curl --fail-with-body --max-time 30 \
|
||||
--retry 3 --retry-delay 2 --retry-connrefused --output "${out}" \
|
||||
--request POST "$2" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data "$3"; then
|
||||
cat "${out}" >&2
|
||||
return 1
|
||||
fi
|
||||
cat "${out}"
|
||||
}
|
||||
CURRENT_RUNTIME="$(admin_post current-runtime \
|
||||
"${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
|
||||
# Two different failures leave the cell on the rollback image, and the image
|
||||
# alone cannot tell them apart. A rollback that failed between its template
|
||||
# apply and its admission restore restarted the cell, so that cell is not
|
||||
# draining and resumes. A wave that stopped after its drain and before its
|
||||
# template apply never restarted anything, so its cell is still draining and
|
||||
# is stranded: the drain flag only clears on a restart, so it has to be rolled.
|
||||
LIVE_IMAGE_DIGEST="$(jq -r '.imageDigest' <<< "${CURRENT_RUNTIME}")"
|
||||
LIVE_DRAINING="$(jq -r '.draining' <<< "${CURRENT_RUNTIME}")"
|
||||
if test "${DEPLOY_MODE}" = rollback \
|
||||
&& test "${LIVE_IMAGE_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"; then
|
||||
if test "${LIVE_DRAINING}" = true; then
|
||||
ROLLBACK_STAGE=stranded
|
||||
else
|
||||
ROLLBACK_STAGE=resume
|
||||
fi
|
||||
PREDECESSOR_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
|
||||
PREDECESSOR_REHOME_PROTOCOL="${DESIRED_REHOME_PROTOCOL}"
|
||||
else
|
||||
ROLLBACK_STAGE=roll
|
||||
PREDECESSOR_IMAGE_DIGEST="${CURRENT_IMAGE_DIGEST}"
|
||||
PREDECESSOR_REHOME_PROTOCOL="${CURRENT_REHOME_PROTOCOL}"
|
||||
fi
|
||||
if test "${ROLLBACK_STAGE}" = resume; then
|
||||
ROLLBACK_RESUME=true
|
||||
else
|
||||
ROLLBACK_RESUME=false
|
||||
fi
|
||||
# A stranded cell's template still carries the image the cell is serving, so that
|
||||
# is the predecessor its plan is reviewed against. A template already moved on to
|
||||
# the target is refused here rather than rolled backwards under a stale review.
|
||||
if test "${ROLLBACK_STAGE}" = stranded; then
|
||||
PLAN_ROLLBACK_IMAGE="${DESIRED_IMAGE}"
|
||||
else
|
||||
PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"
|
||||
fi
|
||||
# Rollback is the documented recovery from a failed canary, which
|
||||
# leaves the cell migration-only (and possibly still marked
|
||||
# draining); apply and verify still require the cell pristine in the
|
||||
# class it is declared to serve in.
|
||||
if test "${DEPLOY_MODE}" = rollback; then
|
||||
PRECHECK_ADMISSION=general-or-migration-only
|
||||
else
|
||||
PRECHECK_ADMISSION="${ENTRY_ADMISSION}"
|
||||
fi
|
||||
# Draining sheds connections, and a migration-only cell holds none, so the flag
|
||||
# carries no precondition there. It also outlives a failed wave, because the drain
|
||||
# that set it is followed by no restart, which is the state a failed canary leaves.
|
||||
if test "${DEPLOY_MODE}" = rollback \
|
||||
|| test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
PRECHECK_DRAINING=either
|
||||
else
|
||||
PRECHECK_DRAINING=forbidden
|
||||
fi
|
||||
# A resumed rollback already restarted, so its cell has to come back not draining;
|
||||
# that is what separates it from a wave that stopped before its template apply.
|
||||
if test "${PRECHECK_DRAINING}" = either && test "${ROLLBACK_RESUME}" != true; then
|
||||
PREDECESSOR_DRAINING_OK=true
|
||||
else
|
||||
PREDECESSOR_DRAINING_OK=false
|
||||
fi
|
||||
RESTORED_MIGRATION_CELLS="$(jq -rn \
|
||||
--arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
|
||||
--arg target "${TARGET_CELL_ID}" \
|
||||
'$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
|
||||
RESTORED_GENERAL_CELLS="$(jq -rn \
|
||||
--arg value "${EXPECTED_GENERAL_CELLS/none/}" \
|
||||
--arg target "${TARGET_CELL_ID}" \
|
||||
'$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')"
|
||||
test -n "${RESTORED_MIGRATION_CELLS}" || RESTORED_MIGRATION_CELLS=none
|
||||
test -n "${RESTORED_GENERAL_CELLS}" || RESTORED_GENERAL_CELLS=none
|
||||
ISOLATED_MIGRATION_CELLS="$(jq -rn \
|
||||
--arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
|
||||
--arg target "${TARGET_CELL_ID}" \
|
||||
'$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')"
|
||||
ISOLATED_GENERAL_CELLS="$(jq -rn \
|
||||
--arg value "${EXPECTED_GENERAL_CELLS/none/}" \
|
||||
--arg target "${TARGET_CELL_ID}" \
|
||||
'$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
|
||||
test -n "${ISOLATED_MIGRATION_CELLS}" || ISOLATED_MIGRATION_CELLS=none
|
||||
test -n "${ISOLATED_GENERAL_CELLS}" || ISOLATED_GENERAL_CELLS=none
|
||||
# A migration-only cell is already isolated and is handed back isolated, so both
|
||||
# halves of its wave see exactly the membership it entered with.
|
||||
if test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
RESTORED_MIGRATION_CELLS="${ISOLATED_MIGRATION_CELLS}"
|
||||
RESTORED_GENERAL_CELLS="${ISOLATED_GENERAL_CELLS}"
|
||||
fi
|
||||
{
|
||||
echo "ROLLBACK_RESUME=${ROLLBACK_RESUME}"
|
||||
echo "ROLLBACK_STAGE=${ROLLBACK_STAGE}"
|
||||
echo "PLAN_ROLLBACK_IMAGE=${PLAN_ROLLBACK_IMAGE}"
|
||||
# The drain wait and the plan review both read the image this cell actually
|
||||
# serves, which is the rollback image on a stranded cell and not the current one.
|
||||
echo "PREDECESSOR_IMAGE_DIGEST=${PREDECESSOR_IMAGE_DIGEST}"
|
||||
# The failsafe consumes these; deriving them here keeps them
|
||||
# defined for a failure in any later step.
|
||||
echo "ISOLATED_MIGRATION_CELLS=${ISOLATED_MIGRATION_CELLS}"
|
||||
echo "ISOLATED_GENERAL_CELLS=${ISOLATED_GENERAL_CELLS}"
|
||||
# No restart happens on resume, so isolate below is skipped and
|
||||
# cannot advance the selector generation.
|
||||
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${EFFECTIVE_SELECTOR_GENERATION}"
|
||||
# A failed-canary rollback enters with the target migration-only,
|
||||
# so the restore inspect cannot reuse the entry membership inputs.
|
||||
echo "RESTORED_MIGRATION_CELLS=${RESTORED_MIGRATION_CELLS}"
|
||||
echo "RESTORED_GENERAL_CELLS=${RESTORED_GENERAL_CELLS}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
if ! jq -e --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \
|
||||
--arg digest "${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
--arg region "${EXPECTED_REGION}" \
|
||||
--argjson hardCap "${EXPECTED_HARD_CAP}" \
|
||||
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
|
||||
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
|
||||
'.role == "cell" and .cellId == $cell and .cellUrl == $origin and
|
||||
(.region == $region or
|
||||
($region == "us-central1" and $protocol == 0 and .region == null)) and
|
||||
.imageDigest == $digest and
|
||||
.connectionCapacity.hardCap == $hardCap and
|
||||
.connectionCapacity.unobservedBound == $unobservedBound and
|
||||
(.draining == false or $drainingOk) and
|
||||
(.regionalRehomeProtocol // 0) == $protocol' <<< "${CURRENT_RUNTIME}" >/dev/null
|
||||
then
|
||||
jq -r --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \
|
||||
--arg digest "${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
--arg region "${EXPECTED_REGION}" \
|
||||
--argjson hardCap "${EXPECTED_HARD_CAP}" \
|
||||
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
|
||||
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
|
||||
'[
|
||||
if .role != "cell" then "role" else empty end,
|
||||
if .cellId != $cell then "cellId" else empty end,
|
||||
if .cellUrl != $origin then "cellUrl" else empty end,
|
||||
if (.region != $region and
|
||||
($region != "us-central1" or $protocol != 0 or .region != null))
|
||||
then "region" else empty end,
|
||||
if .imageDigest != $digest then "imageDigest" else empty end,
|
||||
if .connectionCapacity.hardCap != $hardCap then "hardCap" else empty end,
|
||||
if .connectionCapacity.unobservedBound != $unobservedBound then "unobservedBound" else empty end,
|
||||
if (.draining != false and ($drainingOk | not)) then "draining" else empty end,
|
||||
if (.regionalRehomeProtocol // 0) != $protocol then "regionalRehomeProtocol" else empty end
|
||||
] | "runtime predecessor mismatch fields=" + join(",")' \
|
||||
<<< "${CURRENT_RUNTIME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
# The exact legacy digest binds omitted pre-region fields to US and protocol 0.
|
||||
jq -r '[
|
||||
if .region == null then "region" else empty end,
|
||||
if .regionalRehomeProtocol == null then "regionalRehomeProtocol" else empty end
|
||||
] | if length > 0 then "runtime predecessor normalized legacy fields=" + join(",") else empty end' \
|
||||
<<< "${CURRENT_RUNTIME}"
|
||||
if ! CURRENT_DIRECTOR_STATUS="$(admin_post current-cell-status \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
"$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"; then
|
||||
if jq -e '.error == "cell_in_reserve_mode"' \
|
||||
"${RUNNER_TEMP}/current-cell-status.json" >/dev/null 2>&1; then
|
||||
echo "${TARGET_CELL_ID} is in reserve mode: flip it to admitMode=db with the cell-flags workflow and wait for it to record db before rolling it, then flip it back to reserve" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
SOURCE_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \
|
||||
<<< "${CURRENT_DIRECTOR_STATUS}")"
|
||||
if test "${ROLLBACK_RESUME}" = true && ! jq -e \
|
||||
'.status.admissionState == "migration-only"' \
|
||||
<<< "${CURRENT_DIRECTOR_STATUS}" >/dev/null; then
|
||||
echo 'resume requires the isolated migration-only cell a failed rollback leaves' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ "${SOURCE_INCARNATION}" =~ ^[0-9a-f-]{36}$ ]]
|
||||
echo "SOURCE_INCARNATION=${SOURCE_INCARNATION}" >> "${GITHUB_ENV}"
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh --admission "${PRECHECK_ADMISSION}" \
|
||||
--draining "${PRECHECK_DRAINING}" --activity allowed \
|
||||
--expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}"
|
||||
|
||||
# Hosts drained with nowhere to go keep redialling and pin the cell empty (c28, 2026-10-01).
|
||||
# Every run that drains (apply, rollback) checks first; verify reads the same answer; a resume drains nothing.
|
||||
- name: Require free general-cell slots for the selected cell's hosts
|
||||
if: ${{ env.ROLLBACK_RESUME != 'true' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
HEADROOM_STATUS=0
|
||||
HEADROOM="$(node dev/scripts/check-relay-same-cap-headroom.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --general-cells "${EXPECTED_GENERAL_CELLS}")" \
|
||||
|| HEADROOM_STATUS=$?
|
||||
echo "${HEADROOM}"
|
||||
test "${HEADROOM_STATUS}" = 0
|
||||
# The pre-drain sample sizes its window from this count.
|
||||
TARGET_HOSTS="$(jq -er '.targetHosts' <<< "${HEADROOM}")"
|
||||
echo "TARGET_HOSTS=${TARGET_HOSTS}" >> "${GITHUB_ENV}"
|
||||
|
||||
# The last read before isolation, so a roll drains on health measured minutes ago, not on a
|
||||
# separate monitor run's verdict. Its window grows with the hosts the drain will re-place;
|
||||
# every later wave samples again. Any trip fails here, before anything is changed. Rollback
|
||||
# skips it: getting off a crash-looping image must not wait for the crashes to stop.
|
||||
- name: Sample fleet health for a window sized to this drain
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-pre-drain-sample -- \
|
||||
--target-cell-id "${TARGET_CELL_ID}" \
|
||||
--target-hosts "${TARGET_HOSTS}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \
|
||||
--wave-index "${WAVE_INDEX}" \
|
||||
--selector-wave-delta "${SELECTOR_WAVE_DELTA}"
|
||||
|
||||
- name: Finish read-only verification
|
||||
if: ${{ inputs.mode == 'verify' }}
|
||||
run: echo 'Exact same-cap rollback point verified.'
|
||||
|
||||
# The sample can age the first one-hour token by up to 20 min; the drain's restart-safe
|
||||
# wait (20 min at the default pace, 35 at the slowest) gets a fresh one.
|
||||
- id: drain-auth
|
||||
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Reversibly isolate and drain only the selected cell
|
||||
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
|
||||
id: drain
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.drain-auth.outputs.id_token }}
|
||||
run: |
|
||||
# Before any mutation: a cell whose image caps the pace below this one stops untouched.
|
||||
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode pace-check \
|
||||
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}"
|
||||
# Opens the window the report-only shadow health gate below judges this cell over.
|
||||
echo "drain-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
|
||||
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
# A cell isolated by a failed canary is already migration-only, so
|
||||
# isolate is a no-op there that does not advance the selector; the
|
||||
# result's generation is authoritative either way.
|
||||
ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
|
||||
echo "${ISOLATE_RESULT}"
|
||||
# Isolating a migration-only cell must be a read-only no-op; a change here would
|
||||
# mean the live class is not the one this wave planned around.
|
||||
if test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
jq -e '.changed == false' <<< "${ISOLATE_RESULT}" >/dev/null
|
||||
fi
|
||||
# The director re-places hosts off a cell only when the isolate stamped it, so a
|
||||
# script too old to ask for the stamp produces today's behaviour and the canary
|
||||
# reads as "the fix did nothing" with nothing to tell that from a wrong premise.
|
||||
jq -e '.rollIsolated == true' <<< "${ISOLATE_RESULT}" >/dev/null
|
||||
ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
|
||||
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}"
|
||||
DRAIN_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain \
|
||||
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}")"
|
||||
echo "${DRAIN_RESULT}"
|
||||
# What the cell accepted, which is 0 when an older image fell back to an unpaced drain.
|
||||
echo "drain-applied-pace-window-ms=$(jq -er '.paceWindowMs' <<< "${DRAIN_RESULT}")" \
|
||||
>> "${GITHUB_OUTPUT}"
|
||||
# Restart-safe means an empty runtime that lasts past the drain's pace window; leases
|
||||
# held by unplaceable hosts are reported, not waited on (c28, 2026-10-01).
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat either --admission migration-only --draining required \
|
||||
--activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \
|
||||
--timeout-ms "$((900000 + DRAIN_PACE_WINDOW_MS))"
|
||||
echo "drain-settled-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- id: capacity-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
|
||||
- name: Require converged Terraform state and a stable MIG on resume
|
||||
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }}
|
||||
shell: bash
|
||||
env:
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
# A cell on the root pool default emits no pool line, so pin one only where it exists.
|
||||
POOL_ARGUMENTS=()
|
||||
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
|
||||
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
|
||||
fi
|
||||
# Zero resource changes prove the prior run's apply completed and no
|
||||
# restart will follow, keeping the incarnation check honest. Root
|
||||
# outputs may lag a targeted apply, so judge resource_changes only.
|
||||
# The cell backend service is deliberately not targeted: the capacity role has no
|
||||
# compute.backendServices.update, so naming it fails the apply after the MIG has
|
||||
# rolled. Its drain and log_config settings ship in a root apply by a principal
|
||||
# that holds the permission.
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/production.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
|
||||
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
-out="${RUNNER_TEMP}/relay-same-cap-resume.tfplan"
|
||||
if ! terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
|
||||
| jq -e '[.resource_changes[]?
|
||||
| select(.change.actions | any(. != "no-op" and . != "read"))]
|
||||
| length == 0' >/dev/null
|
||||
then
|
||||
# An apply that failed before its template apply also resumes here
|
||||
# (the cell still serves the rollback image), and repo drift since
|
||||
# the cell's last roll (for example newly added rehome trust
|
||||
# config) then legitimately replaces the template. Nothing is
|
||||
# applied on resume either way, so accept exactly the drift the
|
||||
# reviewed validator would let a real apply ship for the image the
|
||||
# cell already serves: the template leaves and re-enters the
|
||||
# rollback image, as exactly the template-and-MIG change pair.
|
||||
terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
|
||||
| jq -r '"resume found unconverged resources: " +
|
||||
([.resource_changes[]?
|
||||
| select(.change.actions | any(. != "no-op" and . != "read"))
|
||||
| .address] | join(","))'
|
||||
echo 'requiring reviewed rollback-image drift'
|
||||
RESUME_REVIEW="$(terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${DESIRED_IMAGE}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
"${POOL_ARGUMENTS[@]}")"
|
||||
echo "${RESUME_REVIEW}"
|
||||
jq -e '.changes == 2' <<< "${RESUME_REVIEW}" >/dev/null
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
|
||||
- name: Apply only the selected same-cap template and MIG
|
||||
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
|
||||
id: apply
|
||||
shell: bash
|
||||
env:
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
# A cell on the root pool default emits no pool line, so pin one only where it exists.
|
||||
POOL_ARGUMENTS=()
|
||||
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
|
||||
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
|
||||
fi
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/production.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
|
||||
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
-out="${RUNNER_TEMP}/relay-same-cap.tfplan"
|
||||
PLAN_REVIEW="$(terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-same-cap.tfplan" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${PLAN_ROLLBACK_IMAGE}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
"${POOL_ARGUMENTS[@]}")"
|
||||
echo "${PLAN_REVIEW}"
|
||||
# Stamped before the apply, not after it: the new container announces its listener while
|
||||
# the MIG is still converging, so a bound taken at the end of this step is already past
|
||||
# the announcement the shadow gate looks for.
|
||||
echo "apply-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
|
||||
terraform -chdir=infra/terraform apply -auto-approve \
|
||||
"${RUNNER_TEMP}/relay-same-cap.tfplan"
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
# A stranded cell already runs the reviewed template, so the apply above replaces
|
||||
# no instance and the drain flag, which only a restart clears, would survive the
|
||||
# whole wave. Fewer than the template-and-MIG pair means the template stayed put,
|
||||
# including a MIG-only reconciliation. Recreate its one instance from the MIG's
|
||||
# current template; a rolling action would rewrite the MIG's version name outside
|
||||
# Terraform, and every later targeted plan would carry that revert.
|
||||
if test "${ROLLBACK_STAGE}" = stranded \
|
||||
&& jq -e '.changes < 2' <<< "${PLAN_REVIEW}" >/dev/null; then
|
||||
STRANDED_INSTANCE="$(gcloud compute instance-groups managed list-instances \
|
||||
"${MIG_NAME}" --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
|
||||
--format=json \
|
||||
| jq -er 'if length == 1 then .[0].instance | split("/") | last
|
||||
else error("stranded cell MIG does not have exactly one instance") end')"
|
||||
gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \
|
||||
--instances "${STRANDED_INSTANCE}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --quiet
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
fi
|
||||
# Recorded for the operator comparing verdicts; the gate's boot search opens at the
|
||||
# apply-started-at stamp above, not here.
|
||||
echo "apply-completed-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- id: post-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Verify new incarnation, exact image, protocol, and durable safety
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
id: verify-target
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
|
||||
run: |
|
||||
# Rides out one DB stall: 5-7 s, then readiness grace and two 10 s LB
|
||||
# health checks (c18, 10-07 18:49Z: 503 "no healthy upstream", healthy
|
||||
# 16 s later). A cell still down after 60 s fails; other 4xx fail fast.
|
||||
# Retried in this loop: the runner's 7.81 client (Ubuntu 22.04) never
|
||||
# retries under --fail-with-body, so its --retry flags were inert.
|
||||
admin_post() {
|
||||
local out="${RUNNER_TEMP}/$1.json" window=60 delay=5 status
|
||||
local deadline=$((SECONDS + window))
|
||||
while true; do
|
||||
# A refused or timed-out attempt writes no body; never report the last one's.
|
||||
rm -f "${out}"
|
||||
status="$(curl --silent --show-error --max-time 30 --output "${out}" \
|
||||
--write-out '%{http_code}' --request POST "$2" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data "$3")" || status=000
|
||||
case "${status}" in
|
||||
2??) cat "${out}"; return 0 ;;
|
||||
000|408|429|5??) ((SECONDS + delay < deadline)) || break ;;
|
||||
*) break ;;
|
||||
esac
|
||||
sleep "${delay}"
|
||||
done
|
||||
echo "admin_post $1: HTTP ${status}" >&2
|
||||
cat "${out}" >&2 2>/dev/null || true
|
||||
return 1
|
||||
}
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh --admission migration-only --draining forbidden \
|
||||
--activity allowed --expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" --timeout-ms 900000
|
||||
TARGET_RUNTIME="$(admin_post target-runtime \
|
||||
"${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
|
||||
jq -e --arg digest "${DESIRED_IMAGE_DIGEST}" \
|
||||
--argjson protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
'.imageDigest == $digest and (.regionalRehomeProtocol // 0) == $protocol' \
|
||||
<<< "${TARGET_RUNTIME}" >/dev/null
|
||||
TARGET_DIRECTOR_STATUS="$(admin_post target-cell-status \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
"$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
|
||||
TARGET_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \
|
||||
<<< "${TARGET_DIRECTOR_STATUS}")"
|
||||
if test "${ROLLBACK_RESUME}" = true; then
|
||||
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
# No restart happened; the incarnation legitimately stays put.
|
||||
test "${TARGET_INCARNATION}" = "${SOURCE_INCARNATION}"
|
||||
else
|
||||
test "${TARGET_INCARNATION}" != "${SOURCE_INCARNATION}"
|
||||
fi
|
||||
echo "TARGET_INCARNATION=${TARGET_INCARNATION}" >> "${GITHUB_ENV}"
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_ISOLATE}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \
|
||||
--expected-general-cells "${ISOLATED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
|
||||
| jq -e '.control.enabled == false' >/dev/null
|
||||
echo "verify-ended-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Prove exact per-host trust and idempotent no-neighbor behavior
|
||||
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol != '0') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol != '0')) }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/probe-relay-rehome-trust.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-id "${TARGET_CELL_ID}" \
|
||||
--cell-incarnation "${TARGET_INCARNATION}"
|
||||
|
||||
- name: Restore only the verified selected cell to its entry admission
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
|
||||
run: |
|
||||
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
# Activating a migration-only cell would promote it to a serving cell for good, so
|
||||
# restore it with the idempotent isolate that reports the authoritative generation.
|
||||
if test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
RESTORE_MODE=isolate
|
||||
else
|
||||
RESTORE_MODE=activate
|
||||
fi
|
||||
RESTORE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode "${RESTORE_MODE}")"
|
||||
echo "${RESTORE_RESULT}"
|
||||
SELECTOR_GENERATION_AFTER_RESTORE="$(jq -er '.generation' \
|
||||
<<< "${RESTORE_RESULT}")"
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh --admission "${ENTRY_ADMISSION}" \
|
||||
--draining forbidden --activity allowed \
|
||||
--expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_RESTORE}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${RESTORED_MIGRATION_CELLS}" \
|
||||
--expected-general-cells "${RESTORED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
|
||||
| jq -e '.control.enabled == false' >/dev/null
|
||||
|
||||
# Report only: this evaluates the oracles an operator reads by hand after a cell and records
|
||||
# what it would have called, so its verdicts can be compared with the operator's over a full
|
||||
# roll before it is ever allowed to block. Two independent guarantees keep it inert: the
|
||||
# script exits 0 on every verdict, and continue-on-error keeps even a crash off the job's
|
||||
# outcome. The failsafe below therefore cannot fire on anything this step observes.
|
||||
#
|
||||
# It runs after the restore, not before it, for two reasons: the cell goes back into
|
||||
# admission on exactly today's schedule rather than waiting out a minute of log reads, and
|
||||
# the window it judges has closed by then, so Cloud Logging's ingestion lag is behind it.
|
||||
# These are fleet oracles anyway; when this does gate, what it gates is the next cell.
|
||||
- name: Shadow health gate (report only)
|
||||
id: shadow-gate
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
continue-on-error: true
|
||||
# continue-on-error bounds this step's contribution to the job outcome, not its clock,
|
||||
# and its reads are serialised. A timed-out step is a failed step, which continue-on-error
|
||||
# absorbs; without this bound a Logging 429 storm or an expired credential makes every
|
||||
# read cost its full retry budget and can push the job past timeout-minutes, and a
|
||||
# cancelled job takes the whole wave with it. A healthy gate is already minutes of
|
||||
# serial reads, so both bounds sit above that: the script settles at seven minutes and
|
||||
# reaching this eight is the pathological case. Eight on top of a cell of ~14 minutes
|
||||
# (~45 at the slowest pace) still leaves the job's 120 minutes intact.
|
||||
timeout-minutes: 8
|
||||
env:
|
||||
DRAIN_STARTED_AT: ${{ steps.drain.outputs.drain-started-at }}
|
||||
DRAIN_SETTLED_AT: ${{ steps.drain.outputs.drain-settled-at }}
|
||||
DRAIN_APPLIED_PACE_WINDOW_MS: ${{ steps.drain.outputs.drain-applied-pace-window-ms }}
|
||||
APPLY_STARTED_AT: ${{ steps.apply.outputs.apply-started-at }}
|
||||
APPLY_COMPLETED_AT: ${{ steps.apply.outputs.apply-completed-at }}
|
||||
VERIFY_ENDED_AT: ${{ steps.verify-target.outputs.verify-ended-at }}
|
||||
SHADOW_GATE_DIRECTORY: ${{ runner.temp }}/relay-same-cap-shadow-gate
|
||||
SHADOW_GATE_NAME: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json
|
||||
run: |
|
||||
mkdir -p "${SHADOW_GATE_DIRECTORY}"
|
||||
node dev/scripts/relay-same-cap-shadow-gate.mjs \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--cell-host "${TARGET_HOSTNAME}.relay.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" \
|
||||
--director-service orca-cloud-relay \
|
||||
--drain-started-at "${DRAIN_STARTED_AT}" \
|
||||
--apply-started-at "${APPLY_STARTED_AT}" \
|
||||
--apply-completed-at "${APPLY_COMPLETED_AT}" \
|
||||
--verify-ended-at "${VERIFY_ENDED_AT}" \
|
||||
--drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \
|
||||
--drain-applied-pace-window-ms "${DRAIN_APPLIED_PACE_WINDOW_MS}" \
|
||||
--drain-settled-at "${DRAIN_SETTLED_AT}" \
|
||||
--target-hosts "${TARGET_HOSTS:-}" \
|
||||
--summary-file "${GITHUB_STEP_SUMMARY}" \
|
||||
--output-file "${SHADOW_GATE_DIRECTORY}/${SHADOW_GATE_NAME}"
|
||||
|
||||
- name: Publish the shadow health gate verdict
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
continue-on-error: true
|
||||
# One small JSON file; a retrying upload must not spend the wave's remaining minutes either.
|
||||
timeout-minutes: 2
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json
|
||||
path: ${{ runner.temp }}/relay-same-cap-shadow-gate
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
overwrite: true
|
||||
|
||||
- id: cleanup-auth
|
||||
if: ${{ failure() && inputs.mode != 'verify' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Keep a failed cell isolated and rehome disabled
|
||||
if: ${{ failure() && inputs.mode != 'verify' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }}
|
||||
run: |
|
||||
test "${MUTATION_STARTED:-false}" = true || exit 0
|
||||
ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
|
||||
echo "${ISOLATE_RESULT}"
|
||||
# Same reason as the isolate step above: a failed wave leaves this cell isolated
|
||||
# deliberately, and the stamp is what lets its hosts leave.
|
||||
jq -e '.rollIsolated == true' <<< "${ISOLATE_RESULT}" >/dev/null
|
||||
# The isolate result carries the authoritative post-isolate generation;
|
||||
# fixed offsets are wrong whenever an earlier isolate was a no-op.
|
||||
FAILSAFE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${FAILSAFE_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \
|
||||
--expected-general-cells "${ISOLATED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_REHOME_GENERATION}"
|
||||
@@ -1,433 +0,0 @@
|
||||
name: Deploy Relay Production Same-Cap
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Verify, roll one canary, roll a bounded batch, or roll back
|
||||
required: true
|
||||
default: verify
|
||||
type: choice
|
||||
options: [verify, canary-apply, batch-apply, rollback]
|
||||
cell-ids:
|
||||
description: Ordered comma-separated serving cells; one canary or two to ten batch cells
|
||||
required: true
|
||||
type: string
|
||||
target-image-digest:
|
||||
description: Exact immutable compatibility image digest
|
||||
required: true
|
||||
type: string
|
||||
rollback-image-digest:
|
||||
description: Exact immutable currently serving rollback digest
|
||||
required: true
|
||||
type: string
|
||||
target-rehome-protocol:
|
||||
description: Exact target regional-rehome protocol
|
||||
required: true
|
||||
default: '1'
|
||||
type: choice
|
||||
options: ['0', '1', '3']
|
||||
rollback-rehome-protocol:
|
||||
description: Exact rollback regional-rehome protocol
|
||||
required: true
|
||||
default: '0'
|
||||
type: choice
|
||||
options: ['0', '1', '3']
|
||||
expected-selector-generation:
|
||||
description: Exact selector generation before the first cell
|
||||
required: true
|
||||
type: string
|
||||
expected-existing-only-cells:
|
||||
description: Exact existing-only membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-migration-only-cells:
|
||||
description: Exact migration-only membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-general-cells:
|
||||
description: Exact general membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-rehome-generation:
|
||||
description: Exact durable regional-rehome control generation; it must be disabled
|
||||
required: true
|
||||
type: string
|
||||
drain-pace-window-ms:
|
||||
description: Window the cell spreads its drain over; below 300000 only for US general cells; any other value named in the confirmation
|
||||
required: true
|
||||
default: '300000'
|
||||
type: choice
|
||||
options: ['300000', '60000', '30000', '900000', '1200000']
|
||||
canary-run-id:
|
||||
description: Successful same-code canary in this rehome control generation; reusable across batches
|
||||
required: false
|
||||
type: string
|
||||
confirmation:
|
||||
description: Exact digest-and-cell-bound mutation confirmation
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
gate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
# Headroom for the full-history checkout the canary provenance check needs.
|
||||
timeout-minutes: 15
|
||||
environment: production
|
||||
outputs:
|
||||
cells: ${{ steps.wave.outputs.cells }}
|
||||
job-mode: ${{ steps.wave.outputs.job-mode }}
|
||||
steps:
|
||||
# Full history: the canary authority a batch verifies is sealed at an ancestor commit, and
|
||||
# the provenance check fails closed on a commit a shallow clone left out.
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with: { node-version: 24, check-latest: true }
|
||||
|
||||
- id: wave
|
||||
env:
|
||||
MODE: ${{ inputs.mode }}
|
||||
CELL_IDS: ${{ inputs.cell-ids }}
|
||||
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
||||
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
||||
DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }}
|
||||
run: |
|
||||
CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \
|
||||
--mode "${MODE}" --cell-ids "${CELL_IDS}" \
|
||||
--target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \
|
||||
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}" \
|
||||
--drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}")"
|
||||
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
|
||||
if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then
|
||||
echo 'job-mode=apply' >> "${GITHUB_OUTPUT}"
|
||||
else
|
||||
echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
|
||||
- name: Download exact prior canary authority
|
||||
if: ${{ inputs.mode == 'batch-apply' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-same-cap-canary-${{ inputs.canary-run-id }}
|
||||
path: ${{ runner.temp }}/relay-same-cap-canary
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.canary-run-id }}
|
||||
|
||||
- name: Verify canary authority against this batch
|
||||
if: ${{ inputs.mode == 'batch-apply' }}
|
||||
env:
|
||||
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
||||
CELL_IDS: ${{ inputs.cell-ids }}
|
||||
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
||||
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
||||
SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
||||
REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
||||
DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }}
|
||||
run: |
|
||||
node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
|
||||
--file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
|
||||
--commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
|
||||
--cell-ids "${CELL_IDS}" \
|
||||
--target-digest "${TARGET_DIGEST}" \
|
||||
--rollback-digest "${ROLLBACK_DIGEST}" \
|
||||
--drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \
|
||||
--selector-generation "${SELECTOR_GENERATION}" \
|
||||
--rehome-generation "${REHOME_GENERATION}"
|
||||
|
||||
cell_1:
|
||||
needs: gate
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[0] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '0'
|
||||
secrets: inherit
|
||||
|
||||
cell_2:
|
||||
if: ${{ needs.cell_1.result == 'success' && fromJSON(needs.gate.outputs.cells)[1] != null }}
|
||||
needs: [gate, cell_1]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[1] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '1'
|
||||
secrets: inherit
|
||||
|
||||
cell_3:
|
||||
if: ${{ needs.cell_2.result == 'success' && fromJSON(needs.gate.outputs.cells)[2] != null }}
|
||||
needs: [gate, cell_2]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[2] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '2'
|
||||
secrets: inherit
|
||||
|
||||
cell_4:
|
||||
if: ${{ needs.cell_3.result == 'success' && fromJSON(needs.gate.outputs.cells)[3] != null }}
|
||||
needs: [gate, cell_3]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[3] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '3'
|
||||
secrets: inherit
|
||||
|
||||
cell_5:
|
||||
if: ${{ needs.cell_4.result == 'success' && fromJSON(needs.gate.outputs.cells)[4] != null }}
|
||||
needs: [gate, cell_4]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[4] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '4'
|
||||
secrets: inherit
|
||||
|
||||
cell_6:
|
||||
if: ${{ needs.cell_5.result == 'success' && fromJSON(needs.gate.outputs.cells)[5] != null }}
|
||||
needs: [gate, cell_5]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[5] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '5'
|
||||
secrets: inherit
|
||||
|
||||
cell_7:
|
||||
if: ${{ needs.cell_6.result == 'success' && fromJSON(needs.gate.outputs.cells)[6] != null }}
|
||||
needs: [gate, cell_6]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[6] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '6'
|
||||
secrets: inherit
|
||||
|
||||
cell_8:
|
||||
if: ${{ needs.cell_7.result == 'success' && fromJSON(needs.gate.outputs.cells)[7] != null }}
|
||||
needs: [gate, cell_7]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[7] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '7'
|
||||
secrets: inherit
|
||||
|
||||
cell_9:
|
||||
if: ${{ needs.cell_8.result == 'success' && fromJSON(needs.gate.outputs.cells)[8] != null }}
|
||||
needs: [gate, cell_8]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[8] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '8'
|
||||
secrets: inherit
|
||||
|
||||
cell_10:
|
||||
if: ${{ needs.cell_9.result == 'success' && fromJSON(needs.gate.outputs.cells)[9] != null }}
|
||||
needs: [gate, cell_9]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[9] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }}
|
||||
wave-index: '9'
|
||||
secrets: inherit
|
||||
|
||||
seal_canary:
|
||||
if: ${{ inputs.mode == 'canary-apply' }}
|
||||
needs: [gate, cell_1]
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with: { node-version: 24, check-latest: true }
|
||||
|
||||
# Its pace checks are what let a batch drain faster than the default; a missing artifact
|
||||
# seals UNVERIFIED, which only the default accepts. Named from the gate's normalized cell,
|
||||
# as cell_1 uploaded it, so a padded cell-ids input cannot miss it.
|
||||
- name: Download the canary cell's shadow health verdict
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-same-cap-shadow-gate-${{ fromJSON(needs.gate.outputs.cells)[0] }}-${{ github.run_id }}.json
|
||||
path: ${{ runner.temp }}/relay-same-cap-shadow-gate
|
||||
|
||||
- name: Seal exact successful canary authority
|
||||
run: |
|
||||
mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary"
|
||||
node dev/scripts/relay-production-same-cap-wave.mjs create-canary \
|
||||
--cell-id "${{ fromJSON(needs.gate.outputs.cells)[0] }}" \
|
||||
--target-digest "${{ inputs.target-image-digest }}" \
|
||||
--rollback-digest "${{ inputs.rollback-image-digest }}" \
|
||||
--confirmation "${{ inputs.confirmation }}" \
|
||||
--drain-pace-window-ms "${{ inputs.drain-pace-window-ms }}" \
|
||||
--shadow-report "${RUNNER_TEMP}/relay-same-cap-shadow-gate/relay-same-cap-shadow-gate-${{ fromJSON(needs.gate.outputs.cells)[0] }}-${GITHUB_RUN_ID}.json" \
|
||||
--commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \
|
||||
--selector-generation "${{ inputs.expected-selector-generation }}" \
|
||||
--rehome-generation "${{ inputs.expected-rehome-generation }}" \
|
||||
> "${RUNNER_TEMP}/relay-same-cap-canary/authority.json"
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-same-cap-canary-${{ github.run_id }}
|
||||
path: ${{ runner.temp }}/relay-same-cap-canary/authority.json
|
||||
retention-days: 30
|
||||
if-no-files-found: error
|
||||
|
||||
# Every cell job re-enters the run's lease with release: 'false'; only this job frees it.
|
||||
release_lease:
|
||||
if: always()
|
||||
needs:
|
||||
- gate
|
||||
- cell_1
|
||||
- cell_2
|
||||
- cell_3
|
||||
- cell_4
|
||||
- cell_5
|
||||
- cell_6
|
||||
- cell_7
|
||||
- cell_8
|
||||
- cell_9
|
||||
- cell_10
|
||||
- seal_canary
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 10
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
release: 'true'
|
||||
@@ -1,220 +0,0 @@
|
||||
name: Deploy Relay Production Candidate
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source-cell-id:
|
||||
description: Existing Terraform cell ID to evacuate
|
||||
required: true
|
||||
type: string
|
||||
target-cell-id:
|
||||
description: Distinct Terraform candidate cell ID
|
||||
required: true
|
||||
type: string
|
||||
mode:
|
||||
description: Audit/preflight are read-only; recover/continue resume committed work; disable/enable/reset/execute mutate admission
|
||||
required: true
|
||||
default: preflight
|
||||
type: choice
|
||||
options:
|
||||
- audit
|
||||
- preflight
|
||||
- recover-forward
|
||||
- continue-evacuation
|
||||
- disable-cell
|
||||
- enable-empty-cell
|
||||
- reset-empty-candidate
|
||||
- execute
|
||||
confirmation:
|
||||
description: Enter RECOVER_FORWARD, CONTINUE_EVACUATION, DISABLE_CELL, ENABLE_CELL, RESET_CANDIDATE, or EVACUATE for the matching mutation
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-id:
|
||||
description: Successful fresh dry-run monitor workflow run ID
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-attempt:
|
||||
description: Exact dry-run monitor workflow attempt
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
candidate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain
|
||||
SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
|
||||
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Require fresh dry-run evidence reference
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
run: |
|
||||
[[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
|
||||
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
|
||||
- name: Download private dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Verify dry-run artifact before cloud authentication
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run
|
||||
|
||||
- name: Reject previously consumed dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
COUNT="$(gh api \
|
||||
"/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
||||
--jq '.total_count')"
|
||||
test "${COUNT}" = "0"
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
|
||||
- name: Require explicit mutation confirmation
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if [[ "${DEPLOY_MODE}" = "execute" ]]; then
|
||||
test "${CONFIRMATION}" = "EVACUATE"
|
||||
elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then
|
||||
test "${CONFIRMATION}" = "RECOVER_FORWARD"
|
||||
elif [[ "${DEPLOY_MODE}" = "continue-evacuation" ]]; then
|
||||
test "${CONFIRMATION}" = "CONTINUE_EVACUATION"
|
||||
elif [[ "${DEPLOY_MODE}" = "disable-cell" ]]; then
|
||||
test "${CONFIRMATION}" = "DISABLE_CELL"
|
||||
elif [[ "${DEPLOY_MODE}" = "enable-empty-cell" ]]; then
|
||||
test "${CONFIRMATION}" = "ENABLE_CELL"
|
||||
else
|
||||
test "${CONFIRMATION}" = "RESET_CANDIDATE"
|
||||
fi
|
||||
|
||||
- name: Read reviewed Terraform topology
|
||||
run: |
|
||||
node dev/scripts/infra.mjs init --env production
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
|
||||
echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify fresh dry-run evidence against live selector
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
|
||||
--directory "${RUNNER_TEMP}/relay-monitor-evidence" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" \
|
||||
--run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode dry-run \
|
||||
--mutation-mode "${DEPLOY_MODE}" \
|
||||
--source-cell-id "${SOURCE_CELL_ID}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}"
|
||||
|
||||
- name: Recheck all live safety signals
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
|
||||
|
||||
- name: Create single-use dry-run marker
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
|
||||
printf '%s\n' "${GITHUB_RUN_ID}" \
|
||||
> "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
|
||||
|
||||
- name: Consume dry-run evidence
|
||||
if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Preflight or evacuate exact GCE candidate
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/deploy-relay-gce-candidate.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--admin-audience "${ADMIN_AUDIENCE}" \
|
||||
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
|
||||
--source-cell-id "${SOURCE_CELL_ID}" \
|
||||
--target-cell-id "${TARGET_CELL_ID}" \
|
||||
--runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--mode "${DEPLOY_MODE}"
|
||||
@@ -1,112 +0,0 @@
|
||||
name: Deploy Relay Staging GCE Candidate
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source-cell-id:
|
||||
description: Existing Terraform cell ID to evacuate
|
||||
required: true
|
||||
type: string
|
||||
target-cell-id:
|
||||
description: Distinct Terraform candidate cell ID
|
||||
required: true
|
||||
type: string
|
||||
mode:
|
||||
description: Preflight is read-only; reset repairs an empty candidate; execute evacuates
|
||||
required: true
|
||||
default: preflight
|
||||
type: choice
|
||||
options:
|
||||
- preflight
|
||||
- reset-empty-candidate
|
||||
- execute
|
||||
confirmation:
|
||||
description: Enter RESET_CANDIDATE for reset or EVACUATE for execute
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
candidate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
ADMIN_AUDIENCE: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
|
||||
TARGET_CELL_ID: ${{ inputs.target-cell-id }}
|
||||
DEPLOY_MODE: ${{ inputs.mode }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Require explicit mutation confirmation
|
||||
if: ${{ inputs.mode != 'preflight' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if [[ "${DEPLOY_MODE}" = "execute" ]]; then
|
||||
test "${CONFIRMATION}" = "EVACUATE"
|
||||
else
|
||||
test "${CONFIRMATION}" = "RESET_CANDIDATE"
|
||||
fi
|
||||
|
||||
- name: Read reviewed Terraform topology
|
||||
run: |
|
||||
node dev/scripts/infra.mjs init --env staging
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
|
||||
echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Preflight or evacuate exact GCE candidate
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/deploy-relay-gce-candidate.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--admin-audience "${ADMIN_AUDIENCE}" \
|
||||
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
|
||||
--source-cell-id "${SOURCE_CELL_ID}" \
|
||||
--target-cell-id "${TARGET_CELL_ID}" \
|
||||
--runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--mode "${DEPLOY_MODE}"
|
||||
@@ -1,242 +0,0 @@
|
||||
name: Deploy Relay Staging
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
expected-image-digest:
|
||||
description: Exact checked-in production Relay sha256 digest to deploy
|
||||
required: true
|
||||
type: string
|
||||
reserve-placement:
|
||||
description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve
|
||||
required: true
|
||||
default: preserve
|
||||
type: choice
|
||||
options: [preserve, 'off', dry-run, 'on']
|
||||
expected-rehome-generation:
|
||||
description: Exact durable regional-rehome generation; it must remain disabled
|
||||
required: true
|
||||
type: string
|
||||
bootstrap-runtime-identity:
|
||||
description: One-time move from the stamped-cell identity to the director identity
|
||||
required: true
|
||||
default: false
|
||||
type: boolean
|
||||
predecessor-image-digest:
|
||||
description: Bootstrap only. Exact immutable serving predecessor digest
|
||||
required: false
|
||||
type: string
|
||||
capacity-cell-id:
|
||||
description: Register this cell's reviewed staging.tfvars cap and bound with the director
|
||||
required: true
|
||||
default: none
|
||||
type: choice
|
||||
options: [none, staging-gce-c1, staging-gce-c2, staging-gce-c3, staging-gce-c4]
|
||||
min-instances:
|
||||
description: Director floor; preserve keeps the serving revision's, so a reserve run's 2 survives
|
||||
required: true
|
||||
default: preserve
|
||||
type: choice
|
||||
options: [preserve, '0', '2']
|
||||
confirmation:
|
||||
description: BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY for a bootstrap, ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE with a shape baseline
|
||||
required: false
|
||||
type: string
|
||||
shape-baseline-revision:
|
||||
description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts
|
||||
required: false
|
||||
type: string
|
||||
shadow-seat-feed-cells:
|
||||
description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused
|
||||
required: true
|
||||
default: preserve
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# Staging deploy, candidate, auth, and power operations must never overlap.
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
|
||||
DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging
|
||||
REPOSITORY_ID: orca-cloud
|
||||
IMAGE_NAME: relay
|
||||
EXPECTED_IMAGE_DIGEST: ${{ inputs.expected-image-digest }}
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
# The director must trust the identity every staging Relay workflow authenticates as.
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
ASIA_PROOF_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
|
||||
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: orca-cloud-staging-relay-dir@onorca-cloud-staging.iam.gserviceaccount.com
|
||||
PREDECESSOR_RUNTIME_SERVICE_ACCOUNT: orca-cloud-staging-relay@onorca-cloud-staging.iam.gserviceaccount.com
|
||||
REHOME_AUDIENCE: https://relay-staging.onorca.dev/v1/admin/host-drain
|
||||
EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
||||
BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }}
|
||||
PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }}
|
||||
CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }}
|
||||
MIN_INSTANCES: ${{ inputs.min-instances }}
|
||||
SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Require the expected immutable image and identity request
|
||||
shell: bash
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${EXPECTED_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${BOOTSTRAP_RUNTIME_IDENTITY}" =~ ^(true|false)$ ]]
|
||||
[[ "${MIN_INSTANCES}" =~ ^(preserve|0|2)$ ]]
|
||||
if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then
|
||||
[[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test "${CAPACITY_CELL_ID}" = none
|
||||
test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY
|
||||
else
|
||||
test -z "${PREDECESSOR_IMAGE_DIGEST}"
|
||||
fi
|
||||
if test -n "${SHAPE_BASELINE_REVISION}"; then
|
||||
[[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-staging-[0-9]{5}-[a-z0-9]{3}$ ]]
|
||||
test "${BOOTSTRAP_RUNTIME_IDENTITY}" = false
|
||||
test "${CAPACITY_CELL_ID}" = none
|
||||
test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE
|
||||
fi
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
terraform_wrapper: false
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Bind the request to the checked-in staging C4 image
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
terraform -chdir=infra/terraform init -reconfigure \
|
||||
-backend-config=backend/staging.hcl -input=false
|
||||
IMAGE="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'var.relay_gce_cells["staging-gce-c4"].image' | jq -er '.')"
|
||||
test "${IMAGE}" = \
|
||||
"${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${EXPECTED_IMAGE_DIGEST}"
|
||||
echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
|
||||
if test "${CAPACITY_CELL_ID}" != none; then
|
||||
# The deploy script refuses any topology change beyond this one cell's cap and bound.
|
||||
DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'local.relay_director_cells_json' | jq -er '.')"
|
||||
jq -e --arg cell "${CAPACITY_CELL_ID}" \
|
||||
'any(.[]; .id == $cell and .connectionHardCap != null)' \
|
||||
<<< "${DESIRED_CELLS_JSON}" >/dev/null
|
||||
echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Require the mirrored immutable image
|
||||
run: |
|
||||
DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \
|
||||
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${DIGEST}" = "${EXPECTED_IMAGE_DIGEST}"
|
||||
|
||||
- name: Deploy director blue/green
|
||||
env:
|
||||
# An image change reads every cell's admit mode before the traffic move.
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
RESERVE_PLACEMENT: ${{ inputs.reserve-placement }}
|
||||
SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }}
|
||||
run: |
|
||||
regional_version="$(gcloud secrets versions describe latest \
|
||||
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \
|
||||
--format='value(name)' | awk -F/ '{print $NF}')"
|
||||
[[ "${regional_version}" =~ ^[1-9][0-9]*$ ]]
|
||||
RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
|
||||
identity_args=()
|
||||
if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then
|
||||
identity_args=(
|
||||
--bootstrap-runtime-identity true
|
||||
--predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}"
|
||||
--predecessor-image-digest "${PREDECESSOR_IMAGE_DIGEST}"
|
||||
)
|
||||
fi
|
||||
# Omitted, the deploy script inherits the serving revision's floor.
|
||||
floor_args=()
|
||||
if test "${MIN_INSTANCES}" != preserve; then
|
||||
floor_args=(--min-instances "${MIN_INSTANCES}")
|
||||
fi
|
||||
shape_args=()
|
||||
if test -n "${SHAPE_BASELINE_REVISION}"; then
|
||||
shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}")
|
||||
fi
|
||||
topology_args=()
|
||||
if test "${CAPACITY_CELL_ID}" != none; then
|
||||
topology_args=(
|
||||
--capacity-cell-id "${CAPACITY_CELL_ID}"
|
||||
--director-cells-json "${DESIRED_CELLS_JSON}"
|
||||
)
|
||||
fi
|
||||
node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service "${DIRECTOR_SERVICE_NAME}" \
|
||||
--image "${IMAGE}" \
|
||||
--role director \
|
||||
--max-instances 2 \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--deploy-service-account "${DEPLOY_SERVICE_ACCOUNT}" \
|
||||
--asia-proof-service-account "${ASIA_PROOF_SERVICE_ACCOUNT}" \
|
||||
--regional-placement-secret-version "${regional_version}" \
|
||||
"${floor_args[@]}" \
|
||||
--runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
"${identity_args[@]}" \
|
||||
"${shape_args[@]}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience "${REHOME_AUDIENCE}" \
|
||||
--rehome-control-origin https://relay-staging.onorca.dev \
|
||||
--expected-rehome-generation "${EXPECTED_REHOME_GENERATION}" \
|
||||
"${topology_args[@]}" \
|
||||
--admin-audience https://relay-staging.onorca.dev/v1/admin/drain \
|
||||
--reserve-placement "${RESERVE_PLACEMENT}" \
|
||||
--shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}" \
|
||||
--release-id "${RELEASE_ID}"
|
||||
|
||||
- name: Smoke director health
|
||||
run: |
|
||||
URL="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(status.url)')"
|
||||
node dev/scripts/smoke-relay.mjs "${URL}"
|
||||
@@ -1,90 +0,0 @@
|
||||
name: Monitor Relay Cell Clock Skew
|
||||
|
||||
# Why: the 2026-08-01 sustained HOST_OFFLINE incident traced to one cell's
|
||||
# clock running ~100ms ahead, which deterministically failed every host
|
||||
# challenge under a zero-tolerance freshness check. /health and /ready cannot
|
||||
# see clock skew; this monitor alarms before drift reaches the (now 2s)
|
||||
# client tolerance.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '17 * * * *'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
skew:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: cloud/infra/terraform/environments/production.tfvars
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
|
||||
- name: Measure Date-header skew for every relay cell
|
||||
run: |
|
||||
set -u
|
||||
# Date headers carry whole seconds; compare floored seconds on both
|
||||
# sides so perfect sync reads 0/±1 and never flaps. An absolute
|
||||
# floor-skew of >= 2 means real drift of at least ~1s — approaching
|
||||
# the client's 2s challenge tolerance. Millisecond-precision deltas
|
||||
# come from the desktop's named-check logging when activations fail.
|
||||
ALARM_S=2
|
||||
failures=0
|
||||
reachable=0
|
||||
unserved=0
|
||||
# Every relay_gce_cells hostname (the block relayOpsCellsFromTerraform reads), fenced
|
||||
# cells included, so a newly declared cell is probed from its first run.
|
||||
cells="$(awk '
|
||||
/^relay_gce_cells[ \t]*=[ \t]*\{/ { inside = 1; next }
|
||||
inside && /^\}/ { exit }
|
||||
inside && /^[ \t]*hostname[ \t]*=/ { split($0, quoted, "\""); print quoted[2] }
|
||||
' infra/terraform/environments/production.tfvars || true)"
|
||||
if [ -z "${cells}" ]; then
|
||||
echo "::error::read no relay cells from production.tfvars; monitor blind"
|
||||
exit 1
|
||||
fi
|
||||
for cell in ${cells}; do
|
||||
url="https://${cell}.relay.onorca.dev/health"
|
||||
# Why: *.relay.onorca.dev is a wildcard, so the load balancer answers with its own
|
||||
# accurate Date for a fenced, dead, or never-provisioned cell. Timing that reads as
|
||||
# perfect sync. Only an HTTP 200 is the relay process itself answering, so only a
|
||||
# 200 carries a clock worth judging.
|
||||
response="$(curl -sS -D - -o /dev/null --max-time 8 "${url}" 2>/dev/null || true)"
|
||||
status="$(printf '%s' "${response}" | awk 'NR==1 {print $2}')"
|
||||
header="$(printf '%s' "${response}" | tr -d '\r' | grep -i '^date:' || true)"
|
||||
if [ "${status:-000}" != "200" ] || [ -z "${header}" ]; then
|
||||
# Expected for the fenced cells; a dead unfenced cell is caught by the heartbeat
|
||||
# and readiness alerts, not here. Named either way so it is never invisible.
|
||||
echo "${cell}: not serving (status ${status:-none}); no relay clock to judge"
|
||||
unserved=$((unserved + 1))
|
||||
continue
|
||||
fi
|
||||
reachable=$((reachable + 1))
|
||||
server_s="$(date -d "${header#*: }" +%s)"
|
||||
local_s="$(date +%s)"
|
||||
skew=$((server_s - local_s))
|
||||
abs=${skew#-}
|
||||
if [ "${abs}" -ge "${ALARM_S}" ]; then
|
||||
echo "::error::${cell}: clock skew ${skew}s reaches ±${ALARM_S}s alarm"
|
||||
failures=$((failures + 1))
|
||||
else
|
||||
echo "${cell}: skew ${skew}s"
|
||||
fi
|
||||
done
|
||||
echo "cells serving: ${reachable}, not serving: ${unserved}, alarms: ${failures}"
|
||||
if [ "${reachable}" -eq 0 ]; then
|
||||
# Now meaningful: previously the load balancer answered for every name, so this
|
||||
# could never fire and a total fleet outage reported "all healthy".
|
||||
echo "::error::no relay cell is serving; monitor blind"
|
||||
exit 1
|
||||
fi
|
||||
exit "$((failures > 0 ? 1 : 0))"
|
||||
@@ -1,216 +0,0 @@
|
||||
name: Monitor Relay Production Job
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
mode:
|
||||
required: true
|
||||
type: string
|
||||
expected-selector-generation:
|
||||
required: true
|
||||
type: string
|
||||
expected-existing-only-cells:
|
||||
required: true
|
||||
type: string
|
||||
expected-migration-only-cells:
|
||||
required: true
|
||||
type: string
|
||||
expected-general-cells:
|
||||
required: true
|
||||
type: string
|
||||
migration-policy:
|
||||
required: true
|
||||
type: string
|
||||
recovery-source-cell-id:
|
||||
required: true
|
||||
type: string
|
||||
capacity-cell-id:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
monitor:
|
||||
if: ${{ github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 100
|
||||
environment: production
|
||||
env:
|
||||
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
||||
EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
|
||||
EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
|
||||
EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
|
||||
MIGRATION_POLICY: ${{ inputs.migration-policy }}
|
||||
RECOVERY_SOURCE_CELL_ID: ${{ inputs.recovery-source-cell-id }}
|
||||
CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }}
|
||||
INCIDENT_ID: relay-${{ github.run_id }}-${{ inputs.mode }}
|
||||
MONITOR_MODE: ${{ inputs.mode }}
|
||||
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-incident
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- id: prior-attempt
|
||||
if: ${{ github.run_attempt > 1 }}
|
||||
run: echo "value=$((GITHUB_RUN_ATTEMPT - 1))" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Restore prior private monitor state
|
||||
if: ${{ github.run_attempt > 1 }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ steps.prior-attempt.outputs.value }}
|
||||
path: ${{ github.workspace }}/relay-incident
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ github.run_id }}
|
||||
|
||||
- name: Verify restored state provenance
|
||||
if: ${{ github.run_attempt > 1 }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-restore \
|
||||
--directory "${OUTPUT_DIRECTORY}" \
|
||||
--incident-id "${INCIDENT_ID}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--run-attempt "${{ steps.prior-attempt.outputs.value }}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode "${MONITOR_MODE}"
|
||||
echo "RESTART_FLAG=--restart" >> "${GITHUB_ENV}"
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- name: Verify exact-audience admin identity
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
|
||||
|
||||
- name: Run read-only relay dry-run
|
||||
if: ${{ inputs.mode == 'dry-run' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm --filter @orca-cloud/relay-ops incident:monitor \
|
||||
--environment production \
|
||||
--incident-id "${INCIDENT_ID}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--migration-policy "${MIGRATION_POLICY}" \
|
||||
--recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
|
||||
--capacity-cell-id "${CAPACITY_CELL_ID}" \
|
||||
--interval-seconds 60 \
|
||||
--output-directory "${OUTPUT_DIRECTORY}" \
|
||||
--duration-minutes 15 \
|
||||
--pre-drain-dry-run \
|
||||
${RESTART_FLAG:-}
|
||||
|
||||
- name: Run first relay monitor segment
|
||||
if: ${{ inputs.mode == 'monitor' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm --filter @orca-cloud/relay-ops incident:monitor \
|
||||
--environment production \
|
||||
--incident-id "${INCIDENT_ID}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--migration-policy "${MIGRATION_POLICY}" \
|
||||
--recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
|
||||
--capacity-cell-id "${CAPACITY_CELL_ID}" \
|
||||
--interval-seconds 60 \
|
||||
--output-directory "${OUTPUT_DIRECTORY}" \
|
||||
--duration-minutes 90 \
|
||||
--max-samples-this-run 45 \
|
||||
${RESTART_FLAG:-}
|
||||
|
||||
- id: google-auth-refresh
|
||||
if: ${{ inputs.mode == 'monitor' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Run remaining relay monitor window
|
||||
if: ${{ inputs.mode == 'monitor' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-refresh.outputs.id_token }}
|
||||
run: |
|
||||
node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
|
||||
pnpm --filter @orca-cloud/relay-ops incident:monitor \
|
||||
--environment production \
|
||||
--incident-id "${INCIDENT_ID}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--migration-policy "${MIGRATION_POLICY}" \
|
||||
--recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
|
||||
--capacity-cell-id "${CAPACITY_CELL_ID}" \
|
||||
--interval-seconds 60 \
|
||||
--output-directory "${OUTPUT_DIRECTORY}" \
|
||||
--duration-minutes 90 \
|
||||
--restart
|
||||
|
||||
- name: Seal private evidence provenance
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs create \
|
||||
--directory "${OUTPUT_DIRECTORY}" \
|
||||
--incident-id "${INCIDENT_ID}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--run-attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--mode "${MONITOR_MODE}"
|
||||
|
||||
- name: Publish aggregate job summary
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
if [[ -f "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" ]]; then
|
||||
cat "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
else
|
||||
echo "Relay monitor failed before its first aggregate checkpoint." \
|
||||
>> "${GITHUB_STEP_SUMMARY}"
|
||||
fi
|
||||
|
||||
- name: Upload private aggregate evidence
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ github.workspace }}/relay-incident
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
@@ -1,75 +0,0 @@
|
||||
name: Monitor Relay Production
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Run the required 15-minute pre-drain gate or a 90-minute incident watch
|
||||
required: true
|
||||
default: dry-run
|
||||
type: choice
|
||||
options:
|
||||
- dry-run
|
||||
- monitor
|
||||
expected-selector-generation:
|
||||
description: Exact durable admission-selector generation
|
||||
required: true
|
||||
type: string
|
||||
expected-existing-only-cells:
|
||||
description: Exact comma-separated existing-only cells, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-migration-only-cells:
|
||||
description: Exact comma-separated migration-only cells, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-general-cells:
|
||||
description: Exact comma-separated general cells, or none
|
||||
required: true
|
||||
type: string
|
||||
migration-policy:
|
||||
description: Migration checks matched to the intended mutation
|
||||
required: true
|
||||
default: strict
|
||||
type: choice
|
||||
options:
|
||||
- strict
|
||||
- recover-forward
|
||||
- capacity-transition
|
||||
recovery-source-cell-id:
|
||||
description: Existing-only recovery source cell, or none for strict monitoring
|
||||
required: true
|
||||
default: none
|
||||
type: string
|
||||
capacity-cell-id:
|
||||
description: General cell for a capacity-transition monitor, or none
|
||||
required: true
|
||||
default: none
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
monitor:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
uses: ./.github/workflows/cloud-monitor-relay-production-job.yml
|
||||
with:
|
||||
mode: ${{ inputs.mode }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
migration-policy: ${{ inputs.migration-policy }}
|
||||
recovery-source-cell-id: ${{ inputs.recovery-source-cell-id }}
|
||||
capacity-cell-id: ${{ inputs.capacity-cell-id }}
|
||||
@@ -1,558 +0,0 @@
|
||||
name: Operate Relay Asia Admission
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
environment:
|
||||
description: Target Relay environment
|
||||
required: true
|
||||
type: choice
|
||||
options: [staging, production]
|
||||
mode:
|
||||
description: Inspect, initialize, verify, atomically register, promote, or roll back admission
|
||||
required: true
|
||||
default: verify
|
||||
type: choice
|
||||
options: [inspect, initialize, verify, register, configure, promote, rollback]
|
||||
cell-ids:
|
||||
description: Exact reviewed comma-separated cell wave (Asia, or a US cell at the Asia shape)
|
||||
required: true
|
||||
type: string
|
||||
selector-generation:
|
||||
description: Exact live selector generation; leave empty only for inspect
|
||||
required: false
|
||||
type: string
|
||||
selector-membership-sha256:
|
||||
description: Exact fingerprint printed by inspect; required only for initialize
|
||||
required: false
|
||||
type: string
|
||||
selector-attempt-id:
|
||||
description: Durable unique attempt ID; empty for inspect, verify, and configure
|
||||
required: false
|
||||
type: string
|
||||
image-digest:
|
||||
description: Expected compatible Relay sha256 digest
|
||||
required: true
|
||||
type: string
|
||||
director-image-digest:
|
||||
description: Director sha256 digest; required only for configure
|
||||
required: false
|
||||
type: string
|
||||
evidence-run-id:
|
||||
description: Staging evidence run ID for C27, C27 canary run ID for C28/C29; C30-C35 take none and each proves itself by its own canary
|
||||
required: false
|
||||
type: string
|
||||
evidence-run-attempt:
|
||||
description: Exact evidence workflow run attempt; required with an evidence run ID
|
||||
required: false
|
||||
type: string
|
||||
confirmation:
|
||||
description: Exact typed confirmation for a mutation
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
admission:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 30
|
||||
environment: ${{ inputs.environment }}
|
||||
env:
|
||||
DIRECTOR_ORIGIN: ${{ inputs.environment == 'production' && 'https://relay.onorca.dev' || 'https://relay-staging.onorca.dev' }}
|
||||
AUTH_ORIGIN: ${{ inputs.environment == 'production' && 'https://login.onorca.dev' || 'https://auth-staging.onorca.dev' }}
|
||||
DIRECTOR_SERVICE: ${{ inputs.environment == 'production' && 'orca-cloud-relay' || 'orca-cloud-relay-staging' }}
|
||||
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
|
||||
GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }}
|
||||
GCP_REGION: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_REGION || vars.STAGING_GCP_REGION }}
|
||||
DIRECTOR_MAX_INSTANCES: ${{ inputs.environment == 'production' && '5' || '2' }}
|
||||
TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }}
|
||||
TARGET_ENVIRONMENT: ${{ inputs.environment }}
|
||||
OPERATION_MODE: ${{ inputs.mode }}
|
||||
TARGET_CELL_IDS: ${{ inputs.cell-ids }}
|
||||
EXPECTED_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
|
||||
EXPECTED_SELECTOR_MEMBERSHIP_SHA256: ${{ inputs.selector-membership-sha256 }}
|
||||
SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }}
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }}
|
||||
EVIDENCE_RUN_ID: ${{ inputs.evidence-run-id }}
|
||||
EVIDENCE_RUN_ATTEMPT: ${{ inputs.evidence-run-attempt }}
|
||||
OPERATION_CONFIRMATION: ${{ inputs.confirmation }}
|
||||
DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Validate exact operation inputs before authentication
|
||||
id: inputs
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
|
||||
test -n "${DEPLOY_SERVICE_ACCOUNT}"
|
||||
[[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
evidence_kind=none
|
||||
canary_cell=none
|
||||
canary_region=asia-east2
|
||||
artifact_name=none
|
||||
if test "${OPERATION_MODE}" = inspect; then
|
||||
test -z "${EXPECTED_SELECTOR_GENERATION}"
|
||||
test -z "${SELECTOR_ATTEMPT_ID}"
|
||||
test -z "${OPERATION_CONFIRMATION}"
|
||||
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
|
||||
test -z "${DIRECTOR_IMAGE_DIGEST}"
|
||||
else
|
||||
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
fi
|
||||
if test "${OPERATION_MODE}" = initialize; then
|
||||
test "${EXPECTED_SELECTOR_GENERATION}" = 0
|
||||
[[ "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" =~ ^[a-f0-9]{64}$ ]]
|
||||
test -z "${DIRECTOR_IMAGE_DIGEST}"
|
||||
[[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
test "${OPERATION_CONFIRMATION}" = INITIALIZE_ADMISSION_SELECTOR
|
||||
elif test "${OPERATION_MODE}" = verify; then
|
||||
test -z "${SELECTOR_ATTEMPT_ID}"
|
||||
test -z "${OPERATION_CONFIRMATION}"
|
||||
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
|
||||
test -z "${DIRECTOR_IMAGE_DIGEST}"
|
||||
elif test "${OPERATION_MODE}" = inspect; then
|
||||
:
|
||||
elif test "${OPERATION_MODE}" = configure; then
|
||||
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
|
||||
test -z "${SELECTOR_ATTEMPT_ID}"
|
||||
[[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
test "${OPERATION_CONFIRMATION}" = CONFIGURE_ASIA_DIRECTOR
|
||||
else
|
||||
test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
|
||||
test -z "${DIRECTOR_IMAGE_DIGEST}"
|
||||
[[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
case "${OPERATION_MODE}:${OPERATION_CONFIRMATION}" in
|
||||
register:REGISTER_ASIA_MIGRATION_ONLY) ;;
|
||||
promote:PROMOTE_ASIA_GENERAL) ;;
|
||||
rollback:ROLLBACK_ASIA_MIGRATION_ONLY) ;;
|
||||
*) echo "typed confirmation does not match the requested mutation" >&2; exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
if test "${TARGET_ENVIRONMENT}:${OPERATION_MODE}" = production:promote; then
|
||||
case "${TARGET_CELL_IDS}" in
|
||||
production-gce-c27)
|
||||
evidence_kind=staging
|
||||
artifact_name="relay-asia-staging-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}"
|
||||
canary_cell=production-gce-c27
|
||||
;;
|
||||
production-gce-c28,production-gce-c29)
|
||||
evidence_kind=c27
|
||||
artifact_name="relay-asia-c27-canary-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}"
|
||||
;;
|
||||
production-gce-c30|production-gce-c31|production-gce-c34|production-gce-c35)
|
||||
# No earlier proof binds a later cell's generation; its own canary below rolls it back on failure.
|
||||
canary_cell="${TARGET_CELL_IDS}"
|
||||
;;
|
||||
production-gce-c32|production-gce-c33)
|
||||
# A US cell at the Asia shape proves itself the same way, with load aimed at its own region.
|
||||
canary_cell="${TARGET_CELL_IDS}"
|
||||
canary_region=us-central1
|
||||
;;
|
||||
*) echo "production promotion wave is not reviewed" >&2; exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
if test "${evidence_kind}" = none; then
|
||||
test -z "${EVIDENCE_RUN_ID}"
|
||||
test -z "${EVIDENCE_RUN_ATTEMPT}"
|
||||
else
|
||||
[[ "${EVIDENCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${EVIDENCE_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
fi
|
||||
canary=false
|
||||
if test "${canary_cell}" != none; then
|
||||
canary=true
|
||||
# Leaves room for the rollback attempt's -rollback suffix within 128 characters.
|
||||
test "${#SELECTOR_ATTEMPT_ID}" -le 119
|
||||
fi
|
||||
{
|
||||
echo "evidence_kind=${evidence_kind}"
|
||||
echo "artifact_name=${artifact_name}"
|
||||
echo "canary=${canary}"
|
||||
echo "canary_cell=${canary_cell}"
|
||||
echo "canary_region=${canary_region}"
|
||||
echo "canary_hostname=${canary_cell##*-}"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
|
||||
- name: Install exact canary dependencies
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build the canary Relay contract
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
run: pnpm --filter @orca-cloud/relay-contract build
|
||||
|
||||
- name: Download immutable rollout evidence
|
||||
if: ${{ steps.inputs.outputs.evidence_kind != 'none' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ steps.inputs.outputs.artifact_name }}
|
||||
path: ${{ runner.temp }}/relay-asia-input-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.evidence-run-id }}
|
||||
|
||||
- name: Verify evidence provenance and rollout binding before authentication
|
||||
if: ${{ steps.inputs.outputs.evidence_kind != 'none' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
EVIDENCE_KIND: ${{ steps.inputs.outputs.evidence_kind }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
run_json="${RUNNER_TEMP}/relay-asia-evidence-run.json"
|
||||
verified="${RUNNER_TEMP}/relay-asia-evidence-verified"
|
||||
gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${EVIDENCE_RUN_ID}/attempts/${EVIDENCE_RUN_ATTEMPT}" > "${run_json}"
|
||||
evidence_commit_sha="$(
|
||||
jq -er '.head_sha | select(type == "string" and test("^[a-f0-9]{40}$"))' "${run_json}"
|
||||
)"
|
||||
command=(node dev/scripts/relay-asia-rollout-evidence.mjs "verify-${EVIDENCE_KIND}"
|
||||
--evidence "${RUNNER_TEMP}/relay-asia-input-evidence/evidence.json"
|
||||
--run-json "${run_json}"
|
||||
--commit-sha "${evidence_commit_sha}"
|
||||
--image-digest "${IMAGE_DIGEST}"
|
||||
--now "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
--output "${verified}")
|
||||
if test "${EVIDENCE_KIND}" = c27; then
|
||||
command+=(--selector-generation "${EXPECTED_SELECTOR_GENERATION}")
|
||||
fi
|
||||
"${command[@]}"
|
||||
test "$(< "${verified}")" = verified
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ env.DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ env.DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: ${{ env.DIRECTOR_ORIGIN }}/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Require the exact director image before promotion
|
||||
if: ${{ inputs.mode == 'promote' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
runtime="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data '{"v":1}')"
|
||||
test "$(jq -r '.role' <<< "${runtime}")" = director
|
||||
test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}"
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }}
|
||||
object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }}
|
||||
if: ${{ inputs.mode == 'configure' || steps.inputs.outputs.canary == 'true' }}
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
if: ${{ inputs.mode == 'configure' }}
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Run the exact generation-bound admission operation
|
||||
id: admission-operation
|
||||
if: ${{ inputs.mode != 'configure' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
run: |
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--mode "${OPERATION_MODE}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-membership-sha256 "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" \
|
||||
--attempt-id "${SELECTOR_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
generation="$(jq -er '.generation' <<< "${result}")"
|
||||
states="$(jq -cS '.states // {}' <<< "${result}")"
|
||||
membership="$(jq -cS '.membership // empty' <<< "${result}")"
|
||||
membership_sha256="$(jq -r '.membershipSha256 // empty' <<< "${result}")"
|
||||
echo "generation=${generation}" >> "${GITHUB_OUTPUT}"
|
||||
result_dir="${RUNNER_TEMP}/relay-asia-admission-result"
|
||||
mkdir -p "${result_dir}"
|
||||
node dev/scripts/sanitize-relay-asia-admission-result.mjs \
|
||||
<<< "${result}" > "${result_dir}/result.json"
|
||||
{
|
||||
echo "### Relay Asia admission"
|
||||
echo "- Mode: ${OPERATION_MODE}"
|
||||
echo "- Cells: ${TARGET_CELL_IDS}"
|
||||
echo "- Result generation: ${generation}"
|
||||
echo "- States: \`${states}\`"
|
||||
if test -n "${membership}"; then echo "- Membership: \`${membership}\`"; fi
|
||||
if test -n "${membership_sha256}"; then
|
||||
echo "- Membership SHA-256: \`${membership_sha256}\`"
|
||||
fi
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Verify the canary cell state and start the timed canary
|
||||
id: canary-start
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# A later cell's launch cells were checked general by the promotion; verify reads only its digest.
|
||||
case "${CANARY_CELL}" in
|
||||
production-gce-c27)
|
||||
verify_cells=production-gce-c27,production-gce-c28,production-gce-c29
|
||||
expected_states='{"production-gce-c27":"general","production-gce-c28":"migration-only","production-gce-c29":"migration-only"}'
|
||||
;;
|
||||
production-gce-c30|production-gce-c31|production-gce-c32|production-gce-c33|production-gce-c34|production-gce-c35)
|
||||
verify_cells="${CANARY_CELL}"
|
||||
expected_states="{\"${CANARY_CELL}\":\"general\"}"
|
||||
;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment production \
|
||||
--mode verify \
|
||||
--cell-ids "${verify_cells}" \
|
||||
--expected-generation "${{ steps.admission-operation.outputs.generation }}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -cS '.states' <<< "${result}")" = "$(jq -cS '.' <<< "${expected_states}")"
|
||||
echo "verify_cells=${verify_cells}" >> "${GITHUB_OUTPUT}"
|
||||
echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Run a real five-minute canary control and splice
|
||||
id: canary-load
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
env:
|
||||
CANARY_REGION: ${{ steps.inputs.outputs.canary_region }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
log="${RUNNER_TEMP}/relay-asia-canary-load.jsonl"
|
||||
report="${RUNNER_TEMP}/relay-asia-canary-load.json"
|
||||
node dev/scripts/load-relay-controls.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--auth-origin "${AUTH_ORIGIN}" \
|
||||
--preferred-region "${CANARY_REGION}" \
|
||||
--relay-asia-load-principals 1 \
|
||||
--controls 1 \
|
||||
--splices 1 \
|
||||
--capacity-hard-cap 3000 \
|
||||
--ramp-seconds 0 \
|
||||
--duration-seconds 300 \
|
||||
--splice-hold-seconds 60 \
|
||||
--required-lease-horizons 2 > "${log}"
|
||||
jq -cer 'select(.event == "relay_load_complete")' "${log}" | tail -n 1 > "${report}"
|
||||
echo "ended_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Collect regional, Relay SQL, and Cloud SQL canary evidence
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }}
|
||||
CANARY_STARTED_AT: ${{ steps.canary-start.outputs.started_at }}
|
||||
CANARY_VERIFY_CELLS: ${{ steps.canary-start.outputs.verify_cells }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment production \
|
||||
--mode verify \
|
||||
--cell-ids "${CANARY_VERIFY_CELLS}" \
|
||||
--expected-generation "${{ steps.admission-operation.outputs.generation }}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r --arg cell "${CANARY_CELL}" '.states[$cell]' <<< "${result}")" = general
|
||||
ended_at="${{ steps.canary-load.outputs.ended_at }}"
|
||||
sleep 60
|
||||
output="${RUNNER_TEMP}/relay-asia-output-evidence"
|
||||
logs="${RUNNER_TEMP}/relay-asia-canary-runtime-metrics.json"
|
||||
mkdir -p "${output}"
|
||||
gcloud logging read \
|
||||
"timestamp>=\"${CANARY_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--limit 20000 \
|
||||
--format json > "${logs}"
|
||||
node dev/scripts/relay-asia-rollout-evidence.mjs create-canary \
|
||||
--cell-id "${CANARY_CELL}" \
|
||||
--repository "${GITHUB_REPOSITORY}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--run-attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--image-digest "${IMAGE_DIGEST}" \
|
||||
--selector-generation "${{ steps.admission-operation.outputs.generation }}" \
|
||||
--started-at "${CANARY_STARTED_AT}" \
|
||||
--ended-at "${ended_at}" \
|
||||
--load-report "${RUNNER_TEMP}/relay-asia-canary-load.json" \
|
||||
--logs-json "${logs}" \
|
||||
--output "${output}/evidence.json"
|
||||
jq -r '.metrics | to_entries[] | "- \(.key): \(.value)"' \
|
||||
"${output}/evidence.json" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload immutable canary evidence
|
||||
id: canary-evidence-upload
|
||||
if: ${{ steps.inputs.outputs.canary == 'true' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-asia-${{ steps.inputs.outputs.canary_hostname }}-canary-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/relay-asia-output-evidence/evidence.json
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload sanitized admission result
|
||||
if: ${{ inputs.mode != 'configure' && steps.admission-operation.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-asia-admission-result-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/relay-asia-admission-result/result.json
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: Return an unproven canary cell to migration-only
|
||||
if: ${{ always() && steps.inputs.outputs.canary == 'true' && steps.admission-operation.outcome != 'skipped' && steps.canary-evidence-upload.outcome != 'success' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment production \
|
||||
--mode recover-promotion \
|
||||
--cell-ids "${CANARY_CELL}" \
|
||||
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--attempt-id "${SELECTOR_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
|
||||
promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment production \
|
||||
--mode rollback \
|
||||
--cell-ids "${CANARY_CELL}" \
|
||||
--expected-generation "${promoted_generation}" \
|
||||
--attempt-id "${SELECTOR_ATTEMPT_ID}-rollback" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r --arg cell "${CANARY_CELL}" '.states[$cell]' <<< "${result}")" = migration-only
|
||||
|
||||
- name: Require registered migration-only cells before director configuration
|
||||
if: ${{ inputs.mode == 'configure' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
run: |
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--mode registered \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true
|
||||
|
||||
- name: Build the additive director cell configuration
|
||||
if: ${{ inputs.mode == 'configure' }}
|
||||
id: director-config
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}"
|
||||
topology="${RUNNER_TEMP}/relay-asia-state-topology.json"
|
||||
current="${RUNNER_TEMP}/relay-current-director-cells.json"
|
||||
desired="${RUNNER_TEMP}/relay-asia-director-cells.json"
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${topology}"
|
||||
service="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
revision="$(jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end' \
|
||||
<<< "${service}")"
|
||||
gcloud run revisions describe "${revision}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -er '.spec.containers[0].env[] | select(.name == "ORCA_RELAY_CELLS_JSON") | .value | fromjson' \
|
||||
> "${current}"
|
||||
node dev/scripts/prepare-relay-asia-director-cells.mjs \
|
||||
--current-json "${current}" \
|
||||
--topology-json "${topology}" \
|
||||
--output "${desired}" \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--image-digest "${IMAGE_DIGEST}"
|
||||
echo "file=${desired}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Deploy the registered additive director topology
|
||||
if: ${{ inputs.mode == 'configure' }}
|
||||
env:
|
||||
DIRECTOR_CELLS_FILE: ${{ steps.director-config.outputs.file }}
|
||||
run: |
|
||||
current="$(gcloud secrets versions access latest \
|
||||
--project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")"
|
||||
[[ "${current}" =~ ^(true|false)$ ]]
|
||||
image="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}"
|
||||
release_id="asia-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
|
||||
node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service "${DIRECTOR_SERVICE}" \
|
||||
--image "${image}" \
|
||||
--role director \
|
||||
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
|
||||
--release-id "${release_id}" \
|
||||
--director-cells-json "$(< "${DIRECTOR_CELLS_FILE}")" \
|
||||
--prune-revisions false
|
||||
|
||||
- name: Verify selector and heartbeats after director configuration
|
||||
if: ${{ inputs.mode == 'configure' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
run: |
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment "${TARGET_ENVIRONMENT}" \
|
||||
--mode verify \
|
||||
--cell-ids "${TARGET_CELL_IDS}" \
|
||||
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true
|
||||
revision="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end')"
|
||||
revision_json="$(gcloud run revisions describe "${revision}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
jq -e --arg image "us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}" \
|
||||
'.spec.containers[0].image == $image' <<< "${revision_json}" > /dev/null
|
||||
jq -er --arg secret "${REGIONAL_PLACEMENT_SECRET}" \
|
||||
'[.spec.containers[0].env[] |
|
||||
select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
|
||||
(.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
|
||||
{secret: (.secret // .name), version: (.version // .key)} |
|
||||
select(.secret == $secret and (.version | test("^[1-9][0-9]*$")))] |
|
||||
if length == 1 then .[0].version else error("regional switch version missing") end' \
|
||||
<<< "${revision_json}" > "${RUNNER_TEMP}/relay-regional-placement-version"
|
||||
regional_version="$(< "${RUNNER_TEMP}/relay-regional-placement-version")"
|
||||
[[ "$(gcloud secrets versions access "${regional_version}" --project "${GCP_PROJECT_ID}" \
|
||||
--secret "${REGIONAL_PLACEMENT_SECRET}")" =~ ^(true|false)$ ]]
|
||||
echo "Director configuration now lists the registered migration-only Asia cells." >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,127 +0,0 @@
|
||||
name: Operate Relay Production Cell Flags
|
||||
|
||||
# Writes one cell's runtime switch object and waits for the cell to report it applied.
|
||||
# One cell per run; never fleet-wide. dry-run writes nothing.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: dry-run prints the change; write applies it
|
||||
required: true
|
||||
default: dry-run
|
||||
type: choice
|
||||
options: [dry-run, write]
|
||||
cell-id:
|
||||
description: Exact cell id, e.g. production-gce-c26
|
||||
required: true
|
||||
type: string
|
||||
break-glass:
|
||||
description: >-
|
||||
Only for a dead or unreachable reserve cell, with set admitMode=default. Typed as
|
||||
BREAK GLASS <cell-id>; records db without the cell's read-back
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
confirm-reserve:
|
||||
description: Required for admitMode=reserve, typed as RESERVE <cell-id>
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
set:
|
||||
description: >-
|
||||
Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a
|
||||
key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck,
|
||||
admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs,
|
||||
reregisterInFlight, phonePath (memory only takes effect in reserve; a reserve flip
|
||||
must name phonePath)
|
||||
required: true
|
||||
type: string
|
||||
expected-generation:
|
||||
description: Exact current object generation from a dry run, or 0 if none exists
|
||||
required: true
|
||||
type: string
|
||||
confirmation:
|
||||
description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# One write at a time across the fleet keeps the audit trail in order.
|
||||
concurrency:
|
||||
group: relay-production-cell-flags
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
# Explicit bash runs with pipefail, so the script's exit code survives the tee.
|
||||
shell: bash
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
operate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
|
||||
timeout-minutes: 60
|
||||
environment: production
|
||||
env:
|
||||
MODE: ${{ inputs.mode }}
|
||||
CELL_ID: ${{ inputs.cell-id }}
|
||||
FLAG_CHANGES: ${{ inputs.set }}
|
||||
EXPECTED_GENERATION: ${{ inputs.expected-generation }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
CONFIRM_RESERVE: ${{ inputs.confirm-reserve }}
|
||||
BREAK_GLASS: ${{ inputs.break-glass }}
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- name: Write or preview the cell's switch object
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
|
||||
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
|
||||
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
|
||||
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
|
||||
node dev/scripts/operate-relay-cell-flags.mjs \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" \
|
||||
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" \
|
||||
--set "${FLAG_CHANGES}" \
|
||||
--director-origin "https://relay.onorca.dev" --confirm-reserve "${CONFIRM_RESERVE}" \
|
||||
--break-glass "${BREAK_GLASS}" \
|
||||
--expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-cell-flags.jsonl"
|
||||
|
||||
- name: Audit summary
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
{
|
||||
echo "### Relay cell flags: ${MODE} ${CELL_ID}"
|
||||
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
|
||||
echo '```json'
|
||||
cat "${RUNNER_TEMP}/relay-cell-flags.jsonl" 2>/dev/null || echo '(no output)'
|
||||
echo '```'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,342 +0,0 @@
|
||||
name: Operate Relay Production Rehome Job
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
mode: { required: true, type: string }
|
||||
director-image-digest: { required: true, type: string }
|
||||
rollback-image-digest: { required: true, type: string }
|
||||
expected-selector-generation: { required: true, type: string }
|
||||
expected-existing-only-cells: { required: true, type: string }
|
||||
expected-migration-only-cells: { required: true, type: string }
|
||||
expected-general-cells: { required: true, type: string }
|
||||
expected-control-generation: { required: true, type: string }
|
||||
not-before: { required: true, type: string }
|
||||
rate-per-minute: { required: true, type: string }
|
||||
preference-max-age-ms: { required: true, type: string }
|
||||
host-cooldown-ms: { required: true, type: string }
|
||||
drain-grace-ms: { required: true, type: string }
|
||||
confirmation: { required: true, type: string }
|
||||
monitor-run-id: { required: true, type: string }
|
||||
monitor-run-attempt: { required: true, type: string }
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
defaults:
|
||||
run:
|
||||
# `shell: bash` adds pipefail; without it `node ... | tee` reports tee's exit code and a
|
||||
# thrown inspect/apply passed green (Aug 28-29 and Sep 3 2026 runs).
|
||||
shell: bash
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
control:
|
||||
if: ${{ github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 30
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
DIRECTOR_SERVICE: orca-cloud-relay
|
||||
DIRECTOR_ORIGIN: https://relay.onorca.dev
|
||||
REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain
|
||||
MODE: ${{ inputs.mode }}
|
||||
DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }}
|
||||
ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }}
|
||||
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
||||
EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
|
||||
EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
|
||||
EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
|
||||
EXPECTED_CONTROL_GENERATION: ${{ inputs.expected-control-generation }}
|
||||
NOT_BEFORE: ${{ inputs.not-before }}
|
||||
RATE_PER_MINUTE: ${{ inputs.rate-per-minute }}
|
||||
PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }}
|
||||
HOST_COOLDOWN_MS: ${{ inputs.host-cooldown-ms }}
|
||||
DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
||||
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
|
||||
steps:
|
||||
- name: Require exact reusable-workflow configuration
|
||||
working-directory: .
|
||||
env:
|
||||
DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
[[ "${MODE}" =~ ^(inspect|enable|pause|disable)$ ]]
|
||||
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${EXPECTED_CONTROL_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
test -n "${DEPLOY_WIF}"
|
||||
test -n "${DEPLOY_SERVICE_ACCOUNT}"
|
||||
if [[ "${MODE}" =~ ^(inspect|enable)$ ]]; then
|
||||
[[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
[[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test -n "${GCP_REGION}"
|
||||
test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
|
||||
fi
|
||||
case "${MODE}" in
|
||||
inspect)
|
||||
test -z "${CONFIRMATION}"
|
||||
;;
|
||||
enable)
|
||||
test "${CONFIRMATION}" = ENABLE_REGIONAL_REHOMING
|
||||
test "${RATE_PER_MINUTE}" = 10
|
||||
[[ "${NOT_BEFORE}" =~ ^[1-9][0-9]*$ ]]
|
||||
;;
|
||||
pause)
|
||||
test "${CONFIRMATION}" = PAUSE_REGIONAL_REHOMING
|
||||
;;
|
||||
disable)
|
||||
test "${CONFIRMATION}" = DISABLE_REGIONAL_REHOMING
|
||||
;;
|
||||
esac
|
||||
|
||||
# Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
|
||||
# and the provenance check fails closed on a commit a shallow clone left out.
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Apply emergency durable pause or disable before diagnostics
|
||||
if: ${{ inputs.mode == 'pause' || inputs.mode == 'disable' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
|
||||
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
|
||||
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
|
||||
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
with: { package-json-file: cloud/package.json }
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
|
||||
- name: Download fresh aggregate safety evidence
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ github.workspace }}/relay-monitor-evidence
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
- name: Verify enable evidence provenance
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
run: |
|
||||
[[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-authority \
|
||||
--directory "${OUTPUT_DIRECTORY}" \
|
||||
--incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
|
||||
--run-id "${MONITOR_RUN_ID}" --run-attempt "${MONITOR_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" --mode dry-run \
|
||||
--required-migration-policy strict
|
||||
|
||||
- name: Reject previously consumed enable safety evidence
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
COUNT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
|
||||
--jq '.total_count')"
|
||||
test "${COUNT}" = 0
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/production.lock
|
||||
|
||||
- name: Verify exact serving and rollback director identities
|
||||
if: ${{ inputs.mode == 'inspect' || inputs.mode == 'enable' }}
|
||||
env:
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
SERVING_REVISION="$(jq -er \
|
||||
'[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName
|
||||
else error("director does not have one serving revision") end' \
|
||||
<<< "${SERVICE_JSON}")"
|
||||
ROLLBACK_REVISION="$(jq -er \
|
||||
'[.status.traffic[] | select(.tag == "selector-rollback")] |
|
||||
if length == 1 then .[0].revisionName else error("rollback tag missing") end' \
|
||||
<<< "${SERVICE_JSON}")"
|
||||
verify_revision() {
|
||||
local revision="$1" expected_digest="$2"
|
||||
local json
|
||||
json="$(gcloud run revisions describe "${revision}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
test "$(jq -r '.spec.serviceAccountName' <<< "${json}")" = \
|
||||
"${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
|
||||
test "$(jq -r '.spec.containers[0].image | split("@") | last' <<< "${json}")" = \
|
||||
"${expected_digest}"
|
||||
test "$(jq -r '[.spec.containers[0].env[] | select(.name ==
|
||||
"ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT") | .value] | if length == 1
|
||||
then .[0] else empty end' <<< "${json}")" = "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
|
||||
test "$(jq -r '[.spec.containers[0].env[] | select(.name ==
|
||||
"ORCA_RELAY_REHOME_AUDIENCE") | .value] | if length == 1 then .[0]
|
||||
else empty end' <<< "${json}")" = "${REHOME_AUDIENCE}"
|
||||
}
|
||||
verify_revision "${SERVING_REVISION}" "${DIRECTOR_IMAGE_DIGEST}"
|
||||
verify_revision "${ROLLBACK_REVISION}" "${ROLLBACK_IMAGE_DIGEST}"
|
||||
|
||||
- name: Seal 24-hour aggregate region observation evidence
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
run: |
|
||||
mkdir -p "${RUNNER_TEMP}/relay-region-observation"
|
||||
# 6 director instances x 120 samples/hour x 25h = 18000; a clipped
|
||||
# read empties the oldest hourly buckets and fails the seal.
|
||||
gcloud logging read \
|
||||
'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND jsonPayload.event="orca_relay_runtime_metrics" AND jsonPayload.role="director"' \
|
||||
--project "${GCP_PROJECT_ID}" --freshness=25h --limit=30000 --format=json \
|
||||
| node dev/scripts/relay-region-observation-evidence.mjs create \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \
|
||||
--selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
> "${RUNNER_TEMP}/relay-region-observation/evidence.json"
|
||||
|
||||
- name: Upload sealed 24-hour aggregate region evidence
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-region-observation-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/relay-region-observation/evidence.json
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Verify sealed 24-hour enable authority
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
run: |
|
||||
node dev/scripts/relay-region-observation-evidence.mjs verify \
|
||||
--file "${RUNNER_TEMP}/relay-region-observation/evidence.json" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \
|
||||
--selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--control-generation "${EXPECTED_CONTROL_GENERATION}"
|
||||
|
||||
- name: Recheck every aggregate safety signal before enable
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json"
|
||||
|
||||
- name: Seal single-use enable safety authority
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
run: |
|
||||
MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
|
||||
mkdir -p "${RUNNER_TEMP}/relay-rehome-enable-authority"
|
||||
printf '%s\n' "${GITHUB_RUN_ID}" \
|
||||
> "${RUNNER_TEMP}/relay-rehome-enable-authority/${MARKER_NAME}"
|
||||
|
||||
- name: Consume enable safety evidence before durable mutation
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
path: ${{ runner.temp }}/relay-rehome-enable-authority/relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
retention-days: 90
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Inspect regional rehome control
|
||||
if: ${{ inputs.mode == 'inspect' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
|
||||
- name: Apply exact durable regional rehome enable
|
||||
if: ${{ inputs.mode == 'enable' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
|
||||
--expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
|
||||
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
|
||||
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
|
||||
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
|
||||
- name: Read fresh aggregate completion and abort evidence
|
||||
run: |
|
||||
gcloud logging read \
|
||||
'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND textPayload:"[orca-relay] regional rehome inventory"' \
|
||||
--project "${GCP_PROJECT_ID}" --freshness=15m --limit=20 --format=json \
|
||||
| node dev/scripts/relay-rehome-aggregate-evidence.mjs --max-age-ms 900000 \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-inventory.json"
|
||||
|
||||
- name: Publish aggregate control evidence
|
||||
run: |
|
||||
{
|
||||
echo '### Regional rehome control'
|
||||
jq -r '"- mode: `\(.mode)`\n- generation: `\(.control.generation)`\n- enabled: `\(.control.enabled)`"' \
|
||||
"${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
jq -r '"- active: `\(.active)`\n- awaiting receipt: `\(.awaitingReceipt)`\n- target registered: `\(.targetRegistered)`\n- completed (24h): `\(.completedLast24Hours)`\n- aborted (24h): `\(.abortedLast24Hours)`\n- host not arrived (24h): `\(.hostNotArrivedLast24Hours // "not reported")`\n- oldest active age (ms): `\(.oldestActiveAgeMs // "none")`"' \
|
||||
"${RUNNER_TEMP}/relay-rehome-inventory.json"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Fail closed after an unsuccessful enable run
|
||||
if: ${{ failure() && inputs.mode == 'enable' && steps.google-auth.outcome == 'success' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode recover-enable --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
--confirmation RECOVER_FAILED_REGIONAL_REHOME_ENABLE \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-enable-recovery.json"
|
||||
jq -e \
|
||||
'.mode == "recover-enable" and .control.enabled == false' \
|
||||
"${RUNNER_TEMP}/relay-rehome-enable-recovery.json" >/dev/null
|
||||
@@ -1,112 +0,0 @@
|
||||
name: Operate Relay Production Rehome
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Inspect or apply the durable regional-rehome switch
|
||||
required: true
|
||||
default: inspect
|
||||
type: choice
|
||||
options: [inspect, enable, pause, disable]
|
||||
director-image-digest:
|
||||
description: Exact immutable serving director digest
|
||||
required: true
|
||||
type: string
|
||||
rollback-image-digest:
|
||||
description: Exact immutable selector-rollback director digest
|
||||
required: true
|
||||
type: string
|
||||
expected-selector-generation:
|
||||
description: Exact admission selector generation
|
||||
required: true
|
||||
type: string
|
||||
expected-existing-only-cells:
|
||||
description: Exact existing-only membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-migration-only-cells:
|
||||
description: Exact migration-only membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-general-cells:
|
||||
description: Exact general membership, or none
|
||||
required: true
|
||||
type: string
|
||||
expected-control-generation:
|
||||
description: Exact durable rehome generation
|
||||
required: true
|
||||
type: string
|
||||
not-before:
|
||||
description: Exact epoch milliseconds; ignored only by inspect
|
||||
required: true
|
||||
default: '0'
|
||||
type: string
|
||||
rate-per-minute:
|
||||
description: Exact global host rate; initial enable is fixed at 10
|
||||
required: true
|
||||
default: '10'
|
||||
type: string
|
||||
preference-max-age-ms:
|
||||
description: Maximum fresh preference age
|
||||
required: true
|
||||
default: '86400000'
|
||||
type: string
|
||||
host-cooldown-ms:
|
||||
description: Minimum gap between two rehomes of the same host
|
||||
required: true
|
||||
default: '604800000'
|
||||
type: string
|
||||
drain-grace-ms:
|
||||
description: Per-host source drain grace
|
||||
required: true
|
||||
default: '3600000'
|
||||
type: string
|
||||
monitor-run-id:
|
||||
description: Fresh successful aggregate dry-run required only by enable
|
||||
required: false
|
||||
type: string
|
||||
monitor-run-attempt:
|
||||
description: Exact monitor attempt required only by enable
|
||||
required: false
|
||||
type: string
|
||||
confirmation:
|
||||
description: ENABLE_REGIONAL_REHOMING, PAUSE_REGIONAL_REHOMING, or DISABLE_REGIONAL_REHOMING
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: production-cloud-sql-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
operate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
uses: ./.github/workflows/cloud-operate-relay-production-rehome-job.yml
|
||||
with:
|
||||
mode: ${{ inputs.mode }}
|
||||
director-image-digest: ${{ inputs.director-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-control-generation: ${{ inputs.expected-control-generation }}
|
||||
not-before: ${{ inputs.not-before }}
|
||||
rate-per-minute: ${{ inputs.rate-per-minute }}
|
||||
preference-max-age-ms: ${{ inputs.preference-max-age-ms }}
|
||||
host-cooldown-ms: ${{ inputs.host-cooldown-ms }}
|
||||
drain-grace-ms: ${{ inputs.drain-grace-ms }}
|
||||
confirmation: ${{ inputs.confirmation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
secrets: inherit
|
||||
@@ -1,341 +0,0 @@
|
||||
name: Operate Relay Production Reserve Watch
|
||||
|
||||
# Watches one cell already switched to reserve mode and flips it back to db on the first stop
|
||||
# rule it trips (step 5 design §9). Manual only. dry-run evaluates and logs, writes nothing.
|
||||
# An operator watches the run. A refused watch or a flip back that did not land raises an ::error::
|
||||
# annotation, as the clock-skew monitor does, and in write mode logs the line the reserve-watch
|
||||
# alert pages on. Refuses to start while a rollout holds the Cloud SQL rollout lease.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: dry-run logs what a trip would write; write flips the cell back on a trip
|
||||
required: true
|
||||
default: dry-run
|
||||
type: choice
|
||||
options: [dry-run, write]
|
||||
cell-id:
|
||||
description: Exact cell id already in reserve mode, e.g. production-gce-c26
|
||||
required: true
|
||||
type: string
|
||||
duration-minutes:
|
||||
description: How long to watch, 5 to 300
|
||||
required: true
|
||||
default: '120'
|
||||
type: string
|
||||
flipped-at:
|
||||
description: >-
|
||||
When the reserve (or phonePath=memory) write applied, ISO with a zone, e.g.
|
||||
2026-10-14T12:00:00Z. The non-drain 503 background and the phone rules' background are
|
||||
the 10 minutes before it
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
background-503-per-minute:
|
||||
description: >-
|
||||
Director non-drain 503s per minute before the flip (all 503s less drain-return deferrals
|
||||
and a host's own early or duplicate re-dials, from the director's runtime metrics);
|
||||
overrides the one measured before flipped-at. One of the two is required
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
unverifiable-limit:
|
||||
description: Consecutive samples with any unreadable signal that count as a trip
|
||||
required: true
|
||||
default: '3'
|
||||
type: string
|
||||
confirmation:
|
||||
description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# One watch per cell; why not the flag workflow's group: cloud/docs/relay-workflows.md.
|
||||
concurrency:
|
||||
group: relay-production-reserve-watch-${{ inputs.cell-id }}
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
# Explicit bash runs with pipefail, so the script's exit code survives the tee.
|
||||
shell: bash
|
||||
working-directory: cloud
|
||||
|
||||
env:
|
||||
MODE: ${{ inputs.mode }}
|
||||
CELL_ID: ${{ inputs.cell-id }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
|
||||
jobs:
|
||||
watch:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
# Six 55-minute segments cover the longest watch; under the 360-minute job limit.
|
||||
timeout-minutes: 350
|
||||
environment: production
|
||||
outputs:
|
||||
state: ${{ steps.outcome.outputs.state }}
|
||||
verdict: ${{ steps.outcome.outputs.verdict }}
|
||||
# phonePath when a step-6 phone rule tripped; anything else flips the cell back whole.
|
||||
flip: ${{ steps.outcome.outputs.flip }}
|
||||
env:
|
||||
DURATION_MINUTES: ${{ inputs.duration-minutes }}
|
||||
FLIPPED_AT: ${{ inputs.flipped-at }}
|
||||
BACKGROUND_503_PER_MINUTE: ${{ inputs.background-503-per-minute }}
|
||||
UNVERIFIABLE_LIMIT: ${{ inputs.unverifiable-limit }}
|
||||
# One segment, after the auth step before it: fixed text, the inputs arrive as env vars.
|
||||
WATCH_SEGMENT: |
|
||||
set -euo pipefail
|
||||
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
|
||||
node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
|
||||
# A phone-rule trip writes phonePath=db mid-watch, through the flag tool.
|
||||
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
|
||||
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
|
||||
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs watch \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" \
|
||||
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \
|
||||
--director-service orca-cloud-relay --confirmation "${CONFIRMATION}" \
|
||||
--duration-minutes "${DURATION_MINUTES}" --segment-minutes 55 \
|
||||
--flipped-at "${FLIPPED_AT}" --background-503-per-minute "${BACKGROUND_503_PER_MINUTE}" \
|
||||
--unverifiable-limit "${UNVERIFIABLE_LIMIT}" \
|
||||
--state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}" \
|
||||
| tee -a "${RUNNER_TEMP}/relay-reserve-watch.jsonl"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- id: google-auth-1
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- id: watch-1
|
||||
name: Watch segment 1 of 6
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-1.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-2
|
||||
if: ${{ steps.watch-1.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-2
|
||||
name: Watch segment 2 of 6
|
||||
if: ${{ steps.watch-1.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-2.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-3
|
||||
if: ${{ steps.watch-2.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-3
|
||||
name: Watch segment 3 of 6
|
||||
if: ${{ steps.watch-2.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-3.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-4
|
||||
if: ${{ steps.watch-3.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-4
|
||||
name: Watch segment 4 of 6
|
||||
if: ${{ steps.watch-3.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-4.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-5
|
||||
if: ${{ steps.watch-4.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-5
|
||||
name: Watch segment 5 of 6
|
||||
if: ${{ steps.watch-4.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-5.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-6
|
||||
if: ${{ steps.watch-5.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-6
|
||||
name: Watch segment 6 of 6
|
||||
if: ${{ steps.watch-5.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-6.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: outcome
|
||||
name: Record the watch outcome
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs outcome \
|
||||
--state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}"
|
||||
|
||||
# The documented flow writes reserve before dispatching, so a refusal can leave it unwatched.
|
||||
- name: Alarm on a refused watch
|
||||
if: ${{ always() && steps.outcome.outputs.state == 'refused' }}
|
||||
env:
|
||||
WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }}
|
||||
run: |
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind refused \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${WATCH_VERDICT}"
|
||||
|
||||
- name: Audit summary
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
WATCH_STATE: ${{ steps.outcome.outputs.state }}
|
||||
WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }}
|
||||
run: |
|
||||
{
|
||||
echo "### Relay reserve watch: ${MODE} ${CELL_ID}"
|
||||
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "Outcome: ${WATCH_STATE}: ${WATCH_VERDICT}"
|
||||
echo '```json'
|
||||
tail -n 20 "${RUNNER_TEMP}/relay-reserve-watch.jsonl" 2>/dev/null || echo '(no output)'
|
||||
echo '```'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
flip-back:
|
||||
needs: watch
|
||||
# Missing data is never healthy: a watch that crashed, timed out or was cancelled (by an
|
||||
# operator too) flips back like a trip. In dry-run this only plans the write.
|
||||
if: ${{ always() && vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && needs.watch.result != 'skipped' && !contains(fromJSON('["completed","stopped","refused"]'), needs.watch.outputs.state) }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
|
||||
timeout-minutes: 60
|
||||
environment: production
|
||||
# Only the run's per-cell group above, which a running run never yields (cancel-in-progress off).
|
||||
# Every step runs on always() too, so an ordinary cancel cannot skip the switch-file write; a force-cancel
|
||||
# can, and leaves the cell in reserve for the operator to flip back by hand. A cancel during the
|
||||
# lease wait can still kill the process before Postgres records db: the file says db, Postgres reserve.
|
||||
env:
|
||||
VERDICT: ${{ needs.watch.outputs.verdict || format('the watch ended {0} without a verdict', needs.watch.result) }}
|
||||
FLIP: ${{ needs.watch.outputs.flip || 'admitMode' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- id: google-auth
|
||||
if: ${{ always() }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- name: Flip the cell back to db, or preview it
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
|
||||
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
|
||||
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
|
||||
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs flip-back \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" \
|
||||
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \
|
||||
--confirmation "${CONFIRMATION}" --verdict "${VERDICT}" --flip "${FLIP}" \
|
||||
| tee "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"
|
||||
|
||||
# The flip-back line is logged only once the write (or its dry-run plan) finished; without
|
||||
# it a missing token, a failed login or a silent cell left the cell in reserve, unwatched.
|
||||
- name: Alarm when the flip back did not land
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
if grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"; then
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${FLIP}" == phonePath ]]; then
|
||||
echo "## FLIP BACK FAILED: ${CELL_ID} may still have phonePath=memory" >> "${GITHUB_STEP_SUMMARY}"
|
||||
echo "Set phonePath=default by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}"
|
||||
else
|
||||
echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode" >> "${GITHUB_STEP_SUMMARY}"
|
||||
echo "Flip it to db by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}"
|
||||
fi
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind flip-back-failed \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}" --flip "${FLIP}"
|
||||
|
||||
- name: Audit summary
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
{
|
||||
echo "### Relay reserve watch flip back: ${MODE} ${CELL_ID}"
|
||||
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "Verdict: ${VERDICT}"
|
||||
echo '```json'
|
||||
cat "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl" 2>/dev/null || echo '(no output)'
|
||||
echo '```'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,129 +0,0 @@
|
||||
name: Operate Relay Staging Cell Flags
|
||||
|
||||
# The staging twin of Operate Relay Production Cell Flags: writes one staging cell's runtime
|
||||
# switch object and waits for the cell to report it applied. One cell per run. dry-run writes
|
||||
# nothing.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: dry-run prints the change; write applies it
|
||||
required: true
|
||||
default: dry-run
|
||||
type: choice
|
||||
options: [dry-run, write]
|
||||
cell-id:
|
||||
description: Exact cell id, e.g. staging-gce-c2
|
||||
required: true
|
||||
type: string
|
||||
break-glass:
|
||||
description: >-
|
||||
Only for a dead or unreachable reserve cell, with set admitMode=default. Typed as
|
||||
BREAK GLASS <cell-id>; records db without the cell's read-back
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
confirm-reserve:
|
||||
description: Required for admitMode=reserve, typed as RESERVE <cell-id>
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
set:
|
||||
description: >-
|
||||
Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a
|
||||
key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck,
|
||||
admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs,
|
||||
reregisterInFlight, phonePath (memory only takes effect in reserve; a reserve flip
|
||||
must name phonePath)
|
||||
required: true
|
||||
type: string
|
||||
expected-generation:
|
||||
description: Exact current object generation from a dry run, or 0 if none exists
|
||||
required: true
|
||||
type: string
|
||||
confirmation:
|
||||
description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# Not relay-staging-mutation: a switch write must run while a staging deploy or load run is
|
||||
# in flight, and it touches neither Cloud SQL nor a revision.
|
||||
concurrency:
|
||||
group: relay-staging-cell-flags
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
# Explicit bash runs with pipefail, so the script's exit code survives the tee.
|
||||
shell: bash
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
operate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
|
||||
timeout-minutes: 60
|
||||
environment: staging
|
||||
env:
|
||||
MODE: ${{ inputs.mode }}
|
||||
CELL_ID: ${{ inputs.cell-id }}
|
||||
FLAG_CHANGES: ${{ inputs.set }}
|
||||
EXPECTED_GENERATION: ${{ inputs.expected-generation }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
CONFIRM_RESERVE: ${{ inputs.confirm-reserve }}
|
||||
BREAK_GLASS: ${{ inputs.break-glass }}
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- name: Write or preview the cell's switch object
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
[[ "${CELL_ID}" =~ ^staging-gce-c[0-9]{1,3}$ ]]
|
||||
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
|
||||
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
|
||||
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
|
||||
node dev/scripts/operate-relay-cell-flags.mjs \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" \
|
||||
--cell-origin "https://${CELL_ID#staging-gce-}.relay-staging.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" \
|
||||
--set "${FLAG_CHANGES}" \
|
||||
--director-origin "https://relay-staging.onorca.dev" --confirm-reserve "${CONFIRM_RESERVE}" \
|
||||
--break-glass "${BREAK_GLASS}" \
|
||||
--expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-cell-flags.jsonl"
|
||||
|
||||
- name: Audit summary
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
{
|
||||
echo "### Relay staging cell flags: ${MODE} ${CELL_ID}"
|
||||
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
|
||||
echo '```json'
|
||||
cat "${RUNNER_TEMP}/relay-cell-flags.jsonl" 2>/dev/null || echo '(no output)'
|
||||
echo '```'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,104 +0,0 @@
|
||||
name: Power Relay Staging
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# A zero-activity guard makes this a no-op when an internal test is still running.
|
||||
- cron: '0 9 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Inspect, wake, or sleep the staging Relay data plane
|
||||
required: true
|
||||
default: status
|
||||
type: choice
|
||||
options:
|
||||
- status
|
||||
- wake
|
||||
- sleep
|
||||
wake-cells:
|
||||
description: Wake configured admission cells, or include disabled candidate cells
|
||||
required: true
|
||||
default: configured
|
||||
type: choice
|
||||
options:
|
||||
- configured
|
||||
- all
|
||||
confirmation:
|
||||
description: Enter WAKE_STAGING or SLEEP_STAGING for a manual mutation
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
power:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: staging
|
||||
env:
|
||||
POWER_MODE: ${{ github.event_name == 'schedule' && 'sleep' || inputs.mode }}
|
||||
WAKE_CELLS: ${{ inputs.wake-cells || 'configured' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Require explicit manual mutation confirmation
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && inputs.mode != 'status' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
if [[ "${POWER_MODE}" = "wake" ]]; then
|
||||
test "${CONFIRMATION}" = "WAKE_STAGING"
|
||||
else
|
||||
test "${CONFIRMATION}" = "SLEEP_STAGING"
|
||||
fi
|
||||
|
||||
- name: Read reviewed staging topology
|
||||
run: |
|
||||
node dev/scripts/infra.mjs init --env staging
|
||||
terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
|
||||
- name: Inspect or change staging power state
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/power-staging-relay.mjs \
|
||||
--mode "${POWER_MODE}" \
|
||||
--wake-cells "${WAKE_CELLS}" \
|
||||
--topology-file "${RUNNER_TEMP}/relay-gce-topology.json"
|
||||
@@ -1,500 +0,0 @@
|
||||
name: Prove Relay Asia Staging
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: prove runs the fixed C4 proof; load runs the generator with free-form arguments
|
||||
required: true
|
||||
default: prove
|
||||
type: choice
|
||||
options: [prove, load]
|
||||
image-digest:
|
||||
description: Prove only. Exact immutable Relay digest deployed on staging C4
|
||||
required: false
|
||||
type: string
|
||||
selector-generation:
|
||||
description: Prove only. Exact selector generation with C4 migration-only
|
||||
required: false
|
||||
type: string
|
||||
promote-attempt-id:
|
||||
description: Prove only. Durable unique C4 promotion attempt ID
|
||||
required: false
|
||||
type: string
|
||||
rollback-attempt-id:
|
||||
description: Prove only. Durable unique C4 rollback attempt ID
|
||||
required: false
|
||||
type: string
|
||||
load-region:
|
||||
description: >-
|
||||
Load only. asia-east2 runs on the Asia load runner; asia-east2-hosted asks for asia-east2
|
||||
from a hosted runner; us-central1 runs on a hosted runner
|
||||
required: false
|
||||
default: asia-east2
|
||||
type: choice
|
||||
options: [asia-east2, asia-east2-hosted, us-central1]
|
||||
load-shards:
|
||||
description: Load only. Generator processes, each with its own shard of the 32 load principals
|
||||
required: false
|
||||
default: '4'
|
||||
type: choice
|
||||
options: ['1', '2', '3', '4']
|
||||
load-runners:
|
||||
description: >-
|
||||
Load only, hosted runners only. Runner machines, each with load-shards generators; one
|
||||
hosted runner holds about 2,000 connections to one cell
|
||||
required: false
|
||||
default: '1'
|
||||
type: choice
|
||||
options: ['1', '2']
|
||||
load-args:
|
||||
description: >-
|
||||
Load only. Per-shard generator arguments, e.g. --controls 150 --splices 15
|
||||
--reconnect-max-seconds 30 --duration-seconds 900. Origins, region, principals and
|
||||
sharding are set by the workflow
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
confirmation:
|
||||
description: Enter PROVE_ASIA_STAGING (prove) or LOAD_RELAY_STAGING (load)
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# A load run drives traffic only, so it must not queue behind (or hold up) the staging deploys
|
||||
# and switch writes it is there to exercise.
|
||||
concurrency:
|
||||
group: ${{ inputs.mode == 'load' && 'relay-staging-load' || 'relay-staging-mutation' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
prove:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') && inputs.mode == 'prove' }}
|
||||
runs-on: [self-hosted, linux, x64, relay-asia-east2-load]
|
||||
timeout-minutes: 75
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
AUTH_ORIGIN: https://auth-staging.onorca.dev
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
|
||||
PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }}
|
||||
ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Validate the exact staging proof request
|
||||
shell: bash
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "${CONFIRMATION}" = PROVE_ASIA_STAGING
|
||||
[[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
[[ "${INITIAL_SELECTOR_GENERATION}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${PROMOTE_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
[[ "${ROLLBACK_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
|
||||
test "${PROMOTE_ATTEMPT_ID}" != "${ROLLBACK_ATTEMPT_ID}"
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- name: Require the exact staging director image before promotion
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
runtime="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data '{"v":1}')"
|
||||
test "$(jq -r '.role' <<< "${runtime}")" = director
|
||||
test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}"
|
||||
|
||||
- name: Install exact load-harness dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build the Relay load-harness contract
|
||||
run: pnpm --filter @orca-cloud/relay-contract build
|
||||
|
||||
- name: Promote only staging C4
|
||||
id: promote
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging \
|
||||
--mode promote \
|
||||
--cell-ids staging-gce-c4 \
|
||||
--expected-generation "${INITIAL_SELECTOR_GENERATION}" \
|
||||
--attempt-id "${PROMOTE_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
generation="$(jq -er '.generation' <<< "${result}")"
|
||||
test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = general
|
||||
echo "generation=${generation}" >> "${GITHUB_OUTPUT}"
|
||||
echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Run sharded two-horizon and mixed splice proofs
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof"
|
||||
mkdir -p "${proof_dir}"
|
||||
fd_limit="$(ulimit -n)"
|
||||
if test "${fd_limit}" != unlimited; then
|
||||
[[ "${fd_limit}" =~ ^[0-9]+$ ]]
|
||||
test "${fd_limit}" -ge 4096
|
||||
fi
|
||||
run_phase() {
|
||||
phase="$1"
|
||||
controls="$2"
|
||||
splices="$3"
|
||||
pids=()
|
||||
stop_shards() {
|
||||
for pid in "${pids[@]}"; do kill "${pid}" 2>/dev/null || true; done
|
||||
for pid in "${pids[@]}"; do wait "${pid}" 2>/dev/null || true; done
|
||||
}
|
||||
trap stop_shards EXIT
|
||||
for shard in 0 1 2 3; do
|
||||
slow=0
|
||||
wedged=0
|
||||
boundary_args=()
|
||||
request_unit_args=()
|
||||
if test "${phase}" = launch && test "${shard}" = 0; then
|
||||
slow=4
|
||||
wedged=1
|
||||
fi
|
||||
if test "${phase}" = launch && test "${shard}" = 0; then
|
||||
boundary_args=(
|
||||
--region-behavior-probes 1
|
||||
--capacity-cell-id staging-gce-c4
|
||||
--capacity-cell-origin https://c4.relay-staging.onorca.dev
|
||||
--capacity-unobserved-bound 60
|
||||
--rebind-probes 2
|
||||
--skip-rebind-overflow-check
|
||||
)
|
||||
fi
|
||||
node dev/scripts/load-relay-controls.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--auth-origin "${AUTH_ORIGIN}" \
|
||||
--preferred-region asia-east2 \
|
||||
--relay-asia-load-principals 32 \
|
||||
--controls "${controls}" \
|
||||
--splices "${splices}" \
|
||||
--slow-reader-splices "${slow}" \
|
||||
--wedged-reader-splices "${wedged}" \
|
||||
--capacity-hard-cap 3000 \
|
||||
"${boundary_args[@]}" \
|
||||
"${request_unit_args[@]}" \
|
||||
--phase-barrier-dir "${proof_dir}/${phase}-barrier" \
|
||||
--aggregate-controls "$((controls * 4))" \
|
||||
--aggregate-splices "$((splices * 4))" \
|
||||
--aggregate-reader-splices "$([[ "${phase}" = launch ]] && echo 5 || echo 0)" \
|
||||
--aggregate-reader-bytes "$([[ "${phase}" = launch ]] && echo 12582912 || echo 0)" \
|
||||
--required-lease-horizons 2 \
|
||||
--splice-ramp-seconds 120 \
|
||||
--max-generator-rss-growth-mib 512 \
|
||||
--ramp-seconds 180 \
|
||||
--duration-seconds 210 \
|
||||
--shard-count 4 \
|
||||
--shard-index "${shard}" \
|
||||
> "${proof_dir}/${phase}-${shard}.jsonl" &
|
||||
pids+=("$!")
|
||||
done
|
||||
failed=0
|
||||
for pid in "${pids[@]}"; do
|
||||
if ! wait "${pid}"; then failed=1; break; fi
|
||||
done
|
||||
if test "${failed}" = 1; then
|
||||
stop_shards
|
||||
for shard in 0 1 2 3; do
|
||||
jq -cer 'select(.event == "relay_load_progress" or .event == "relay_load_complete") |
|
||||
{event, shardIndex, active, peakActive, connected, connectionFailures,
|
||||
rampConnectionFailures, connectionFailuresByReason, unexpectedCloses,
|
||||
protocolErrors, refreshErrors, socketErrors, elapsedSeconds}' \
|
||||
"${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1 || true
|
||||
done
|
||||
trap - EXIT
|
||||
return 1
|
||||
fi
|
||||
trap - EXIT
|
||||
for shard in 0 1 2 3; do
|
||||
jq -cer 'select(.event == "relay_load_complete")' \
|
||||
"${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1
|
||||
done | jq -s . > "${proof_dir}/${phase}.json"
|
||||
}
|
||||
run_phase launch 5 5
|
||||
|
||||
- name: Collect and validate aggregate staging proof evidence
|
||||
env:
|
||||
PROOF_STARTED_AT: ${{ steps.promote.outputs.started_at }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof"
|
||||
ended_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
sleep 60
|
||||
gcloud logging read \
|
||||
"timestamp>=\"${PROOF_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \
|
||||
--project "${GCP_PROJECT_ID}" --limit 20000 --format json \
|
||||
> "${proof_dir}/runtime-metrics.json"
|
||||
node dev/scripts/relay-asia-rollout-evidence.mjs create-staging \
|
||||
--repository "${GITHUB_REPOSITORY}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--run-attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit-sha "${GITHUB_SHA}" \
|
||||
--image-digest "${IMAGE_DIGEST}" \
|
||||
--selector-generation "${{ steps.promote.outputs.generation }}" \
|
||||
--started-at "${PROOF_STARTED_AT}" \
|
||||
--ended-at "${ended_at}" \
|
||||
--launch-report "${proof_dir}/launch.json" \
|
||||
--logs-json "${proof_dir}/runtime-metrics.json" \
|
||||
--output "${proof_dir}/evidence.json"
|
||||
|
||||
- name: Return staging C4 to migration-only
|
||||
if: ${{ always() && steps.promote.outcome != 'skipped' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging \
|
||||
--mode recover-promotion \
|
||||
--cell-ids staging-gce-c4 \
|
||||
--expected-generation "${INITIAL_SELECTOR_GENERATION}" \
|
||||
--attempt-id "${PROMOTE_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
|
||||
promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging \
|
||||
--mode rollback \
|
||||
--cell-ids staging-gce-c4 \
|
||||
--expected-generation "${promoted_generation}" \
|
||||
--attempt-id "${ROLLBACK_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only
|
||||
|
||||
- name: Upload immutable staging readiness evidence
|
||||
if: ${{ success() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-asia-staging-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/relay-asia-staging-proof/evidence.json
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
recover:
|
||||
if: ${{ always() && github.ref == 'refs/heads/main' && inputs.mode == 'prove' }}
|
||||
needs: prove
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 15
|
||||
environment: staging
|
||||
env:
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
|
||||
PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }}
|
||||
ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Recover staging C4 with a fresh identity
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging \
|
||||
--mode recover-promotion \
|
||||
--cell-ids staging-gce-c4 \
|
||||
--expected-generation "${INITIAL_SELECTOR_GENERATION}" \
|
||||
--attempt-id "${PROMOTE_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
|
||||
promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging \
|
||||
--mode rollback \
|
||||
--cell-ids staging-gce-c4 \
|
||||
--expected-generation "${promoted_generation}" \
|
||||
--attempt-id "${ROLLBACK_ATTEMPT_ID}" \
|
||||
--image-digest "${IMAGE_DIGEST}")"
|
||||
test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only
|
||||
|
||||
# The staging auth service mints relay load principals only for this workflow file, so the
|
||||
# free-form staging load lives here rather than in a workflow of its own.
|
||||
load:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') && inputs.mode == 'load' }}
|
||||
runs-on: ${{ inputs.load-region == 'asia-east2' && fromJSON('["self-hosted","linux","x64","relay-asia-east2-load"]') || 'ubuntu-latest' }}
|
||||
strategy:
|
||||
# One runner's failure must not drop the other half of the load mid-run.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: ${{ fromJSON(inputs.load-runners == '2' && '[0, 1]' || '[0]') }}
|
||||
timeout-minutes: 180
|
||||
environment: staging
|
||||
env:
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
AUTH_ORIGIN: https://auth-staging.onorca.dev
|
||||
LOAD_REGION: ${{ inputs.load-region }}
|
||||
LOAD_SHARDS: ${{ inputs.load-shards }}
|
||||
LOAD_RUNNERS: ${{ inputs.load-runners }}
|
||||
LOAD_RUNNER_INDEX: ${{ matrix.runner }}
|
||||
LOAD_ARGS: ${{ inputs.load-args }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Validate the staging load request
|
||||
shell: bash
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "${CONFIRMATION}" = LOAD_RELAY_STAGING
|
||||
[[ "${LOAD_REGION}" =~ ^(asia-east2|asia-east2-hosted|us-central1)$ ]]
|
||||
[[ "${LOAD_SHARDS}" =~ ^[1-4]$ ]]
|
||||
[[ "${LOAD_RUNNERS}" =~ ^[12]$ ]]
|
||||
# The Asia runner is one machine; a second matrix job would only queue behind the first.
|
||||
if test "${LOAD_RUNNERS}" != 1 && test "${LOAD_REGION}" = asia-east2; then
|
||||
echo "load-runners 2 needs a hosted runner" >&2
|
||||
exit 1
|
||||
fi
|
||||
read -r -a args <<< "${LOAD_ARGS}"
|
||||
for arg in "${args[@]}"; do
|
||||
[[ "${arg}" =~ ^[A-Za-z0-9=:/._,-]+$ ]] || { echo "unsupported characters in load-args" >&2; exit 1; }
|
||||
case "${arg}" in
|
||||
--target-origin|--director-origin|--auth-origin|--signing-key-file|--preferred-region|\
|
||||
--relay-asia-load-principals|--shard-count|--shard-index|--phase-barrier-dir)
|
||||
echo "${arg} is set by the workflow" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- name: Install exact load-harness dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build the Relay load-harness contract
|
||||
run: pnpm --filter @orca-cloud/relay-contract build
|
||||
|
||||
- name: Run the sharded staging load
|
||||
env:
|
||||
# Probes that read director capacity need an admin token; it lasts about an hour.
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
load_dir="${RUNNER_TEMP}/relay-staging-load"
|
||||
mkdir -p "${load_dir}"
|
||||
fd_limit="$(ulimit -n)"
|
||||
if test "${fd_limit}" != unlimited; then
|
||||
[[ "${fd_limit}" =~ ^[0-9]+$ ]]
|
||||
test "${fd_limit}" -ge 4096
|
||||
fi
|
||||
read -r -a args <<< "${LOAD_ARGS}"
|
||||
pids=()
|
||||
stop_shards() {
|
||||
for pid in "${pids[@]}"; do kill "${pid}" 2>/dev/null || true; done
|
||||
for pid in "${pids[@]}"; do wait "${pid}" 2>/dev/null || true; done
|
||||
}
|
||||
trap stop_shards EXIT
|
||||
for ((shard = 0; shard < LOAD_SHARDS; shard++)); do
|
||||
node dev/scripts/load-relay-controls.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--auth-origin "${AUTH_ORIGIN}" \
|
||||
--preferred-region "${LOAD_REGION%-hosted}" \
|
||||
--relay-asia-load-principals 32 \
|
||||
--shard-count "$((LOAD_SHARDS * LOAD_RUNNERS))" \
|
||||
--shard-index "$((LOAD_RUNNER_INDEX * LOAD_SHARDS + shard))" \
|
||||
"${args[@]}" \
|
||||
> "${load_dir}/shard-${shard}.jsonl" &
|
||||
pids+=("$!")
|
||||
done
|
||||
failed=0
|
||||
for pid in "${pids[@]}"; do
|
||||
if ! wait "${pid}"; then failed=1; fi
|
||||
done
|
||||
trap - EXIT
|
||||
for ((shard = 0; shard < LOAD_SHARDS; shard++)); do
|
||||
jq -c 'select(.event == "relay_load_complete" or .event == "relay_load_progress")' \
|
||||
"${load_dir}/shard-${shard}.jsonl" | tail -n 1 || true
|
||||
done
|
||||
exit "${failed}"
|
||||
|
||||
- name: Upload the load reports
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-staging-load-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.runner }}
|
||||
path: ${{ runner.temp }}/relay-staging-load/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
@@ -1,928 +0,0 @@
|
||||
name: Prove Relay Staging Capacity
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Verify or change C3 capacity, restore admission, or refresh empty Asia C4
|
||||
required: true
|
||||
default: verify
|
||||
type: choice
|
||||
options:
|
||||
- verify
|
||||
- apply
|
||||
- restore-admission
|
||||
- refresh-asia-c4-image
|
||||
expected-hard-cap:
|
||||
description: Exact cap declared for staging-gce-c3 in the reviewed staging tfvars
|
||||
required: true
|
||||
default: '600'
|
||||
type: choice
|
||||
options:
|
||||
- '1000'
|
||||
- '600'
|
||||
expected-unobserved-bound:
|
||||
description: Exact bound declared for staging-gce-c3 in the reviewed staging tfvars
|
||||
required: true
|
||||
default: '60'
|
||||
type: choice
|
||||
options:
|
||||
- '60'
|
||||
- '0'
|
||||
confirmation:
|
||||
description: Exact confirmation required for a mutation
|
||||
required: false
|
||||
type: string
|
||||
expected-selector-generation:
|
||||
description: Exact staging selector generation for a C4 image refresh
|
||||
required: false
|
||||
type: string
|
||||
predecessor-image-digest:
|
||||
description: Exact current C4 sha256 digest
|
||||
required: false
|
||||
type: string
|
||||
target-image-digest:
|
||||
description: Exact desired C4 sha256 digest
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
capacity:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && inputs.mode != 'refresh-asia-c4-image' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
|
||||
DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
CELL_ORIGIN: https://c3.relay-staging.onorca.dev
|
||||
# These drains are unpaced, but restart-safe still waits out the production pace window
|
||||
# from the drain, the same one the same-cap drain uses.
|
||||
RESTART_SAFE_PACE_WINDOW_MS: '300000'
|
||||
TARGET_CELL_ID: staging-gce-c3
|
||||
FALLBACK_CELL_ORIGIN: https://c2.relay-staging.onorca.dev
|
||||
FALLBACK_CELL_ID: staging-gce-c2
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
EXPECTED_HARD_CAP: ${{ inputs.expected-hard-cap }}
|
||||
EXPECTED_UNOBSERVED_BOUND: ${{ inputs.expected-unobserved-bound }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
if: ${{ inputs.mode != 'restore-admission' }}
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Initialize the exact staging backend
|
||||
if: ${{ inputs.mode != 'restore-admission' }}
|
||||
run: node dev/scripts/infra.mjs init --env staging
|
||||
|
||||
- name: Require reviewed desired capacity and image
|
||||
if: ${{ inputs.mode != 'restore-admission' }}
|
||||
shell: bash
|
||||
run: |
|
||||
CAP_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_hard_cap"
|
||||
BOUND_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_unobserved_bound"
|
||||
IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image"
|
||||
ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone"
|
||||
DESIRED_CAP="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars <<< "${CAP_EXPRESSION}")"
|
||||
DESIRED_BOUND="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars <<< "${BOUND_EXPRESSION}")"
|
||||
DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars <<< "${IMAGE_EXPRESSION}" | jq -r '.')"
|
||||
TARGET_ZONE="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars <<< "${ZONE_EXPRESSION}" | jq -r '.')"
|
||||
MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
|
||||
| jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
|
||||
DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'local.relay_director_cells_json' | jq -r '.')"
|
||||
CELL_ORIGIN="$(jq -r --arg cell "${TARGET_CELL_ID}" \
|
||||
'.[] | select(.id == $cell) | .url' <<< "${DESIRED_CELLS_JSON}")"
|
||||
test "${DESIRED_CAP}" = "${EXPECTED_HARD_CAP}"
|
||||
test "${DESIRED_BOUND}" = "${EXPECTED_UNOBSERVED_BOUND}"
|
||||
[[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]]
|
||||
[[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]]
|
||||
[[ "${MIG_NAME}" =~ ^[a-z0-9-]+$ ]]
|
||||
test "${CELL_ORIGIN}" = "https://c3.relay-staging.onorca.dev"
|
||||
jq -e \
|
||||
--arg cell "${TARGET_CELL_ID}" \
|
||||
--arg fallback "${FALLBACK_CELL_ID}" \
|
||||
--argjson cap "${EXPECTED_HARD_CAP}" \
|
||||
--argjson bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
'(any(.[]; .id == $cell and .connectionHardCap == $cap and .connectionUnobservedBound == $bound)) and
|
||||
(any(.[]; .id == $fallback and .connectionHardCap == 600 and .connectionUnobservedBound == 60))' \
|
||||
<<< "${DESIRED_CELLS_JSON}" >/dev/null
|
||||
echo "DESIRED_IMAGE=${DESIRED_IMAGE}" >> "${GITHUB_ENV}"
|
||||
echo "TARGET_ZONE=${TARGET_ZONE}" >> "${GITHUB_ENV}"
|
||||
echo "MIG_NAME=${MIG_NAME}" >> "${GITHUB_ENV}"
|
||||
echo "CELL_ORIGIN=${CELL_ORIGIN}" >> "${GITHUB_ENV}"
|
||||
echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify exact compatible director image
|
||||
if: ${{ inputs.mode != 'restore-admission' }}
|
||||
shell: bash
|
||||
run: |
|
||||
SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
ACTIVE_REVISION="$(jq -r \
|
||||
'[.status.traffic[] | select((.percent // 0) > 0)]
|
||||
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \
|
||||
<<< "${SERVICE_JSON}")"
|
||||
test -n "${ACTIVE_REVISION}"
|
||||
ACTIVE_IMAGE="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format='value(spec.containers[0].image)')"
|
||||
test "${ACTIVE_IMAGE}" = "${DESIRED_IMAGE}"
|
||||
echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Require explicit transition confirmation
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: test "${CONFIRMATION}" = "FENCE_AND_TRANSITION_STAGING_C3"
|
||||
|
||||
- name: Require explicit admission restore confirmation
|
||||
if: ${{ inputs.mode == 'restore-admission' }}
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: test "${CONFIRMATION}" = "RESTORE_STAGING_C2_C3_GENERAL"
|
||||
|
||||
- name: Require capacity identity and exact predecessor state
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "${EXPECTED_HARD_CAP}/${EXPECTED_UNOBSERVED_BOUND}" in
|
||||
1000/0)
|
||||
PREDECESSOR_C3_CAP=600
|
||||
PREDECESSOR_C3_BOUND=60
|
||||
;;
|
||||
1000/60)
|
||||
PREDECESSOR_C3_CAP=1000
|
||||
PREDECESSOR_C3_BOUND=0
|
||||
;;
|
||||
600/60)
|
||||
PREDECESSOR_C3_CAP=1000
|
||||
PREDECESSOR_C3_BOUND=60
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported staging capacity transition" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
ACTIVE_REVISION="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '
|
||||
[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test -n "${ACTIVE_REVISION}"
|
||||
CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -cer '
|
||||
[.spec.containers[0].env[]? |
|
||||
select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
|
||||
if length == 1 then .[0] | fromjson else error("missing director topology") end')"
|
||||
PREDECESSOR_CELLS_JSON="$(jq -ce \
|
||||
--arg cell "${TARGET_CELL_ID}" \
|
||||
--argjson cap "${PREDECESSOR_C3_CAP}" \
|
||||
--argjson bound "${PREDECESSOR_C3_BOUND}" \
|
||||
'map(if .id == $cell then . + {
|
||||
connectionHardCap: $cap,
|
||||
connectionUnobservedBound: $bound
|
||||
} else . end)' <<< "${DESIRED_CELLS_JSON}")"
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${FALLBACK_CELL_ORIGIN}" \
|
||||
--cell-id "${FALLBACK_CELL_ID}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission either \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
if jq -ne \
|
||||
--argjson current "${CURRENT_CELLS_JSON}" \
|
||||
--argjson expected "${DESIRED_CELLS_JSON}" \
|
||||
'$current == $expected'; then
|
||||
if node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed; then
|
||||
TRANSITION_PHASE=cell-active
|
||||
elif node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining forbidden \
|
||||
--activity allowed; then
|
||||
TRANSITION_PHASE=cell-ready
|
||||
else
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
|
||||
TRANSITION_PHASE=director-ready
|
||||
fi
|
||||
else
|
||||
jq -ne \
|
||||
--argjson current "${CURRENT_CELLS_JSON}" \
|
||||
--argjson expected "${PREDECESSOR_CELLS_JSON}" \
|
||||
'$current == $expected'
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${PREDECESSOR_C3_CAP}" \
|
||||
--unobserved-bound "${PREDECESSOR_C3_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission either \
|
||||
--draining either \
|
||||
--activity allowed
|
||||
TRANSITION_PHASE=predecessor
|
||||
fi
|
||||
echo "TRANSITION_PHASE=${TRANSITION_PHASE}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Restore C2 as the safe placement fallback
|
||||
if: ${{ inputs.mode == 'restore-admission' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${FALLBACK_CELL_ORIGIN}" \
|
||||
--cell-id "${FALLBACK_CELL_ID}" \
|
||||
--heartbeat fresh \
|
||||
--admission either \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode restore-fallback \
|
||||
--general-cell-ids "${FALLBACK_CELL_ID}"
|
||||
|
||||
- name: Require a healthy non-draining C3 before restoring it
|
||||
if: ${{ inputs.mode == 'restore-admission' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission either \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Require a healthy general C2 before isolating C3
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
if test "${TRANSITION_PHASE}" = cell-active; then
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode restore-fallback \
|
||||
--general-cell-ids "${FALLBACK_CELL_ID}"
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${FALLBACK_CELL_ORIGIN}" \
|
||||
--cell-id "${FALLBACK_CELL_ID}" \
|
||||
--hard-cap 600 \
|
||||
--unobserved-bound 60 \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Reversibly isolate and drain C3
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
if test "${TRANSITION_PHASE}" = cell-ready; then
|
||||
exit 0
|
||||
fi
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode isolate
|
||||
|
||||
- name: Verify restart-safe migration-only target
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
if test "${TRANSITION_PHASE}" = cell-ready; then
|
||||
exit 0
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--heartbeat either \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
|
||||
|
||||
- name: Verify current capacity
|
||||
if: ${{ inputs.mode == 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Deploy reviewed director topology and remove pre-protocol revisions
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
run: |
|
||||
if test "${TRANSITION_PHASE}" != predecessor; then
|
||||
echo "DIRECTOR_CONFIG_CHANGED=false" >> "${GITHUB_ENV}"
|
||||
exit 0
|
||||
fi
|
||||
RELEASE_ID="capacity-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
|
||||
DEPLOY_RESULT="$(node dev/scripts/deploy-relay-blue-green.mjs \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--service "${DIRECTOR_SERVICE_NAME}" \
|
||||
--image "${ACTIVE_IMAGE}" \
|
||||
--role director \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--capacity-cell-id "${TARGET_CELL_ID}" \
|
||||
--director-cells-json "${DESIRED_CELLS_JSON}" \
|
||||
--prune-revisions true \
|
||||
--release-id "${RELEASE_ID}")"
|
||||
echo "${DEPLOY_RESULT}"
|
||||
DIRECTOR_CONFIG_CHANGED="$(jq -r '.topologyChanged' <<< "${DEPLOY_RESULT}")"
|
||||
[[ "${DIRECTOR_CONFIG_CHANGED}" =~ ^(true|false)$ ]]
|
||||
echo "DIRECTOR_CONFIG_CHANGED=${DIRECTOR_CONFIG_CHANGED}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Require fail-closed director transition
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
if test "${TRANSITION_PHASE}" = cell-ready; then
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
exit 0
|
||||
fi
|
||||
HEARTBEAT_EXPECTATION=either
|
||||
if test "${DIRECTOR_CONFIG_CHANGED}" = true; then
|
||||
HEARTBEAT_EXPECTATION=stale
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat "${HEARTBEAT_EXPECTATION}" \
|
||||
--admission migration-only \
|
||||
--draining required \
|
||||
--activity restart-safe \
|
||||
--pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
|
||||
|
||||
- name: Plan and apply only the exact empty cell
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
shell: bash
|
||||
run: |
|
||||
recreate_fixed_one_instance() {
|
||||
local instance
|
||||
instance="$(gcloud compute instance-groups managed list-instances \
|
||||
"${MIG_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" \
|
||||
--format=json \
|
||||
| jq -er '
|
||||
if length == 1 and .[0].instanceStatus == "RUNNING" and
|
||||
.[0].currentAction == "NONE"
|
||||
then .[0].instance | split("/") | last
|
||||
else error("capacity cell does not have one stable running instance") end')"
|
||||
gcloud compute instance-groups managed recreate-instances \
|
||||
"${MIG_NAME}" \
|
||||
--instances "${instance}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" \
|
||||
--quiet
|
||||
}
|
||||
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c3"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]' \
|
||||
-out="${RUNNER_TEMP}/relay-capacity-cell.tfplan"
|
||||
PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-capacity-cell.tfplan" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode cell \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--image "${DESIRED_IMAGE}")"
|
||||
echo "${PLAN_RESULT}"
|
||||
CELL_PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")"
|
||||
[[ "${CELL_PLAN_CHANGES}" =~ ^(0|2)$ ]]
|
||||
if test "${TRANSITION_PHASE}" = cell-ready; then
|
||||
test "${CELL_PLAN_CHANGES}" = 0
|
||||
elif test "${TRANSITION_PHASE}" = cell-active; then
|
||||
test "${CELL_PLAN_CHANGES}" = 0
|
||||
recreate_fixed_one_instance
|
||||
elif test "${CELL_PLAN_CHANGES}" = 0; then
|
||||
recreate_fixed_one_instance
|
||||
else
|
||||
terraform -chdir=infra/terraform apply \
|
||||
-auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan"
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until \
|
||||
"${MIG_NAME}" \
|
||||
--stable \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" \
|
||||
--timeout 900
|
||||
|
||||
- name: Verify exact live cap and fresh matching heartbeat
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission migration-only \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Make C3 the only staging placement cell
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode activate
|
||||
|
||||
- name: Verify the sole general canary after transition
|
||||
if: ${{ inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Restore the reviewed C2 and C3 placement set
|
||||
if: ${{ inputs.mode == 'restore-admission' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode restore \
|
||||
--general-cell-ids staging-gce-c2,staging-gce-c3
|
||||
|
||||
- name: Verify restored C3 admission and capacity
|
||||
if: ${{ inputs.mode == 'restore-admission' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh \
|
||||
--admission general \
|
||||
--draining forbidden \
|
||||
--activity allowed
|
||||
|
||||
- name: Preserve C2 as the safe fallback after a failed transition
|
||||
if: ${{ failure() && inputs.mode == 'apply' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" \
|
||||
--mode restore-fallback \
|
||||
--general-cell-ids "${FALLBACK_CELL_ID}"
|
||||
|
||||
refresh-asia-c4-image:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && github.ref == 'refs/heads/main' && inputs.mode == 'refresh-asia-c4-image' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 90
|
||||
environment: staging
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
CELL_ORIGIN: https://c4.relay-staging.onorca.dev
|
||||
TARGET_CELL_ID: staging-gce-c4
|
||||
EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
||||
PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }}
|
||||
TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }}
|
||||
APPROVED_PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Require the exact bounded C4 refresh
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "${CONFIRMATION}" = REFRESH_STAGING_ASIA_C4_IMAGE
|
||||
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
[[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test "${PREDECESSOR_IMAGE_DIGEST}" != "${TARGET_IMAGE_DIGEST}"
|
||||
test "${PREDECESSOR_IMAGE_DIGEST}" = "${APPROVED_PREDECESSOR_IMAGE_DIGEST}"
|
||||
|
||||
- id: google-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Initialize the exact staging backend
|
||||
run: node dev/scripts/infra.mjs init --env staging
|
||||
|
||||
- name: Resolve the reviewed C4 image and shape
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cells="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
|
||||
shape="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${cells}")"
|
||||
test "$(jq -r '.hostname' <<< "${shape}")" = c4
|
||||
test "$(jq -r '.region' <<< "${shape}")" = asia-east2
|
||||
test "$(jq -r '.zone' <<< "${shape}")" = asia-east2-a
|
||||
test "$(jq -r '.machine_type' <<< "${shape}")" = e2-standard-4
|
||||
test "$(jq -r '.capacity_requests' <<< "${shape}")" = 6000
|
||||
test "$(jq -r '.database_pool_max' <<< "${shape}")" = 10
|
||||
test "$(jq -r '.connection_hard_cap' <<< "${shape}")" = 3000
|
||||
test "$(jq -r '.connection_unobserved_bound' <<< "${shape}")" = 60
|
||||
test "$(jq -r '.initially_enabled' <<< "${shape}")" = false
|
||||
desired_image="$(jq -r '.image' <<< "${shape}")"
|
||||
test "${desired_image}" = \
|
||||
"us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${TARGET_IMAGE_DIGEST}"
|
||||
served_digest="$(gcloud artifacts docker images describe "${desired_image}" \
|
||||
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${served_digest}" = "${TARGET_IMAGE_DIGEST}"
|
||||
mig_name="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
|
||||
| jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
|
||||
test "${mig_name}" = orca-cloud-staging-relay-gce-c4
|
||||
{
|
||||
echo "DESIRED_IMAGE=${desired_image}"
|
||||
echo "ROLLBACK_IMAGE=us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${PREDECESSOR_IMAGE_DIGEST}"
|
||||
echo "TARGET_ZONE=asia-east2-a"
|
||||
echo "MIG_NAME=${mig_name}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Save, validate, and classify the exact C4 plan
|
||||
id: plan
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
plan="${RUNNER_TEMP}/relay-c4-image-refresh.tfplan"
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
|
||||
-out="${plan}"
|
||||
result="$(terraform -chdir=infra/terraform show -json "${plan}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
|
||||
--unobserved-bound 60 --image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${ROLLBACK_IMAGE}")"
|
||||
case "$(jq -r '[.changes,.changeKind] | join(":")' <<< "${result}")" in
|
||||
2:replacement) refresh_phase=predecessor ;;
|
||||
0:none|*:obsolete-template-delete) refresh_phase=applied ;;
|
||||
*:manager-convergence|*:replacement-with-obsolete-template) refresh_phase=converging ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
{
|
||||
echo "PLAN_CHANGES=$(jq -r '.changes' <<< "${result}")"
|
||||
echo "REFRESH_PHASE=${refresh_phase}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- id: state-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Verify the exact selector and current C4 state
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.state-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \
|
||||
--expected-generation '' --expected-membership-sha256 '' --attempt-id '' \
|
||||
--image-digest "${TARGET_IMAGE_DIGEST}")"
|
||||
test "$(jq -r '.generation' <<< "${inspect}")" = "${EXPECTED_SELECTOR_GENERATION}"
|
||||
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \
|
||||
migration-only
|
||||
expected_digests="${TARGET_IMAGE_DIGEST}"
|
||||
if test "${REFRESH_PHASE}" = predecessor; then
|
||||
expected_digests="${PREDECESSOR_IMAGE_DIGEST}"
|
||||
elif test "${REFRESH_PHASE}" = converging; then
|
||||
expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
|
||||
fi
|
||||
if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \
|
||||
"${CELL_ORIGIN}/v1/admin/runtime-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data '{"v":1}')"; then
|
||||
current_digest="$(jq -er '.imageDigest' <<< "${runtime}")"
|
||||
case ",${expected_digests}," in
|
||||
*,"${current_digest}",*) ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
source_incarnation=''
|
||||
draining=forbidden
|
||||
if test "${REFRESH_PHASE}" != applied; then
|
||||
status="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
|
||||
source_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")"
|
||||
[[ "${source_incarnation}" =~ ^[0-9a-f-]{36}$ ]]
|
||||
if test "$(jq -r '.draining' <<< "${runtime}")" = true; then
|
||||
draining=required
|
||||
fi
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
|
||||
--heartbeat fresh --admission migration-only --draining "${draining}" \
|
||||
--activity quiescent --expected-image-digests "${expected_digests}"
|
||||
runtime_available=true
|
||||
else
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \
|
||||
--admission migration-only --draining either --activity restart-safe \
|
||||
--timeout-ms 300000
|
||||
source_incarnation=''
|
||||
runtime_available=false
|
||||
fi
|
||||
{
|
||||
echo "MIG_STABLE_AT_MS=0"
|
||||
echo "MUTATION_STARTED=false"
|
||||
echo "REPLACEMENT_STARTED_AT_MS=0"
|
||||
echo "RUNTIME_AVAILABLE=${runtime_available}"
|
||||
echo "SOURCE_INCARNATION=${source_incarnation}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- id: fence-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Fence C4 and prove it stayed empty before replacement
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test "${REFRESH_PHASE}" = applied; then exit 0; fi
|
||||
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
if test "${RUNTIME_AVAILABLE}" = false; then exit 0; fi
|
||||
node dev/scripts/prepare-relay-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --mode isolate
|
||||
fence_digests="${PREDECESSOR_IMAGE_DIGEST}"
|
||||
if test "${REFRESH_PHASE}" = converging; then
|
||||
fence_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
|
||||
--heartbeat fresh --admission migration-only --draining required \
|
||||
--activity quiescent --expected-image-digests "${fence_digests}"
|
||||
|
||||
- name: Apply the exact saved C4 plan
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test "${PLAN_CHANGES}" = 0 && test "${RUNTIME_AVAILABLE}" = true; then exit 0; fi
|
||||
if test "${REFRESH_PHASE}" = applied && test "${RUNTIME_AVAILABLE}" = false; then
|
||||
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
if test "${REFRESH_PHASE}" != applied; then
|
||||
replacement_started_at_ms="$(date -u +%s%3N)"
|
||||
echo "REPLACEMENT_STARTED_AT_MS=${replacement_started_at_ms}" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
if test "${PLAN_CHANGES}" != 0; then
|
||||
terraform -chdir=infra/terraform apply -auto-approve \
|
||||
"${RUNNER_TEMP}/relay-c4-image-refresh.tfplan"
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
if test "${RUNTIME_AVAILABLE}" = false && test "${REFRESH_PHASE}" = applied; then
|
||||
instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \
|
||||
| jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and
|
||||
.[0].currentAction == "NONE" then .[0].instance | split("/") | last
|
||||
else error("C4 is not one stable running instance") end')"
|
||||
gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \
|
||||
--instances "${instance}" --project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" --quiet
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
fi
|
||||
echo "MIG_STABLE_AT_MS=$(date -u +%s%3N)" >> "${GITHUB_ENV}"
|
||||
|
||||
- id: post-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Verify new C4 incarnation, image, and unchanged isolation
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
|
||||
--heartbeat fresh --admission migration-only --draining forbidden \
|
||||
--activity quiescent --expected-image-digests "${TARGET_IMAGE_DIGEST}" \
|
||||
--timeout-ms 240000
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \
|
||||
--expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--expected-membership-sha256 '' --attempt-id '' \
|
||||
--image-digest "${TARGET_IMAGE_DIGEST}")"
|
||||
test "$(jq -r '.generation' <<< "${result}")" = "${EXPECTED_SELECTOR_GENERATION}"
|
||||
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \
|
||||
migration-only
|
||||
for _ in $(seq 1 36); do
|
||||
status="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
|
||||
if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \
|
||||
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
|
||||
-ge "${MIG_STABLE_AT_MS}"; then break; fi
|
||||
sleep 5
|
||||
done
|
||||
target_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")"
|
||||
test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true
|
||||
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
|
||||
-ge "${MIG_STABLE_AT_MS}"
|
||||
if test "${REFRESH_PHASE}" != applied; then
|
||||
if test -n "${SOURCE_INCARNATION}"; then
|
||||
test "${target_incarnation}" != "${SOURCE_INCARNATION}"
|
||||
fi
|
||||
test "$(jq -r '.status.runtime.startedAt' <<< "${status}")" \
|
||||
-ge "${REPLACEMENT_STARTED_AT_MS}"
|
||||
fi
|
||||
|
||||
- name: Require an empty targeted Terraform readback
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
plan="${RUNNER_TEMP}/relay-c4-image-readback.tfplan"
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
|
||||
-out="${plan}"
|
||||
result="$(terraform -chdir=infra/terraform show -json "${plan}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
|
||||
--unobserved-bound 60 --image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${ROLLBACK_IMAGE}")"
|
||||
test "$(jq -r '.changes' <<< "${result}")" = 0
|
||||
@@ -1,133 +0,0 @@
|
||||
name: Publish Relay Production Image
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: Publish a new production image or mirror an existing immutable image to staging
|
||||
required: true
|
||||
default: publish
|
||||
type: choice
|
||||
options: [publish, mirror-staging]
|
||||
image-digest:
|
||||
description: Exact existing production digest for mirror-staging mode
|
||||
required: false
|
||||
type: string
|
||||
confirmation:
|
||||
description: Enter MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING for mirror-staging mode
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: publish-relay-production
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
REPOSITORY_ID: orca-cloud
|
||||
IMAGE_NAME: relay
|
||||
PUBLISH_MODE: ${{ inputs.mode }}
|
||||
MIRROR_DIGEST: ${{ inputs.image-digest }}
|
||||
MIRROR_CONFIRMATION: ${{ inputs.confirmation }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Validate the exact publish request before authentication
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test "${PUBLISH_MODE}" = mirror-staging; then
|
||||
[[ "${MIRROR_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
test "${MIRROR_CONFIRMATION}" = MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING
|
||||
else
|
||||
test "${PUBLISH_MODE}" = publish
|
||||
test -z "${MIRROR_DIGEST}"
|
||||
test -z "${MIRROR_CONFIRMATION}"
|
||||
fi
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Configure Docker auth
|
||||
run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
|
||||
|
||||
- name: Build and publish immutable image
|
||||
if: ${{ inputs.mode == 'publish' }}
|
||||
run: |
|
||||
IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${GITHUB_SHA}"
|
||||
docker build -f apps/relay/Dockerfile -t "${IMAGE_TAG}" .
|
||||
docker push "${IMAGE_TAG}"
|
||||
DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')"
|
||||
test -n "${DIGEST}"
|
||||
IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${DIGEST}"
|
||||
{
|
||||
echo '### Terraform candidate image'
|
||||
echo
|
||||
echo "\`${IMAGE}\`"
|
||||
echo
|
||||
echo 'Declare this digest on a distinct disabled candidate cell in a reviewed Terraform PR.'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Build and publish immutable fence broker
|
||||
if: ${{ inputs.mode == 'publish' }}
|
||||
run: |
|
||||
IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker:sha-${GITHUB_SHA}"
|
||||
docker build \
|
||||
--build-arg "ORCA_RELAY_FENCE_IMAGE_COMMIT=${GITHUB_SHA}" \
|
||||
-f apps/relay-fence-broker/Dockerfile \
|
||||
-t "${IMAGE_TAG}" .
|
||||
docker push "${IMAGE_TAG}"
|
||||
DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')"
|
||||
test -n "${DIGEST}"
|
||||
IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker@${DIGEST}"
|
||||
{
|
||||
echo
|
||||
echo '### Terraform fence broker image'
|
||||
echo
|
||||
echo "\`${IMAGE}\`"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Mirror the exact production manifest to staging
|
||||
if: ${{ inputs.mode == 'mirror-staging' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
source_image="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${MIRROR_DIGEST}"
|
||||
target_tag="${GCP_REGION}-docker.pkg.dev/onorca-cloud-staging/${REPOSITORY_ID}/${IMAGE_NAME}:production-${MIRROR_DIGEST#sha256:}"
|
||||
source_digest="$(gcloud artifacts docker images describe "${source_image}" \
|
||||
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${source_digest}" = "${MIRROR_DIGEST}"
|
||||
docker pull "${source_image}"
|
||||
docker tag "${source_image}" "${target_tag}"
|
||||
docker push "${target_tag}"
|
||||
target_digest="$(gcloud artifacts docker images describe "${target_tag}" \
|
||||
--project onorca-cloud-staging --format='value(image_summary.digest)')"
|
||||
test "${target_digest}" = "${MIRROR_DIGEST}"
|
||||
{
|
||||
echo '### Mirrored Relay image'
|
||||
echo
|
||||
printf 'Production and staging now resolve the same immutable digest: %s.\n' \
|
||||
"${MIRROR_DIGEST}"
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -1,568 +0,0 @@
|
||||
name: Deploy Push Gateway Production
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_sha:
|
||||
description: Full reviewed commit SHA to build (feature may remain unmerged)
|
||||
required: true
|
||||
type: string
|
||||
confirmation:
|
||||
description: Enter DEPLOY_PUSH_GATEWAY to shift production traffic
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# Serialize push traffic changes independently of Relay and the shared database.
|
||||
concurrency:
|
||||
group: production-push-rollout
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: >-
|
||||
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
|
||||
github.ref == 'refs/heads/main' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
environment: production
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
SERVICE_NAME: orca-cloud-push
|
||||
REPOSITORY_ID: orca-cloud
|
||||
IMAGE_NAME: push
|
||||
PUSH_ORIGIN: https://push.onorca.dev
|
||||
PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud.iam.gserviceaccount.com
|
||||
# Scaling the serving revision must already hold, matching push_min_instances and
|
||||
# push_max_instances. Terraform owns both, and the candidate inherits them from the
|
||||
# service, so this deploy never passes a scaling flag: doing so would write a
|
||||
# Terraform-owned field that `lifecycle.ignore_changes` does not cover, and a later
|
||||
# `push_max_instances` raise would then be reverted by every deploy. These two values
|
||||
# are the expected shape, asserted before the candidate is created and again on the
|
||||
# candidate itself, so a deploy that would change the gateway's Cloud SQL draw fails.
|
||||
PUSH_MIN_INSTANCES: 1
|
||||
PUSH_MAX_INSTANCES: 2
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
SOURCE_SHA: ${{ inputs.source_sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Require the explicit deploy confirmation
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "${CONFIRMATION}" = DEPLOY_PUSH_GATEWAY
|
||||
[[ "${SOURCE_SHA}" =~ ^[a-f0-9]{40}$ ]]
|
||||
|
||||
# Keep the workflow and rollout lease on main; only the Docker build uses candidate code.
|
||||
- name: Fetch the immutable gateway source
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin "${SOURCE_SHA}"
|
||||
test "$(git rev-parse FETCH_HEAD)" = "${SOURCE_SHA}"
|
||||
mkdir -p "${RUNNER_TEMP}/push-source"
|
||||
git -C "${GITHUB_WORKSPACE}" archive "${SOURCE_SHA}" cloud \
|
||||
| tar -x -C "${RUNNER_TEMP}/push-source"
|
||||
|
||||
- uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Configure Docker auth
|
||||
run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
|
||||
|
||||
# Building an image does not need the deployment lease.
|
||||
- name: Build and publish the immutable gateway image
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
image_tag="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${SOURCE_SHA}"
|
||||
docker buildx build --push --platform linux/amd64 --provenance=false --metadata-file "${RUNNER_TEMP}/push-image.json" \
|
||||
-f "${RUNNER_TEMP}/push-source/cloud/apps/push/Dockerfile" \
|
||||
-t "${image_tag}" "${RUNNER_TEMP}/push-source/cloud"
|
||||
digest="$(jq -er '."containerimage.digest"' "${RUNNER_TEMP}/push-image.json")"
|
||||
[[ "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
echo "IMAGE=${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${digest}" \
|
||||
>> "${GITHUB_ENV}"
|
||||
echo "IMAGE_DIGEST=${digest}" >> "${GITHUB_ENV}"
|
||||
|
||||
# Refuse older images before they ever boot against production.
|
||||
- name: Require image support for inert validation
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker run --rm --network none --entrypoint node "${IMAGE}" --input-type=module -e '
|
||||
import { loadPushConfig } from "./apps/push/dist/config.js";
|
||||
const env = { ORCA_PUSH_PUBLIC_URL: "https://push.onorca.dev", ORCA_PUSH_MODE: "validation", ORCA_PUSH_FCM_PROJECT_ID: "onorca-cloud" };
|
||||
if (loadPushConfig(env).mode !== "validation") throw new Error("validation_mode_unsupported");
|
||||
let rejected = false;
|
||||
try { loadPushConfig({ ...env, ORCA_PUSH_MODE: "invalid" }); } catch { rejected = true; }
|
||||
if (!rejected) throw new Error("validation_mode_not_fail_closed");
|
||||
'
|
||||
|
||||
# Held across the deploy, not just a separate schema step: the gateway opens its pool and
|
||||
# applies its schema while the new revision starts, so the revision is the schema step.
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-terraform-state
|
||||
object: terraform/state/push-rollout/production.lock
|
||||
|
||||
# Why: the candidate inherits the serving revision's scaling. A serving revision that has
|
||||
# drifted below the floor would hand the candidate a cold start on every notification, and
|
||||
# one that has drifted above the ceiling would hand it a larger Cloud SQL draw than the
|
||||
# rollout lease was taken for. Refuse to inherit either rather than latch it.
|
||||
- name: Record the serving revision and require its Terraform-owned scaling
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
|
||||
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test -n "${serving}"
|
||||
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
|
||||
if test "${revisions}" != "${serving}"; then
|
||||
echo 'Retire leftover revisions under the rollout lease before deploying; three pools are the limit.' >&2
|
||||
exit 1
|
||||
fi
|
||||
floor="$(gcloud run revisions describe "${serving}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
|
||||
if [[ "${floor:-0}" -lt "${PUSH_MIN_INSTANCES}" ]]; then
|
||||
echo "serving revision ${serving} holds ${floor:-0} minimum instances," \
|
||||
"below ${PUSH_MIN_INSTANCES}; deploying would inherit and latch it." >&2
|
||||
echo "Restore the floor first: gcloud run services update ${SERVICE_NAME}" \
|
||||
"--region ${GCP_REGION} --min-instances=${PUSH_MIN_INSTANCES}" >&2
|
||||
exit 1
|
||||
fi
|
||||
ceiling="$(gcloud run revisions describe "${serving}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
|
||||
test "${ceiling}" = "${PUSH_MAX_INSTANCES}"
|
||||
echo "serving revision ${serving} holds ${floor} minimum and ${ceiling} maximum instances"
|
||||
echo "ROLLBACK_REVISION=${serving}" >> "${GITHUB_ENV}"
|
||||
gcloud run revisions describe "${serving}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
> "${RUNNER_TEMP}/push-rollback-revision.json"
|
||||
image="$(jq -er '.status.imageDigest' "${RUNNER_TEMP}/push-rollback-revision.json")"
|
||||
[[ "${image}" =~ @sha256:[a-f0-9]{64}$ ]]
|
||||
echo "ROLLBACK_IMAGE=${image}" >> "${GITHUB_ENV}"
|
||||
jq -e 'all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE" or .value == "active")' \
|
||||
"${RUNNER_TEMP}/push-rollback-revision.json" > /dev/null
|
||||
|
||||
# Validation has no schema writes, HTTP mutations, worker, or pruners; tags alone do not
|
||||
# isolate background consumers from production.
|
||||
- name: Deploy the candidate revision with no traffic
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag="c${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
{
|
||||
echo "VALIDATION_DEPLOY_ATTEMPTED=true"
|
||||
echo "VALIDATION_REVISION=${SERVICE_NAME}-${tag}"
|
||||
echo "VALIDATION_TAG=${tag}"
|
||||
echo "CANDIDATE_TAG=${tag}"
|
||||
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
gcloud run deploy "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--image "${IMAGE}" \
|
||||
--tag "${tag}" \
|
||||
--revision-suffix "${tag}" \
|
||||
--no-traffic \
|
||||
--update-env-vars ORCA_PUSH_MODE=validation \
|
||||
--quiet
|
||||
candidate="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -er --arg tag "${tag}" \
|
||||
'[.status.traffic[] | select(.tag == $tag)]
|
||||
| if length == 1 then .[0] else error("tagged candidate is not unique") end')"
|
||||
test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
|
||||
echo "CANDIDATE_URL=$(jq -r '.url' <<< "${candidate}")" >> "${GITHUB_ENV}"
|
||||
|
||||
# A tagged revision is directly addressable and sits outside the service-wide cap, so the
|
||||
# candidate and the serving revision each draw up to the ceiling during the probe window.
|
||||
# Successor creation later requires three revision pools; assert the inherited ceiling.
|
||||
- name: Require the candidate to serve the exact image and inherited scaling
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
served="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format='value(spec.containers[0].image)')"
|
||||
test "${served}" = "${IMAGE}"
|
||||
test "${CANDIDATE_REVISION}" != "${ROLLBACK_REVISION}"
|
||||
candidate_ceiling="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
|
||||
test "${candidate_ceiling}" = "${PUSH_MAX_INSTANCES}"
|
||||
|
||||
- name: Probe the candidate readiness endpoint
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${CANDIDATE_URL}" =~ ^https://[^/]+$ ]]
|
||||
for attempt in $(seq 1 30); do
|
||||
code="$(curl -sS -o "${RUNNER_TEMP}/push-ready.json" -w '%{http_code}' \
|
||||
--max-time 10 "${CANDIDATE_URL}/ready" || true)"
|
||||
if test "${code}" = 200; then
|
||||
jq -e . < "${RUNNER_TEMP}/push-ready.json" > /dev/null
|
||||
curl --fail --silent --show-error --max-time 10 "${CANDIDATE_URL}/health" \
|
||||
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "validation"' > /dev/null
|
||||
echo "candidate ${CANDIDATE_REVISION} is ready after ${attempt} attempt(s)"
|
||||
exit 0
|
||||
fi
|
||||
echo "attempt ${attempt}: /ready returned ${code}"
|
||||
sleep 5
|
||||
done
|
||||
echo "candidate ${CANDIDATE_REVISION} never reported ready" >&2
|
||||
exit 1
|
||||
|
||||
# Why: a gateway that boots and answers /ready can still be unable to send. This proves the
|
||||
# runtime account's FCM grant end to end without delivering anything: validate_only stops
|
||||
# Google before any push, and the deliberately invalid token means a healthy credential
|
||||
# answers INVALID_ARGUMENT. PERMISSION_DENIED is the failure this step exists to catch.
|
||||
#
|
||||
# Only the four verdicts below are conclusive. A 429, a 5xx, or a transport failure says
|
||||
# nothing about the credential, so it is retried rather than treated as either answer; a
|
||||
# denied credential still fails on the first attempt, without burning the retries.
|
||||
- name: Prove the runtime identity can reach FCM
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
token="$(gcloud auth print-access-token \
|
||||
--impersonate-service-account "${PUSH_RUNTIME_SERVICE_ACCOUNT}")"
|
||||
test -n "${token}"
|
||||
echo "::add-mask::${token}"
|
||||
body='{"validate_only":true,"message":{"token":"orca-push-deploy-probe-invalid-token","notification":{"title":"Orca","body":"deploy probe"}}}'
|
||||
for attempt in $(seq 1 5); do
|
||||
code="$(curl -sS -o "${RUNNER_TEMP}/push-fcm.json" -w '%{http_code}' --max-time 20 \
|
||||
-X POST "https://fcm.googleapis.com/v1/projects/${GCP_PROJECT_ID}/messages:send" \
|
||||
-H "Authorization: Bearer ${token}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
--data "${body}" || true)"
|
||||
status="$(jq -r '.error.status // empty' < "${RUNNER_TEMP}/push-fcm.json" || true)"
|
||||
echo "attempt ${attempt}: FCM validate-only send returned HTTP ${code} status ${status:-OK}"
|
||||
if test "${status}" = PERMISSION_DENIED || test "${status}" = INVALID_ARGUMENT ||
|
||||
test "${code}" = 401 || test "${code}" = 403; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
if test "${status}" = PERMISSION_DENIED || test "${code}" = 401 || test "${code}" = 403; then
|
||||
echo "the push runtime identity cannot send through FCM" >&2
|
||||
exit 1
|
||||
fi
|
||||
test "${status}" = INVALID_ARGUMENT
|
||||
|
||||
# Cloud Run requires a successor before the latest revision can be deleted.
|
||||
- name: Retire inert validation and activate the verified image
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag="a${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
{
|
||||
echo "CANDIDATE_TAG=${tag}"
|
||||
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
|
||||
echo "ACTIVATION_ATTEMPTED=true"
|
||||
} >> "${GITHUB_ENV}"
|
||||
# This is the production-effect boundary: schema, pruners and workers start here.
|
||||
gcloud run deploy "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--image "${IMAGE}" --tag "${tag}" --revision-suffix "${tag}" \
|
||||
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--remove-tags "${VALIDATION_TAG}" --quiet
|
||||
gcloud run revisions delete "${VALIDATION_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
||||
echo "VALIDATION_RETIRED=true" >> "${GITHUB_ENV}"
|
||||
revision="$(gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
||||
jq -e --arg image "${IMAGE}" --arg account "${PUSH_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--arg ceiling "${PUSH_MAX_INSTANCES}" --arg floor "${PUSH_MIN_INSTANCES}" \
|
||||
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" '
|
||||
def shape: del(.containers[0].image) |
|
||||
.containers[0].env = ((.containers[0].env // []) |
|
||||
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
|
||||
.spec.containers[0].image == $image and .spec.serviceAccountName == $account and
|
||||
all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
|
||||
(.spec | shape) == ($prior[0].spec | shape) and
|
||||
.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
|
||||
(.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber)' \
|
||||
<<< "${revision}" > /dev/null
|
||||
candidate="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -er --arg tag "${tag}" '[.status.traffic[] | select(.tag == $tag)]
|
||||
| if length == 1 then .[0] else error("active candidate is not unique") end')"
|
||||
test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
|
||||
url="$(jq -er '.url' <<< "${candidate}")"
|
||||
[[ "${url}" =~ ^https://[^/]+$ ]]
|
||||
curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
|
||||
curl --fail --silent --show-error --max-time 10 "${url}/health" \
|
||||
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
|
||||
|
||||
- name: Shift all traffic to the verified candidate
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "TRAFFIC_SHIFT_ATTEMPTED=true" >> "${GITHUB_ENV}"
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--to-revisions "${CANDIDATE_REVISION}=100" \
|
||||
--quiet
|
||||
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
|
||||
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test "${serving}" = "${CANDIDATE_REVISION}"
|
||||
echo "TRAFFIC_SHIFTED=true" >> "${GITHUB_ENV}"
|
||||
|
||||
# Why: the summary is written before the origin check, not after it. Once traffic has
|
||||
# moved, the rollback target is the single thing an operator needs, and a summary that only
|
||||
# appeared on success would be missing in exactly the run that needs it.
|
||||
- name: Publish the rollout summary
|
||||
if: ${{ always() && env.CANDIDATE_REVISION != '' && env.ROLLBACK_REVISION != '' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
{
|
||||
echo '### Push gateway rollout'
|
||||
echo
|
||||
echo "Source: ${SOURCE_SHA}"
|
||||
echo
|
||||
echo "Revision: \`${CANDIDATE_REVISION}\`"
|
||||
echo
|
||||
echo "Image: \`${IMAGE_DIGEST}\`"
|
||||
echo "Known-good image: \`${ROLLBACK_IMAGE}\`"
|
||||
echo
|
||||
echo "Recovery: deploy \`${ROLLBACK_IMAGE}\` as a new revision with" \
|
||||
"\`--remove-env-vars ORCA_PUSH_MODE --no-traffic --tag <unique-recovery-tag> --revision-suffix <unique-recovery-suffix>\`."
|
||||
echo 'Verify its exact digest, configuration, readiness and active mode, then promote and check the public origin.'
|
||||
echo 'Only then remove obsolete tags and delete rejected/previous revisions; never delete the latest revision.'
|
||||
echo 'The previous revision is retired after public checks; retain this immutable image for recovery under the rollout lease.'
|
||||
echo "Activation attempted: ${ACTIVATION_ATTEMPTED:-false}; traffic rollback cannot undo schema or deliveries."
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Verify the public origin after the shift
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for attempt in $(seq 1 30); do
|
||||
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \
|
||||
"${PUSH_ORIGIN}/ready" || true)"
|
||||
if test "${code}" = 200; then
|
||||
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
|
||||
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' > /dev/null
|
||||
echo "ROLLOUT_VERIFIED=true" >> "${GITHUB_ENV}"
|
||||
echo "${PUSH_ORIGIN} is ready after ${attempt} attempt(s)"
|
||||
exit 0
|
||||
fi
|
||||
echo "attempt ${attempt}: ${PUSH_ORIGIN}/ready returned ${code}"
|
||||
sleep 5
|
||||
done
|
||||
echo "${PUSH_ORIGIN} never reported ready after the shift" >&2
|
||||
exit 1
|
||||
|
||||
# Why: everything after the shift runs with production on the candidate. A failure there
|
||||
# is not a failure to deploy, it is a live gateway that has to go back, so the traffic move
|
||||
# is undone here rather than left to whoever reads the run.
|
||||
- name: Roll traffic back to the previous revision
|
||||
if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -n "${ROLLBACK_REVISION:-}"
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--to-revisions "${ROLLBACK_REVISION}=100" \
|
||||
--quiet
|
||||
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
|
||||
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test "${serving}" = "${ROLLBACK_REVISION}"
|
||||
echo "TRAFFIC_ROLLED_BACK=true" >> "${GITHUB_ENV}"
|
||||
{
|
||||
echo
|
||||
echo '### Push gateway rolled back'
|
||||
echo
|
||||
echo "Traffic returned to \`${ROLLBACK_REVISION}\`; the candidate" \
|
||||
"\`${CANDIDATE_REVISION}\` no longer serves HTTP; deletion below must stop its workers."
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
# Deleting a revision does not restore the service template that Terraform reconciles.
|
||||
- name: Restore the known-good service template
|
||||
if: ${{ (failure() || cancelled()) && env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "${TRAFFIC_SHIFT_ATTEMPTED:-false}" != true || test "${TRAFFIC_ROLLED_BACK:-false}" = true
|
||||
test -n "${ROLLBACK_IMAGE}"
|
||||
# A partial activation may leave validation plus active; free one slot before recovery.
|
||||
latest="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--format='value(status.latestCreatedRevisionName)')"
|
||||
test -n "${latest}"
|
||||
if test "${VALIDATION_RETIRED:-false}" != true && test "${latest}" != "${VALIDATION_REVISION}"; then
|
||||
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--filter "metadata.name=${VALIDATION_REVISION}" --format='value(metadata.name)')"
|
||||
if test -n "${existing}"; then
|
||||
test "${existing}" = "${VALIDATION_REVISION}"
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--remove-tags "${VALIDATION_TAG}" --quiet
|
||||
gcloud run revisions delete "${VALIDATION_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
||||
fi
|
||||
fi
|
||||
tag="r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
echo "RECOVERY_TAG=${tag}" >> "${GITHUB_ENV}"
|
||||
echo "TEMPLATE_RECOVERY_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
|
||||
echo "Template recovery attempted: ${SERVICE_NAME}-${tag}, image ${ROLLBACK_IMAGE}." \
|
||||
>> "${GITHUB_STEP_SUMMARY}"
|
||||
# Known-good schema/workers can run here even though HTTP stays on the old revision.
|
||||
gcloud run deploy "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--image "${ROLLBACK_IMAGE}" --revision-suffix "${tag}" --tag "${tag}" \
|
||||
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
|
||||
gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
> "${RUNNER_TEMP}/push-recovered-service.json"
|
||||
gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
> "${RUNNER_TEMP}/push-recovery-revision.json"
|
||||
jq -e --arg image "${ROLLBACK_IMAGE}" --arg serving "${ROLLBACK_REVISION}" \
|
||||
--arg floor "${PUSH_MIN_INSTANCES}" --arg ceiling "${PUSH_MAX_INSTANCES}" \
|
||||
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" \
|
||||
--slurpfile recovered "${RUNNER_TEMP}/push-recovery-revision.json" '
|
||||
def shape: del(.containers[0].image) |
|
||||
.containers[0].env = ((.containers[0].env // []) |
|
||||
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
|
||||
.spec.template.spec.containers[0].image == $image and
|
||||
all(.spec.template.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
|
||||
$recovered[0].spec.containers[0].image == $image and
|
||||
all($recovered[0].spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
|
||||
($recovered[0].spec | shape) == ($prior[0].spec | shape) and
|
||||
.spec.template.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
|
||||
(.spec.template.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber) and
|
||||
([.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
length == 1 and .[0].revisionName == $serving and .[0].percent == 100)
|
||||
' "${RUNNER_TEMP}/push-recovered-service.json" > /dev/null
|
||||
echo "TEMPLATE_RESTORED=true" >> "${GITHUB_ENV}"
|
||||
echo 'Known-good image and normal mode restored; previous revision still serves HTTP.' \
|
||||
>> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Promote and verify the known-good recovery revision
|
||||
if: ${{ always() && env.TEMPLATE_RESTORED == 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
url="$(jq -er --arg tag "${RECOVERY_TAG}" \
|
||||
'.status.traffic[] | select(.tag == $tag) | .url' "${RUNNER_TEMP}/push-recovered-service.json")"
|
||||
[[ "${url}" =~ ^https://[^/]+$ ]]
|
||||
curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
|
||||
curl --fail --silent --show-error --max-time 10 "${url}/health" \
|
||||
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--to-revisions "${TEMPLATE_RECOVERY_REVISION}=100" --quiet
|
||||
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
||||
| jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
|
||||
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
||||
test "${serving}" = "${TEMPLATE_RECOVERY_REVISION}"
|
||||
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/ready" | jq -e '.ok == true'
|
||||
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
|
||||
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
|
||||
echo "RECOVERY_VERIFIED=true" >> "${GITHUB_ENV}"
|
||||
echo "Recovery revision ${TEMPLATE_RECOVERY_REVISION} now serves the known-good image." \
|
||||
>> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Delete the rejected candidate revision
|
||||
if: ${{ always() && env.RECOVERY_VERIFIED == 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test -z "${CANDIDATE_REVISION:-}"; then
|
||||
echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
|
||||
exit 0
|
||||
fi
|
||||
if test -n "${CANDIDATE_TAG:-}"; then
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--remove-tags "${CANDIDATE_TAG}" \
|
||||
--quiet
|
||||
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
||||
--filter "metadata.name=${CANDIDATE_REVISION}" --format='value(metadata.name)')"
|
||||
if test -n "${existing}"; then
|
||||
test "${existing}" = "${CANDIDATE_REVISION}"
|
||||
gcloud run revisions delete "${CANDIDATE_REVISION}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
||||
fi
|
||||
echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
|
||||
echo "candidate revision ${CANDIDATE_REVISION} is absent"
|
||||
|
||||
# Public checks commit the serving revision; cleanup failures must not roll it back.
|
||||
- name: Retire previous consumers after public checks
|
||||
if: ${{ always() && (env.ROLLOUT_VERIFIED == 'true' || (env.RECOVERY_VERIFIED == 'true' && env.CANDIDATE_DELETED == 'true')) }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
serving="${CANDIDATE_REVISION}"
|
||||
if test "${RECOVERY_VERIFIED:-false}" = true; then
|
||||
serving="${TEMPLATE_RECOVERY_REVISION}"
|
||||
fi
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --clear-tags --quiet
|
||||
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
|
||||
while IFS= read -r revision; do
|
||||
test -n "${revision}" || continue
|
||||
test "${revision}" != "${serving}" || continue
|
||||
case "${revision}" in
|
||||
"${ROLLBACK_REVISION}"|"${VALIDATION_REVISION}"|"${CANDIDATE_REVISION}") ;;
|
||||
*) echo "Unexpected revision ${revision}; manual retirement required." >&2; exit 1 ;;
|
||||
esac
|
||||
gcloud run revisions delete "${revision}" \
|
||||
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
||||
done <<< "${revisions}"
|
||||
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
|
||||
echo 'Obsolete revision resources retired; verify actual SQL session drain operationally.' \
|
||||
>> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Drop the candidate traffic tag
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -n "${CANDIDATE_TAG:-}" || exit 0
|
||||
gcloud run services update-traffic "${SERVICE_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" \
|
||||
--region "${GCP_REGION}" \
|
||||
--remove-tags "${CANDIDATE_TAG}" \
|
||||
--quiet
|
||||
@@ -1,414 +0,0 @@
|
||||
name: Recover Relay Staging C4 Image
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [Prove Relay Staging Capacity]
|
||||
types: [completed]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
confirmation:
|
||||
description: Enter RECOVER_STAGING_ASIA_C4_IMAGE
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
gate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event_name == 'workflow_dispatch' || (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion != 'success')) }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
recover: ${{ steps.trigger.outputs.recover }}
|
||||
steps:
|
||||
- name: Bind recovery to the exact failed C4 job
|
||||
working-directory: .
|
||||
id: trigger
|
||||
env:
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
SOURCE_RUN_EVENT: ${{ github.event.workflow_run.event }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test "${GITHUB_EVENT_NAME}" = workflow_dispatch; then
|
||||
test "${CONFIRMATION}" = RECOVER_STAGING_ASIA_C4_IMAGE
|
||||
echo "recover=true" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
test "${SOURCE_RUN_EVENT}" = workflow_dispatch
|
||||
[[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
||||
jobs="$(gh api --paginate \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")"
|
||||
count="$(jq -s '[.[].jobs[] | select(.name == "refresh-asia-c4-image" and
|
||||
(.conclusion == "failure" or .conclusion == "cancelled" or
|
||||
.conclusion == "timed_out"))] | length' <<< "${jobs}")"
|
||||
if test "${count}" = 0; then
|
||||
echo "recover=false" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
test "${count}" = 1
|
||||
echo "recover=true" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
recover:
|
||||
needs: gate
|
||||
if: ${{ needs.gate.outputs.recover == 'true' }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 90
|
||||
environment: staging
|
||||
concurrency:
|
||||
group: relay-staging-mutation
|
||||
cancel-in-progress: false
|
||||
env:
|
||||
GCP_PROJECT_ID: onorca-cloud-staging
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
CELL_ORIGIN: https://c4.relay-staging.onorca.dev
|
||||
TARGET_CELL_ID: staging-gce-c4
|
||||
TARGET_ZONE: asia-east2-a
|
||||
MIG_NAME: orca-cloud-staging-relay-gce-c4
|
||||
PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7
|
||||
TARGET_IMAGE_DIGEST: sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- uses: ./.github/actions/cloud-sql-rollout-lease
|
||||
with:
|
||||
bucket: onorca-cloud-staging-terraform-state
|
||||
object: terraform/state/cloud-sql-rollout/staging.lock
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
terraform_wrapper: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- name: Initialize the exact staging backend
|
||||
run: node dev/scripts/infra.mjs init --env staging
|
||||
|
||||
- id: preflight-auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Inspect the exact C4 recovery state
|
||||
id: preflight
|
||||
timeout-minutes: 3
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.preflight-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \
|
||||
--expected-generation '' --expected-membership-sha256 '' --attempt-id '' \
|
||||
--image-digest "${PREDECESSOR_IMAGE_DIGEST}")"
|
||||
generation="$(jq -er '.generation' <<< "${inspect}")"
|
||||
[[ "${generation}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \
|
||||
migration-only
|
||||
echo "selector_generation=${generation}" >> "${GITHUB_OUTPUT}"
|
||||
if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \
|
||||
"${CELL_ORIGIN}/v1/admin/runtime-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data '{"v":1}')"; then
|
||||
current_digest="$(jq -er '.imageDigest' <<< "${runtime}")"
|
||||
[[ "${current_digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
status="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
|
||||
jq -e '.draining | type == "boolean"' <<< "${runtime}" >/dev/null
|
||||
{
|
||||
echo "runtime_available=true"
|
||||
echo "current_digest=${current_digest}"
|
||||
echo "draining=$(jq -r '.draining' <<< "${runtime}")"
|
||||
echo "ready=$(jq -r '.status.runtime.ready == true' <<< "${status}")"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
else
|
||||
echo "runtime_available=false" >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
|
||||
- name: Classify both exact recovery end states
|
||||
id: recovery-plan
|
||||
timeout-minutes: 10
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay"
|
||||
predecessor_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}"
|
||||
target_image="${image_repository}@${TARGET_IMAGE_DIGEST}"
|
||||
served_digest="$(gcloud artifacts docker images describe "${predecessor_image}" \
|
||||
--project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${served_digest}" = "${PREDECESSOR_IMAGE_DIGEST}"
|
||||
cells="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
|
||||
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].image' <<< "${cells}")" = \
|
||||
"${target_image}"
|
||||
target_plan="${RUNNER_TEMP}/relay-c4-image-target.tfplan"
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars -lock-timeout=5m \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
|
||||
-out="${target_plan}"
|
||||
target_result="$(terraform -chdir=infra/terraform show -json "${target_plan}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
|
||||
--unobserved-bound 60 --image "${target_image}" \
|
||||
--rollback-image "${predecessor_image}")"
|
||||
target_state="$(jq -r '[.changes,.changeKind] | join(":")' <<< "${target_result}")"
|
||||
case "${target_state}" in
|
||||
0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
|
||||
--arg image "${predecessor_image}" '.[$cell].image = $image' <<< "${cells}")"
|
||||
jq -n --argjson cells "${recovery_cells}" \
|
||||
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-recovery.tfvars.json"
|
||||
plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan"
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-c4-recovery.tfvars.json" \
|
||||
-lock-timeout=5m \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
|
||||
-out="${plan}"
|
||||
result="$(terraform -chdir=infra/terraform show -json "${plan}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
|
||||
--unobserved-bound 60 --image "${predecessor_image}" \
|
||||
--rollback-image "${target_image}")"
|
||||
changes="$(jq -r '.changes' <<< "${result}")"
|
||||
change_kind="$(jq -r '.changeKind' <<< "${result}")"
|
||||
case "${changes}:${change_kind}" in
|
||||
0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
mig="$(gcloud compute instance-groups managed describe "${MIG_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)"
|
||||
mig_stable="$(jq -r '.status.isStable == true and .status.versionTarget.isReached == true' \
|
||||
<<< "${mig}")"
|
||||
instances="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)"
|
||||
instance_stable="$(jq -r 'length == 1 and .[0].instanceStatus == "RUNNING" and
|
||||
.[0].currentAction == "NONE"' <<< "${instances}")"
|
||||
action=rollback-predecessor
|
||||
recovery_digest="${PREDECESSOR_IMAGE_DIGEST}"
|
||||
if test "${{ steps.preflight.outputs.runtime_available }}" = true && \
|
||||
test "${{ steps.preflight.outputs.ready }}" = true && \
|
||||
test "${{ steps.preflight.outputs.draining }}" = false && \
|
||||
test "${mig_stable}" = true && test "${instance_stable}" = true; then
|
||||
if test "${{ steps.preflight.outputs.current_digest }}" = "${TARGET_IMAGE_DIGEST}" && \
|
||||
test "${target_state}" = 0:none; then
|
||||
action=verify-target
|
||||
recovery_digest="${TARGET_IMAGE_DIGEST}"
|
||||
elif test "${{ steps.preflight.outputs.current_digest }}" = \
|
||||
"${PREDECESSOR_IMAGE_DIGEST}" && test "${changes}:${change_kind}" = 0:none; then
|
||||
action=verify-predecessor
|
||||
fi
|
||||
fi
|
||||
{
|
||||
echo "changes=${changes}"
|
||||
echo "change_kind=${change_kind}"
|
||||
echo "action=${action}"
|
||||
echo "recovery_digest=${recovery_digest}"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- id: fence-auth
|
||||
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Fence C4 before predecessor recovery
|
||||
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
|
||||
timeout-minutes: 6
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
|
||||
current_digest="${{ steps.preflight.outputs.current_digest }}"
|
||||
if test -n "${current_digest}" && [[ ",${expected_digests}," != *",${current_digest},"* ]]; then
|
||||
expected_digests="${expected_digests},${current_digest}"
|
||||
fi
|
||||
if curl --silent --show-error --fail-with-body --max-time 30 --request POST \
|
||||
"${CELL_ORIGIN}/v1/admin/drain" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' --data '{"v":1,"graceMs":0}' \
|
||||
>/dev/null; then
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
|
||||
--heartbeat fresh --admission migration-only --draining required \
|
||||
--activity quiescent \
|
||||
--expected-image-digests "${expected_digests}" \
|
||||
--timeout-ms 240000
|
||||
else
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \
|
||||
--admission migration-only --draining either --activity restart-safe \
|
||||
--timeout-ms 240000
|
||||
fi
|
||||
|
||||
- name: Apply and stabilize the saved predecessor plan
|
||||
id: apply
|
||||
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
CHANGES: ${{ steps.recovery-plan.outputs.changes }}
|
||||
CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan"
|
||||
if test "${CHANGES}" != 0; then
|
||||
terraform -chdir=infra/terraform apply -auto-approve "${plan}"
|
||||
fi
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Restart only when the plan did not replace C4
|
||||
id: restore
|
||||
if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
APPLY_STABLE_AT_MS: ${{ steps.apply.outputs.stable_at_ms }}
|
||||
CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ "${CHANGE_KIND}" =~ ^(replacement|replacement-with-obsolete-template|manager-convergence)$ ]]; then
|
||||
echo "stable_at_ms=${APPLY_STABLE_AT_MS}" >> "${GITHUB_OUTPUT}"
|
||||
exit 0
|
||||
fi
|
||||
instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \
|
||||
| jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and
|
||||
.[0].currentAction == "NONE" then .[0].instance | split("/") | last
|
||||
else error("C4 is not one stable running instance") end')"
|
||||
gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \
|
||||
--instances "${instance}" --project "${GCP_PROJECT_ID}" \
|
||||
--zone "${TARGET_ZONE}" --quiet
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Require an empty selected-image recovery readback
|
||||
timeout-minutes: 8
|
||||
env:
|
||||
RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay"
|
||||
recovery_image="${image_repository}@${RECOVERY_DIGEST}"
|
||||
other_image="${image_repository}@${TARGET_IMAGE_DIGEST}"
|
||||
if test "${RECOVERY_DIGEST}" = "${TARGET_IMAGE_DIGEST}"; then
|
||||
other_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}"
|
||||
fi
|
||||
cells="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/staging.tfvars \
|
||||
<<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
|
||||
recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" --arg image "${recovery_image}" \
|
||||
'.[$cell].image = $image' <<< "${cells}")"
|
||||
jq -n --argjson cells "${recovery_cells}" \
|
||||
'{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-readback.tfvars.json"
|
||||
readback="${RUNNER_TEMP}/relay-c4-image-recovery-readback.tfplan"
|
||||
terraform -chdir=infra/terraform plan \
|
||||
-var-file=environments/staging.tfvars \
|
||||
-var-file="${RUNNER_TEMP}/relay-c4-readback.tfvars.json" \
|
||||
-lock-timeout=5m \
|
||||
'-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
|
||||
'-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
|
||||
-out="${readback}"
|
||||
readback_result="$(terraform -chdir=infra/terraform show -json "${readback}" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
|
||||
--unobserved-bound 60 --image "${recovery_image}" \
|
||||
--rollback-image "${other_image}")"
|
||||
test "$(jq -r '.changes' <<< "${readback_result}")" = 0
|
||||
|
||||
- id: verify-auth
|
||||
if: ${{ always() }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- name: Verify the recovered image and unchanged isolation
|
||||
if: ${{ always() && steps.verify-auth.outcome == 'success' }}
|
||||
timeout-minutes: 8
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.verify-auth.outputs.id_token }}
|
||||
SELECTOR_GENERATION: ${{ steps.preflight.outputs.selector_generation }}
|
||||
STABLE_AT_MS: ${{ steps.restore.outputs.stable_at_ms }}
|
||||
RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
|
||||
--heartbeat fresh --admission migration-only --draining forbidden \
|
||||
--activity quiescent --expected-image-digests "${RECOVERY_DIGEST}" \
|
||||
--timeout-ms 240000
|
||||
stable_at_ms="${STABLE_AT_MS:-0}"
|
||||
for _ in $(seq 1 36); do
|
||||
status="$(curl --fail-with-body --max-time 30 --request POST \
|
||||
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
|
||||
--header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
|
||||
if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \
|
||||
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
|
||||
-ge "${stable_at_ms}"; then break; fi
|
||||
sleep 5
|
||||
done
|
||||
test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true
|
||||
test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
|
||||
-ge "${stable_at_ms}"
|
||||
result="$(node dev/scripts/operate-relay-asia-admission.mjs \
|
||||
--environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \
|
||||
--expected-generation "${SELECTOR_GENERATION}" \
|
||||
--expected-membership-sha256 '' --attempt-id '' \
|
||||
--image-digest "${RECOVERY_DIGEST}")"
|
||||
test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \
|
||||
migration-only
|
||||
@@ -1,60 +0,0 @@
|
||||
name: Requeue Relay Staging C4 Recovery
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [Recover Relay Staging C4 Image]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: relay-staging-c4-recovery-requeue
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
requeue:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion == 'cancelled') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Requeue only a cancelled protected recovery job
|
||||
working-directory: .
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
||||
jobs="$(gh api --paginate \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")"
|
||||
count="$(jq -s '[.[].jobs[] | select(.name == "recover" and
|
||||
.conclusion == "cancelled" and .started_at == null)] | length' <<< "${jobs}")"
|
||||
if test "${count}" = 0; then exit 0; fi
|
||||
test "${count}" = 1
|
||||
runs="$(gh api \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/cloud-recover-relay-staging-c4-image.yml/runs?branch=main&per_page=100")"
|
||||
active=0
|
||||
while IFS= read -r run_id; do
|
||||
active_jobs="$(gh api --paginate \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?filter=latest")"
|
||||
if jq -se '
|
||||
([.[].jobs[] | select(.name == "gate" and .conclusion == "success")] | length) == 1 and
|
||||
([.[].jobs[] | select(.name == "recover" and .status != "completed")] | length) == 1
|
||||
' <<< "${active_jobs}" >/dev/null; then
|
||||
active=1
|
||||
break
|
||||
fi
|
||||
done < <(jq -r --arg source "${SOURCE_RUN_ID}" \
|
||||
'.workflow_runs[] | select((.id | tostring) != $source and
|
||||
.status != "completed") | .id' <<< "${runs}")
|
||||
if test "${active}" != 0; then exit 0; fi
|
||||
gh workflow run cloud-recover-relay-staging-c4-image.yml \
|
||||
--repo "${GITHUB_REPOSITORY}" --ref main \
|
||||
-f confirmation=RECOVER_STAGING_ASIA_C4_IMAGE
|
||||
@@ -116,7 +116,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
app: [relay, push, relay-fence-broker, cloud-sql-proxy]
|
||||
app: [relay, push]
|
||||
env:
|
||||
APP: ${{ matrix.app }}
|
||||
TRIVY_IMAGE: aquasec/trivy:0.76.1@sha256:790fd66d1697395d8693d1acaa1ea702406a20f109b6fc7bac8db623eca6c06c
|
||||
@@ -126,20 +126,12 @@ jobs:
|
||||
- name: Build or resolve the runtime image
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if test "$APP" = cloud-sql-proxy; then
|
||||
IMAGE="$(sed -n '/variable "relay_gce_cloud_sql_proxy_image"/,/^}/s/.*default *= "\([^"]*\)"/\1/p' infra/terraform/variables.tf)"
|
||||
[[ "$IMAGE" =~ ^gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:[a-f0-9]{64}$ ]]
|
||||
docker pull --platform linux/amd64 "$IMAGE"
|
||||
else
|
||||
IMAGE="orca-cloud-security:$APP"
|
||||
docker build --platform linux/amd64 \
|
||||
--build-arg "ORCA_RELAY_FENCE_IMAGE_COMMIT=$GITHUB_SHA" \
|
||||
--file "apps/$APP/Dockerfile" --tag "$IMAGE" .
|
||||
fi
|
||||
IMAGE="orca-cloud-security:$APP"
|
||||
docker build --platform linux/amd64 \
|
||||
--file "apps/$APP/Dockerfile" --tag "$IMAGE" .
|
||||
echo "IMAGE=$IMAGE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Smoke test the nonroot Node runtime
|
||||
if: matrix.app != 'cloud-sql-proxy'
|
||||
run: |
|
||||
docker run --rm --platform linux/amd64 --entrypoint sh "$IMAGE" -ec '
|
||||
test "$(id -u)" -ne 0
|
||||
@@ -150,23 +142,6 @@ jobs:
|
||||
node --input-type=module -e "await import(\"./apps/$1/dist/$module.js\")"
|
||||
' sh "$APP"
|
||||
|
||||
- name: Smoke test broker tools and checksum fallback
|
||||
if: matrix.app == 'relay-fence-broker'
|
||||
run: |
|
||||
docker run --rm --platform linux/amd64 --entrypoint sh "$IMAGE" -ec '
|
||||
gcloud version
|
||||
printf relay-security > /tmp/hash-input
|
||||
test "$(gcloud storage hash --format="value(crc32c_hash)" /tmp/hash-input)" = "6oayYQ=="
|
||||
terraform version
|
||||
cd infra/terraform
|
||||
terraform init -backend=false -input=false -lockfile=readonly
|
||||
terraform validate
|
||||
'
|
||||
|
||||
- name: Smoke test the official SQL proxy binary
|
||||
if: matrix.app == 'cloud-sql-proxy'
|
||||
run: docker run --rm --platform linux/amd64 "$IMAGE" --version
|
||||
|
||||
- name: Scan every severity and retain the complete inventory
|
||||
run: |
|
||||
mkdir -p security-reports
|
||||
@@ -177,16 +152,12 @@ jobs:
|
||||
docker run --rm --volume "$PWD/security-reports:/reports" "$TRIVY_IMAGE" convert \
|
||||
--format cyclonedx --output /reports/sbom.cdx.json /reports/vulnerabilities.json
|
||||
|
||||
- name: Reject available fixes except dated upstream proxy rebuilds
|
||||
- name: Reject available fixes
|
||||
run: |
|
||||
jq '(.Results[]?.Vulnerabilities) |= ((. // []) | map(select(.FixedVersion != null and .FixedVersion != "")))' \
|
||||
security-reports/vulnerabilities.json > security-reports/fixable.json
|
||||
ignorefile=''
|
||||
if test "$APP" = cloud-sql-proxy; then
|
||||
ignorefile=/repo/security/cloud-sql-proxy.trivyignore.yaml
|
||||
fi
|
||||
docker run --rm --volume "$PWD:/repo:ro" "$TRIVY_IMAGE" convert \
|
||||
--ignorefile "$ignorefile" --show-suppressed --exit-code 1 \
|
||||
--ignorefile '' --show-suppressed --exit-code 1 \
|
||||
/repo/security-reports/fixable.json
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
@@ -234,38 +205,3 @@ jobs:
|
||||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: pnpm test
|
||||
|
||||
# Fork pull requests reach this job, so it never configures a backend, never plans, and never
|
||||
# holds a credential. Only the relay root ships here; foundation and apps stay private.
|
||||
terraform:
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: 1.17.0
|
||||
|
||||
- run: terraform -chdir=infra/terraform fmt -check -recursive
|
||||
- run: terraform -chdir=infra/terraform init -backend=false -input=false
|
||||
- run: terraform -chdir=infra/terraform validate
|
||||
|
||||
- name: Inventory provider binary vulnerabilities
|
||||
env:
|
||||
TRIVY_IMAGE: aquasec/trivy:0.76.1@sha256:790fd66d1697395d8693d1acaa1ea702406a20f109b6fc7bac8db623eca6c06c
|
||||
run: |
|
||||
mkdir -p security-reports
|
||||
docker run --rm --volume "$PWD:/repo" "$TRIVY_IMAGE" rootfs \
|
||||
--scanners vuln --ignorefile '' --format json \
|
||||
--output /repo/security-reports/providers.json /repo/infra/terraform/.terraform/providers
|
||||
docker run --rm --volume "$PWD/security-reports:/reports" "$TRIVY_IMAGE" convert \
|
||||
--format cyclonedx --output /reports/providers.cdx.json /reports/providers.json
|
||||
docker run --rm --volume "$PWD/security-reports:/reports" "$TRIVY_IMAGE" convert /reports/providers.json
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: cloud-terraform-provider-security
|
||||
path: cloud/security-reports/
|
||||
retention-days: 30
|
||||
|
||||
@@ -1386,7 +1386,7 @@ jobs:
|
||||
run: |
|
||||
$files = node config/scripts/pr-package-test-paths.mjs
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
pnpm exec vitest run --config config/vitest.config.ts $files
|
||||
node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts @files
|
||||
|
||||
# Why its own step: the CPU burners that make taskkill slow would starve
|
||||
# every other test file sharing the runner.
|
||||
|
||||
+10
-46
@@ -25,13 +25,9 @@ the validation required before rolling out the updated images and host pins.
|
||||
mutation lease, the Terraform checkout, and the narrow Compute mutation used
|
||||
when a registered target is superseded. The workflow that calls it holds read
|
||||
and invoke rights only, never those mutation permissions.
|
||||
- `apps/relay-ops`: the relay operations console and the incident monitor
|
||||
behind `pnpm ops:relay`, `pnpm incident:relay`, and
|
||||
`pnpm incident:relay-preflight`.
|
||||
- `apps/push` and `packages/push-contract`: the mobile push gateway that holds
|
||||
the APNs key and sends to phones through APNs and FCM, and its wire contract.
|
||||
It is deployed and operated from here but is not part of the relay data path;
|
||||
see [docs/push-gateway.md](docs/push-gateway.md).
|
||||
Deployment and operations are owned by the private `stablyai/orca-cloud` repository.
|
||||
|
||||
## Mobile push gateway
|
||||
|
||||
@@ -58,53 +54,21 @@ and local development. Configure it with `ORCA_PUSH_PUBLIC_URL`, `ORCA_PUSH_FCM_
|
||||
`ORCA_PUSH_APNS_KEY_ID`, `ORCA_PUSH_APPLE_TEAM_ID`, all three or none), and
|
||||
optionally `ORCA_PUSH_APNS_TOPIC`. The FCM credential comes from
|
||||
the runtime service account, so no key material is configured for Android. See
|
||||
[push gateway operations](docs/push-gateway.md) for deployment and recovery.
|
||||
[push gateway operations](https://github.com/stablyai/orca-cloud/blob/main/cloud/docs/push-gateway.md) for deployment and recovery.
|
||||
|
||||
Logging is aggregate counters only. Tokens, notification titles, notification
|
||||
bodies, and full host fingerprints never reach a log line.
|
||||
|
||||
## Infrastructure and operations
|
||||
## Deployment configuration
|
||||
|
||||
- `infra/terraform`: the relay Terraform root. It owns the cells, the director,
|
||||
the shared Cloud SQL instance, DNS, observability, and every GitHub Workload
|
||||
Identity provider the relay workflows authenticate through. `backend/` holds
|
||||
the per-environment backend configuration and `environments/` the tfvars.
|
||||
Drive it through `pnpm infra:init`, `pnpm infra:plan`, and `pnpm infra:apply`.
|
||||
- `dev/scripts`: the deploy, capacity, admission, rehome, monitoring, and load
|
||||
scripts the workflows call, plus the contract tests that pin each workflow
|
||||
and Terraform surface. Run them with `pnpm test`.
|
||||
- `dev/contracts` and `dev/fixtures`: the checked-in data those contract tests
|
||||
read, including the Terraform root partition.
|
||||
- `docs/`: the relay runbooks, capacity-testing guide, incident-monitor
|
||||
reference, the workflow variable reference in `docs/relay-workflows.md`, and
|
||||
the push gateway runbook in `docs/push-gateway.md`.
|
||||
The relay runtime and wire contracts are public. Production and staging Terraform,
|
||||
backend configuration, operator tools, runbooks, broker image assembly, and deployment
|
||||
workflows live in the private `stablyai/orca-cloud` repository. That repository pins
|
||||
an immutable public source revision and combines it with its deployment configuration.
|
||||
|
||||
## Workflows
|
||||
|
||||
The 25 `.github/workflows/cloud-*.yml` workflows are the deploy and operate
|
||||
surface: publish and deploy the director, roll GCE cell capacity, operate Asia
|
||||
admission and regional rehoming, prove staging capacity, monitor production,
|
||||
power staging up and down, and deploy the mobile push gateway.
|
||||
`.github/actions/cloud-sql-rollout-lease` is the compare-and-swap lease that
|
||||
serializes rollouts against the shared Cloud SQL instance. Push reuses that
|
||||
action with its own lease object and deployment concurrency group.
|
||||
|
||||
Every one of them is inert. Each top-level job is gated on
|
||||
`vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'`, a repository variable that is
|
||||
unset here, so the two scheduled triggers and every manual dispatch skip
|
||||
without running a step. Only the repository owner, holding the GCP identities
|
||||
these workflows authenticate as, can turn them on.
|
||||
|
||||
`Cloud Verify` is not gated. It builds, typechecks, lints, tests, secret-scans,
|
||||
and validates the relay Terraform on every change under `cloud/`, and it runs
|
||||
on fork pull requests, so it configures no backend and holds no credential.
|
||||
|
||||
## What is not here
|
||||
|
||||
The `terraform-foundation` and `terraform-apps` roots and the API and auth
|
||||
services live in the private `stablyai/orca-cloud` repository. Scripts and
|
||||
tests that spanned both trees were narrowed to the relay side rather than
|
||||
carrying a dangling reference.
|
||||
`Cloud Verify` here builds and tests public runtime code, audits dependencies, scans
|
||||
relay and push images, and checks for secrets without cloud credentials. Private CI
|
||||
also validates Terraform, scans the SQL proxy and broker, and runs deployment contracts.
|
||||
|
||||
## Local development
|
||||
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { expect, it } from 'vitest'
|
||||
import { loadPushConfig } from './config.js'
|
||||
|
||||
it('runs the deployment image preflight against the actual config loader', () => {
|
||||
const workflow = readFileSync(
|
||||
new URL('../../../../.github/workflows/cloud-push-deploy.yml', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const step = workflow.split('- name: Require image support for inert validation')[1]!
|
||||
const script = step.match(/--input-type=module -e '([\s\S]*?)'/)?.[1]
|
||||
expect(script).toBeDefined()
|
||||
const run = new Function('loadPushConfig', script!.replace(/import .*?;/, ''))
|
||||
expect(() => run(loadPushConfig)).not.toThrow()
|
||||
expect(() => run(() => ({ mode: 'active' }))).toThrow('validation_mode_unsupported')
|
||||
expect(() => run(() => ({ mode: 'validation' }))).toThrow('validation_mode_not_fail_closed')
|
||||
})
|
||||
@@ -1,51 +0,0 @@
|
||||
FROM node:24.22.1-alpine@sha256:feb76650b7195319268c59cb2d419c732d6c499b13bf5296f16ab59eb1f0a4f3 AS build
|
||||
WORKDIR /workspace
|
||||
ENV PATH="/opt/pnpm/node_modules/.bin:$PATH"
|
||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./
|
||||
RUN npm install --prefix /opt/pnpm --ignore-scripts --no-audit --no-fund "$(node -p 'require("./package.json").packageManager.split("+")[0]')" \
|
||||
&& npm audit signatures --prefix /opt/pnpm
|
||||
COPY apps/relay-fence-broker/package.json apps/relay-fence-broker/package.json
|
||||
RUN pnpm install --frozen-lockfile
|
||||
COPY apps/relay-fence-broker apps/relay-fence-broker
|
||||
RUN pnpm --filter @orca-cloud/relay-fence-broker build
|
||||
|
||||
FROM hashicorp/terraform:1.17.0@sha256:4d1b496a256f2989bccdb04b9104a4ede97234b10fba90259e173e905ec4fb58 AS terraform
|
||||
|
||||
FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:590.0.0-alpine@sha256:2d6788b06641d02cff063b337dd01a66b71b813bd22fb1fa1b9ee2e7c0ef0743
|
||||
RUN apk upgrade --no-cache && apk add --no-cache libstdc++
|
||||
ARG ORCA_RELAY_FENCE_IMAGE_COMMIT
|
||||
RUN test "$(printf '%s' "${ORCA_RELAY_FENCE_IMAGE_COMMIT}" | grep -E '^[a-f0-9]{40}$')"
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=8080
|
||||
ENV ORCA_RELAY_FENCE_IMAGE_COMMIT=${ORCA_RELAY_FENCE_IMAGE_COMMIT}
|
||||
ENV IAC_TOOL=terraform
|
||||
ENV CLOUDSDK_STORAGE_USE_GCLOUD_CRC32C=false
|
||||
WORKDIR /workspace
|
||||
COPY --from=build /usr/local /usr/local
|
||||
COPY --from=build /opt/pnpm /opt/pnpm
|
||||
ENV PATH="/opt/pnpm/node_modules/.bin:$PATH"
|
||||
COPY --from=terraform /bin/terraform /usr/local/bin/terraform
|
||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||
COPY apps/relay-fence-broker/package.json apps/relay-fence-broker/package.json
|
||||
COPY --from=build /workspace/apps/relay-fence-broker/dist apps/relay-fence-broker/dist
|
||||
COPY dev/scripts dev/scripts
|
||||
COPY infra/terraform infra/terraform
|
||||
COPY apps/relay-fence-broker/requirements.txt /tmp/broker-requirements.txt
|
||||
RUN /google-cloud-sdk/platform/bundledpythonunixmusl/bin/python3 -m pip install \
|
||||
--no-cache-dir --no-deps --require-hashes --upgrade \
|
||||
--target /google-cloud-sdk/lib/third_party -r /tmp/broker-requirements.txt \
|
||||
&& rm -rf /google-cloud-sdk/platform/bq /google-cloud-sdk/platform/gsutil \
|
||||
/google-cloud-sdk/platform/bundledpythonunixmusl/lib/python3.14/site-packages/pip* \
|
||||
/google-cloud-sdk/platform/bundledpythonunixmusl/lib/python3.14/site-packages/wheel* \
|
||||
&& rm -f /google-cloud-sdk/bin/gcloud-crc32c /google-cloud-sdk/bin/bq \
|
||||
/google-cloud-sdk/bin/gsutil /google-cloud-sdk/platform/bundledpythonunixmusl/bin/pip* \
|
||||
/google-cloud-sdk/platform/bundledpythonunixmusl/bin/wheel /tmp/broker-requirements.txt
|
||||
RUN pnpm install --prod --frozen-lockfile --filter @orca-cloud/relay-fence-broker... \
|
||||
&& rm -rf /opt/pnpm /usr/local/lib/node_modules /opt/yarn-* /root/.cache /root/.local \
|
||||
&& rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack \
|
||||
/usr/local/bin/pnpm /usr/local/bin/pnpx /usr/local/bin/yarn /usr/local/bin/yarnpkg \
|
||||
&& adduser -D -u 10001 broker \
|
||||
&& chown -R broker:broker /workspace
|
||||
USER broker
|
||||
EXPOSE 8080
|
||||
CMD ["node", "apps/relay-fence-broker/dist/index.js"]
|
||||
@@ -1,66 +0,0 @@
|
||||
# Orca Relay Operations
|
||||
|
||||
A private, aggregate dashboard for the Orca Relay control and data planes. It reads local `gcloud` and `gh` credentials on the server; credentials and per-user Relay state never enter the browser. One cached `gcloud auth print-access-token` refresh feeds concurrent read-only Google APIs so the collector does not stampede the local credential store.
|
||||
|
||||
## Run locally
|
||||
|
||||
Prerequisites:
|
||||
|
||||
- Node 24 and pnpm 10
|
||||
- `gcloud` authenticated for `onorca-cloud` and `onorca-cloud-staging`
|
||||
- `gh` authenticated with read access to `stablyai/orca-cloud`
|
||||
|
||||
From the repository root:
|
||||
|
||||
```sh
|
||||
pnpm install
|
||||
pnpm ops:relay
|
||||
```
|
||||
|
||||
Open <http://127.0.0.1:2455>. The server binds only to loopback and refreshes aggregate data every minute. Production and staging are read-only by default.
|
||||
|
||||
The cost panel is a labeled planning estimate. There is currently no Cloud Billing export in either project, so the dashboard cannot claim exact billed spend. GCP Billing remains authoritative.
|
||||
|
||||
## Share through Tailscale
|
||||
|
||||
Keep the dashboard bound to loopback and let Tailscale provide identity, TLS, and tailnet ACL enforcement:
|
||||
|
||||
```sh
|
||||
tailscale serve --bg http://127.0.0.1:2455
|
||||
tailscale serve status
|
||||
```
|
||||
|
||||
Share the HTTPS URL printed by `tailscale serve status` with the team. Limit access to the intended operator group in the tailnet ACL. Do not use a public funnel. Stop sharing with:
|
||||
|
||||
```sh
|
||||
tailscale serve reset
|
||||
```
|
||||
|
||||
For a persistent host, run `pnpm --filter @orca-cloud/relay-ops build` and supervise `pnpm --filter @orca-cloud/relay-ops start` with the host's normal process manager. The process needs the same non-interactive `gcloud` and `gh` identities.
|
||||
|
||||
## Optional staging controls
|
||||
|
||||
Controls are intentionally local-only and disabled unless explicitly enabled:
|
||||
|
||||
```sh
|
||||
RELAY_OPS_ENABLE_STAGING_CONTROLS=1 pnpm ops:relay
|
||||
```
|
||||
|
||||
Even in this mode the service never changes GCP directly. It dispatches `.github/workflows/power-relay-staging.yml`, preserves the workflow's typed `WAKE_STAGING` / `SLEEP_STAGING` confirmation, and always wakes only configured-admission cells. Requests require the loopback origin and a per-process CSRF token, so controls stay unavailable through the Tailscale view.
|
||||
|
||||
## Data and security boundaries
|
||||
|
||||
- Browser payloads contain aggregate Monitoring points, resource health, immutable image digests, alert-policy metadata, and workflow metadata.
|
||||
- Account IDs, host IDs, device IDs, pairing state, assignment rows, bearer tokens, service-account tokens, startup scripts, secret values, and individual Relay-admin state are excluded.
|
||||
- Sleeping staging is inventory-only. Viewing it does not probe or cold-start Cloud Run services and cannot resize empty MIGs.
|
||||
- Partial GCP or GitHub failures degrade the affected panel and produce a sanitized warning.
|
||||
- Missing cell inventory renders as `Unknown`, never `Sleeping`. After one successful read, transient credential or collector failures retain the last good snapshot and mark it stale.
|
||||
- Responses use `no-store`, a restrictive CSP, frame denial, and no-referrer headers.
|
||||
|
||||
## Verification
|
||||
|
||||
```sh
|
||||
pnpm --filter @orca-cloud/relay-ops test
|
||||
pnpm --filter @orca-cloud/relay-ops typecheck
|
||||
pnpm --filter @orca-cloud/relay-ops build
|
||||
```
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"name": "@orca-cloud/relay-ops",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
"build": "pnpm clean && tsc -p tsconfig.build.json && node -e \"require('fs').cpSync('public','dist/public',{recursive:true})\"",
|
||||
"clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"incident:monitor": "tsx src/incident-monitor-cli.ts",
|
||||
"incident:preflight": "tsx src/incident-live-preflight-cli.ts",
|
||||
"incident:pre-drain-sample": "tsx src/pre-drain-sample-cli.ts",
|
||||
"lint": "tsc -p tsconfig.json --noEmit",
|
||||
"start": "node dist/index.js",
|
||||
"test": "vitest run",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^2.1.5",
|
||||
"hono": "^4.13.13",
|
||||
"zod": "^4.6.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.19.2",
|
||||
"tsx": "^4.23.15",
|
||||
"typescript": "^7.0.2",
|
||||
"vitest": "^5.0.5"
|
||||
}
|
||||
}
|
||||
@@ -1,273 +0,0 @@
|
||||
const state = { environment: 'production', window: 360, snapshot: null, config: null, loading: false }
|
||||
|
||||
const $ = (selector) => document.querySelector(selector)
|
||||
const all = (selector) => [...document.querySelectorAll(selector)]
|
||||
const escapeHtml = (value) => String(value).replace(/[&<>'"]/g, (character) => ({
|
||||
'&': '&', '<': '<', '>': '>', "'": ''', '"': '"'
|
||||
})[character])
|
||||
|
||||
function formatNumber(value, maximumFractionDigits = 0) {
|
||||
if (value === null || value === undefined) return '—'
|
||||
return new Intl.NumberFormat('en-US', { notation: value >= 10_000 ? 'compact' : 'standard', maximumFractionDigits }).format(value)
|
||||
}
|
||||
|
||||
function formatBytes(value) {
|
||||
if (value === null || value === undefined) return '—'
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB']
|
||||
let amount = value
|
||||
let unit = 0
|
||||
while (Math.abs(amount) >= 1000 && unit < units.length - 1) { amount /= 1000; unit += 1 }
|
||||
return `${formatNumber(amount, amount < 10 ? 1 : 0)} ${units[unit]}`
|
||||
}
|
||||
|
||||
function formatMetric(metric, value) {
|
||||
if (value === null || value === undefined) return '—'
|
||||
if (metric.unit === 'bytes') return formatBytes(value)
|
||||
if (metric.unit === 'milliseconds') return `${formatNumber(value, 1)} ms`
|
||||
return formatNumber(value, value < 10 ? 1 : 0)
|
||||
}
|
||||
|
||||
function timeAgo(value) {
|
||||
const seconds = Math.max(0, Math.round((Date.now() - Date.parse(value)) / 1000))
|
||||
if (seconds < 60) return `${seconds}s ago`
|
||||
if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`
|
||||
if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`
|
||||
return `${Math.floor(seconds / 86400)}d ago`
|
||||
}
|
||||
|
||||
function shortImage(value) {
|
||||
const digest = value?.match(/sha256:([a-f0-9]{64})/)?.[1]
|
||||
if (digest) return digest.slice(0, 10)
|
||||
const tag = value?.split(':').at(-1)
|
||||
return tag?.slice(0, 16) ?? '—'
|
||||
}
|
||||
|
||||
function badge(label, healthy) {
|
||||
return `<span class="badge ${healthy === true ? 'healthy' : healthy === false ? 'unhealthy' : ''}">${escapeHtml(label)}</span>`
|
||||
}
|
||||
|
||||
function renderSummary(snapshot) {
|
||||
const { summary } = snapshot
|
||||
const utilization = summary.poweredCapacity === null
|
||||
? null
|
||||
: summary.poweredCapacity > 0
|
||||
? summary.observedConnections / summary.poweredCapacity * 100
|
||||
: 0
|
||||
const items = [
|
||||
['Observed connections', formatNumber(summary.observedConnections, 1), 'Latest 1-minute aggregate mean'],
|
||||
['Active relay sessions', formatNumber(summary.observedSplices, 1), `${formatNumber(summary.observedControls, 1)} desktop-control mean`],
|
||||
['Healthy cells', `${summary.activeCells ?? '—'} / ${summary.totalCells}`, summary.poweredCapacity === null ? 'Cell inventory unavailable' : `${formatNumber(summary.poweredCapacity)} powered request units`],
|
||||
['Capacity signal', utilization === null ? '—' : `${formatNumber(utilization, 2)}%`, `${formatNumber(summary.configuredCapacity)} configured admission units`]
|
||||
]
|
||||
$('#summary').innerHTML = items.map(([label, value, caption]) => `
|
||||
<article class="metric-card">
|
||||
<p class="eyebrow">${escapeHtml(label)}</p>
|
||||
<div class="value">${escapeHtml(value)}</div>
|
||||
<div class="caption">${escapeHtml(caption)}</div>
|
||||
</article>`).join('')
|
||||
}
|
||||
|
||||
function cellState(cell) {
|
||||
if (cell.targetSize === null) return ['Unknown', null]
|
||||
if (cell.targetSize === 0) return ['Sleeping', null]
|
||||
if (cell.backendHealth === 'healthy' && cell.endpoint.ready) return ['Healthy', true]
|
||||
if (cell.stable && cell.backendHealth === 'empty') return ['Starting', null]
|
||||
return ['Attention', false]
|
||||
}
|
||||
|
||||
function renderTopology(snapshot) {
|
||||
const director = snapshot.resources.director
|
||||
const sleeping = snapshot.resources.cells.every((cell) => cell.targetSize === 0)
|
||||
&& snapshot.resources.sql?.activationPolicy === 'NEVER'
|
||||
const directorHealthy = director?.ready && snapshot.resources.directorEndpoint.health
|
||||
$('#topology-meta').textContent = `${snapshot.environment.project} · ${snapshot.environment.region}`
|
||||
const cellNodes = snapshot.resources.cells.map((cell) => {
|
||||
const [label, healthy] = cellState(cell)
|
||||
return `<div class="topology-node">
|
||||
<strong>${escapeHtml(cell.hostname.toUpperCase())} ${badge(label, healthy)}</strong>
|
||||
<span>${escapeHtml(cell.region)} · ${escapeHtml(cell.zone)} · ${formatNumber(cell.capacityRequests)} units</span>
|
||||
</div>`
|
||||
}).join('')
|
||||
$('#topology').innerHTML = `
|
||||
<div class="topology-node"><strong>Desktop + phone</strong><span>Encrypted Relay traffic</span></div>
|
||||
<div class="connector" aria-hidden="true"></div>
|
||||
<div class="topology-node"><strong>Director ${badge(sleeping ? 'Sleeping' : directorHealthy ? 'Ready' : 'Attention', sleeping ? null : Boolean(directorHealthy))}</strong><span>${escapeHtml(snapshot.environment.directorOrigin)}</span></div>
|
||||
<div class="connector" aria-hidden="true"></div>
|
||||
<div class="topology-cells">${cellNodes}</div>`
|
||||
}
|
||||
|
||||
function chartPaths(points, width = 400, height = 112) {
|
||||
if (points.length === 0) return null
|
||||
const values = points.map((point) => point.value)
|
||||
const minimum = Math.min(0, ...values)
|
||||
const maximum = Math.max(...values)
|
||||
const spread = maximum - minimum || 1
|
||||
const coordinates = points.map((point, index) => {
|
||||
const x = points.length === 1 ? width : index / (points.length - 1) * width
|
||||
const y = height - ((point.value - minimum) / spread * (height - 10) + 5)
|
||||
return [x, y]
|
||||
})
|
||||
const line = coordinates.map(([x, y], index) => `${index === 0 ? 'M' : 'L'}${x.toFixed(1)},${y.toFixed(1)}`).join(' ')
|
||||
const area = `${line} L${width},${height} L0,${height} Z`
|
||||
return { line, area }
|
||||
}
|
||||
|
||||
function renderCharts(snapshot) {
|
||||
const names = ['controls', 'splices', 'assignment_5xx', 'postgres_retries', 'auth_failures', 'event_loop_ms_p99']
|
||||
$('#charts').innerHTML = names.map((name) => {
|
||||
const metric = snapshot.monitoring.metrics[name]
|
||||
const paths = chartPaths(metric.points)
|
||||
const graph = paths ? `<svg class="chart" viewBox="0 0 400 112" preserveAspectRatio="none" role="img" aria-label="${escapeHtml(metric.label)} over time">
|
||||
<line x1="0" y1="111" x2="400" y2="111"></line>
|
||||
<path class="area" d="${paths.area}"></path><path class="line" d="${paths.line}"></path>
|
||||
</svg>` : '<div class="chart-empty">No samples in this window</div>'
|
||||
return `<article class="panel chart-card"><div class="chart-head"><div><p class="eyebrow">${escapeHtml(metric.label)}</p><div class="chart-value">${escapeHtml(formatMetric(metric, metric.latest))}</div></div><span class="chart-unit">${escapeHtml(metric.unit)}</span></div>${graph}</article>`
|
||||
}).join('')
|
||||
}
|
||||
|
||||
function renderCells(snapshot) {
|
||||
const controlsByCell = snapshot.monitoring.metrics.controls.latestByCell
|
||||
const splicesByCell = snapshot.monitoring.metrics.splices.latestByCell
|
||||
$('#cells').innerHTML = snapshot.resources.cells.map((cell) => {
|
||||
const [label, healthy] = cellState(cell)
|
||||
const observed = (controlsByCell[cell.cellId] ?? 0) + (splicesByCell[cell.cellId] ?? 0)
|
||||
return `<tr>
|
||||
<td><strong>${escapeHtml(cell.hostname.toUpperCase())}</strong><div class="row-caption">${escapeHtml(cell.region)} · ${escapeHtml(cell.zone)}</div></td>
|
||||
<td>${badge(label, healthy)}<div class="row-caption">MIG ${cell.runningInstances ?? '—'}/${cell.targetSize ?? '—'}</div></td>
|
||||
<td>${formatNumber(observed, 1)}<div class="row-caption">1-minute mean</div></td>
|
||||
<td>${formatNumber(cell.capacityRequests)}<div class="row-caption">DB pool ${formatNumber(cell.databasePoolMax)} · ${cell.configuredAdmission ? 'configured' : 'candidate only'}</div></td>
|
||||
<td class="mono">${escapeHtml(shortImage(cell.imageDigest))}</td>
|
||||
</tr>`
|
||||
}).join('')
|
||||
}
|
||||
|
||||
function serviceRow(name, healthy, detail, suffix = '') {
|
||||
return `<div class="service-row"><div><div class="row-title">${escapeHtml(name)}</div><div class="row-caption">${escapeHtml(detail)}</div></div>${badge(suffix || (healthy ? 'Ready' : 'Attention'), healthy)}</div>`
|
||||
}
|
||||
|
||||
function renderServices(snapshot) {
|
||||
const { resources } = snapshot
|
||||
const sleeping = resources.cells.every((cell) => cell.targetSize === 0)
|
||||
&& resources.sql?.activationPolicy === 'NEVER'
|
||||
const certDays = resources.certificate?.expireTime
|
||||
? Math.floor((Date.parse(resources.certificate.expireTime) - Date.now()) / 86400000)
|
||||
: null
|
||||
$('#services').innerHTML = [
|
||||
serviceRow('Director', sleeping ? null : Boolean(resources.director?.ready && resources.directorEndpoint.health), `${resources.director?.revision ?? 'Revision unavailable'} · ${shortImage(resources.director?.image)}`, sleeping ? 'Sleeping' : ''),
|
||||
serviceRow('Authentication', sleeping ? null : Boolean(resources.auth?.ready && resources.authEndpoint.health), `${resources.auth?.revision ?? 'Revision unavailable'} · ${shortImage(resources.auth?.image)}`, sleeping ? 'Sleeping' : ''),
|
||||
serviceRow('Cloud SQL', resources.sql?.state === 'RUNNABLE' || resources.sql?.activationPolicy === 'NEVER', `${resources.sql?.tier ?? 'unknown'} · ${resources.sql?.activationPolicy ?? 'unknown'}`, resources.sql?.state ?? 'Unknown'),
|
||||
serviceRow('Wildcard TLS', resources.certificate?.state === 'ACTIVE', resources.certificate?.domains.join(', ') ?? 'Certificate unavailable', certDays === null ? 'Unknown' : `${certDays}d`)
|
||||
].join('')
|
||||
}
|
||||
|
||||
function renderAlerts(snapshot) {
|
||||
const policies = snapshot.monitoring.alertPolicies
|
||||
$('#alerts').innerHTML = policies.length ? policies.map((policy) => `<div class="list-row"><div><div class="row-title">${escapeHtml(policy.displayName.replace('Orca Relay: ', ''))}</div><div class="row-caption">Cloud Monitoring policy</div></div>${badge(policy.enabled ? 'Enabled' : 'Disabled', policy.enabled)}</div>`).join('') : '<p class="muted">No Relay alert policies returned.</p>'
|
||||
}
|
||||
|
||||
function renderWorkflows(snapshot) {
|
||||
$('#workflows').innerHTML = snapshot.workflows.length ? snapshot.workflows.slice(0, 6).map((run) => {
|
||||
const healthy = run.conclusion === 'success'
|
||||
const stateLabel = run.status === 'completed' ? (run.conclusion ?? 'completed') : run.status
|
||||
return `<a class="list-row" href="${escapeHtml(run.url)}" target="_blank" rel="noreferrer"><div><div class="row-title">${escapeHtml(run.name)}</div><div class="row-caption">${escapeHtml(run.headSha)} · ${timeAgo(run.updatedAt)}</div></div>${badge(stateLabel, healthy)}</a>`
|
||||
}).join('') : '<p class="muted">Workflow history unavailable.</p>'
|
||||
}
|
||||
|
||||
function renderCost(snapshot) {
|
||||
const cost = snapshot.cost
|
||||
$('#cost').innerHTML = `<div class="cost-total"><strong>$${formatNumber(cost.monthlyUsd)}</strong><span class="muted">/ month</span></div>
|
||||
<p class="row-caption">Modeled range $${formatNumber(cost.rangeUsd[0])}–$${formatNumber(cost.rangeUsd[1])}. Not billed cost.</p>
|
||||
<div class="cost-lines">${cost.lines.map((line) => `<div class="cost-line"><span>${escapeHtml(line.label)}</span><strong>$${formatNumber(line.monthlyUsd, 2)}</strong></div>`).join('')}</div>
|
||||
<p class="cost-note"><strong>Exact billing:</strong> ${escapeHtml(cost.actualBilling.reason)}<br>${escapeHtml(cost.caveats[1])}</p>`
|
||||
}
|
||||
|
||||
function renderWarnings(snapshot) {
|
||||
const warning = $('#warnings')
|
||||
warning.classList.toggle('hidden', snapshot.warnings.length === 0)
|
||||
warning.textContent = snapshot.warnings.length ? `Partial data: ${snapshot.warnings.join(' ')}` : ''
|
||||
}
|
||||
|
||||
function render(snapshot) {
|
||||
state.snapshot = snapshot
|
||||
$('#environment-label').textContent = `${snapshot.environment.label} · ${snapshot.environment.region}`
|
||||
$('#freshness').textContent = snapshot.stale
|
||||
? `Last good update ${timeAgo(snapshot.generatedAt)} · refresh degraded`
|
||||
: `Updated ${timeAgo(snapshot.generatedAt)}`
|
||||
$('#freshness-dot').className = snapshot.stale ? 'status-dot neutral' : 'status-dot healthy'
|
||||
renderWarnings(snapshot)
|
||||
renderSummary(snapshot)
|
||||
renderTopology(snapshot)
|
||||
renderCharts(snapshot)
|
||||
renderCells(snapshot)
|
||||
renderServices(snapshot)
|
||||
renderAlerts(snapshot)
|
||||
renderWorkflows(snapshot)
|
||||
renderCost(snapshot)
|
||||
$('#power-panel').classList.toggle('hidden', !(state.environment === 'staging' && state.config?.stagingControlsEnabled))
|
||||
$('#compute-link').href = snapshot.environment.consoleLinks.compute
|
||||
$('#alert-link').href = snapshot.environment.consoleLinks.alerts
|
||||
}
|
||||
|
||||
async function loadSnapshot() {
|
||||
if (state.loading) return
|
||||
state.loading = true
|
||||
$('#refresh').disabled = true
|
||||
$('#error').classList.add('hidden')
|
||||
$('#freshness-dot').className = 'status-dot neutral'
|
||||
$('#freshness').textContent = 'Refreshing current state…'
|
||||
try {
|
||||
const response = await fetch(`/api/snapshot?environment=${state.environment}&window=${state.window}`)
|
||||
const body = await response.json()
|
||||
if (!response.ok) throw new Error(body.error ?? 'Snapshot failed')
|
||||
render(body)
|
||||
} catch (error) {
|
||||
$('#freshness-dot').className = 'status-dot unhealthy'
|
||||
$('#freshness').textContent = 'Refresh failed'
|
||||
$('#error').textContent = error instanceof Error ? error.message : 'Relay operations data is unavailable.'
|
||||
$('#error').classList.remove('hidden')
|
||||
} finally {
|
||||
state.loading = false
|
||||
$('#refresh').disabled = false
|
||||
}
|
||||
}
|
||||
|
||||
async function dispatchPower(mode) {
|
||||
const confirmation = mode === 'wake' ? 'WAKE_STAGING' : mode === 'sleep' ? 'SLEEP_STAGING' : ''
|
||||
if (confirmation) {
|
||||
const entered = window.prompt(`Type ${confirmation} to dispatch the guarded workflow.`) ?? ''
|
||||
if (entered !== confirmation) return
|
||||
}
|
||||
all('[data-power]').forEach((button) => { button.disabled = true })
|
||||
$('#power-result').textContent = 'Dispatching…'
|
||||
try {
|
||||
const response = await fetch('/api/staging/power', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', 'x-csrf-token': state.config.csrfToken },
|
||||
body: JSON.stringify({ mode, confirmation })
|
||||
})
|
||||
const body = await response.json()
|
||||
if (!response.ok) throw new Error(body.error)
|
||||
$('#power-result').textContent = 'Workflow accepted. Refresh after it completes.'
|
||||
} catch (error) {
|
||||
$('#power-result').textContent = error instanceof Error ? error.message : 'Dispatch failed.'
|
||||
} finally {
|
||||
all('[data-power]').forEach((button) => { button.disabled = false })
|
||||
}
|
||||
}
|
||||
|
||||
all('[data-environment]').forEach((button) => button.addEventListener('click', () => {
|
||||
state.environment = button.dataset.environment
|
||||
all('[data-environment]').forEach((candidate) => candidate.setAttribute('aria-pressed', String(candidate === button)))
|
||||
loadSnapshot()
|
||||
}))
|
||||
$('#window').addEventListener('change', (event) => { state.window = Number(event.target.value); loadSnapshot() })
|
||||
$('#refresh').addEventListener('click', loadSnapshot)
|
||||
all('[data-power]').forEach((button) => button.addEventListener('click', () => dispatchPower(button.dataset.power)))
|
||||
|
||||
async function start() {
|
||||
try { state.config = await fetch('/api/config').then((response) => response.json()) } catch { state.config = {} }
|
||||
await loadSnapshot()
|
||||
window.setInterval(loadSnapshot, 60_000)
|
||||
}
|
||||
|
||||
start()
|
||||
@@ -1,115 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="color-scheme" content="light dark" />
|
||||
<title>Orca Relay Operations</title>
|
||||
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'%3E%3Crect width='32' height='32' rx='9' fill='%23171717'/%3E%3Ctext x='16' y='22' text-anchor='middle' font-family='sans-serif' font-weight='700' font-size='17' fill='white'%3EO%3C/text%3E%3C/svg%3E" />
|
||||
<link rel="stylesheet" href="/styles.css" />
|
||||
<script type="module" src="/app.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<header class="topbar">
|
||||
<div class="brand">
|
||||
<span class="brand-mark" aria-hidden="true">O</span>
|
||||
<div>
|
||||
<strong>Orca Relay</strong>
|
||||
<span>Operations</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="toolbar">
|
||||
<div class="segmented" aria-label="Environment">
|
||||
<button type="button" data-environment="production" aria-pressed="true">Production</button>
|
||||
<button type="button" data-environment="staging" aria-pressed="false">Staging</button>
|
||||
</div>
|
||||
<select id="window" aria-label="Metrics window">
|
||||
<option value="60">Last hour</option>
|
||||
<option value="360" selected>Last 6 hours</option>
|
||||
<option value="1440">Last 24 hours</option>
|
||||
</select>
|
||||
<button type="button" id="refresh" class="button secondary">Refresh</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<section class="page-heading">
|
||||
<div>
|
||||
<p class="eyebrow" id="environment-label">Production · us-central1</p>
|
||||
<h1>Relay data plane</h1>
|
||||
<p class="lede">Private, aggregate operations view. No account, host, device, or pairing identifiers.</p>
|
||||
</div>
|
||||
<div class="freshness">
|
||||
<span class="status-dot neutral" id="freshness-dot"></span>
|
||||
<span id="freshness">Loading current state…</span>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div id="error" class="banner error hidden" role="alert"></div>
|
||||
<div id="warnings" class="banner warning hidden"></div>
|
||||
|
||||
<section class="summary-grid" id="summary" aria-label="Relay summary">
|
||||
<article class="metric-card skeleton"></article>
|
||||
<article class="metric-card skeleton"></article>
|
||||
<article class="metric-card skeleton"></article>
|
||||
<article class="metric-card skeleton"></article>
|
||||
</section>
|
||||
|
||||
<section class="panel topology-panel">
|
||||
<div class="panel-heading">
|
||||
<div><p class="eyebrow">Topology</p><h2>Request path</h2></div>
|
||||
<span class="meta" id="topology-meta"></span>
|
||||
</div>
|
||||
<div class="topology" id="topology"></div>
|
||||
</section>
|
||||
|
||||
<section class="section-heading"><div><p class="eyebrow">Traffic</p><h2>Signals in selected window</h2></div></section>
|
||||
<section class="chart-grid" id="charts"></section>
|
||||
|
||||
<section class="two-column">
|
||||
<article class="panel">
|
||||
<div class="panel-heading"><div><p class="eyebrow">Compute</p><h2>Relay cells</h2></div><a class="meta panel-link" id="compute-link" target="_blank" rel="noreferrer">Open in GCP ↗</a></div>
|
||||
<p class="panel-note">Configured admission is shown below. Live director admission and heartbeat state stays unavailable because this dashboard has no Relay-admin identity.</p>
|
||||
<div class="table-wrap"><table>
|
||||
<thead><tr><th>Cell</th><th>State</th><th>Observed</th><th>Capacity</th><th>Image</th></tr></thead>
|
||||
<tbody id="cells"></tbody>
|
||||
</table></div>
|
||||
</article>
|
||||
<article class="panel">
|
||||
<div class="panel-heading"><div><p class="eyebrow">Control plane</p><h2>Services</h2></div></div>
|
||||
<div class="service-list" id="services"></div>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section class="three-column">
|
||||
<article class="panel">
|
||||
<div class="panel-heading"><div><p class="eyebrow">Guardrails</p><h2>Alert policies</h2></div><a class="meta panel-link" id="alert-link" target="_blank" rel="noreferrer">Open incidents ↗</a></div>
|
||||
<div class="list" id="alerts"></div>
|
||||
</article>
|
||||
<article class="panel">
|
||||
<div class="panel-heading"><div><p class="eyebrow">Delivery</p><h2>Recent workflows</h2></div></div>
|
||||
<div class="list" id="workflows"></div>
|
||||
</article>
|
||||
<article class="panel">
|
||||
<div class="panel-heading"><div><p class="eyebrow">Planning</p><h2>Monthly run-rate</h2></div></div>
|
||||
<div id="cost"></div>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section class="panel hidden" id="power-panel">
|
||||
<div class="panel-heading">
|
||||
<div><p class="eyebrow">Staging only</p><h2>Power workflow</h2></div>
|
||||
<span class="badge">GitHub guarded</span>
|
||||
</div>
|
||||
<p class="muted">Dispatches the reviewed GitHub workflow. It never changes GCP resources directly from this process.</p>
|
||||
<div class="power-actions">
|
||||
<button class="button secondary" type="button" data-power="status">Check status</button>
|
||||
<button class="button secondary" type="button" data-power="wake">Wake configured cells</button>
|
||||
<button class="button secondary" type="button" data-power="sleep">Sleep staging</button>
|
||||
<span id="power-result" class="meta"></span>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
<footer>Orca Relay Operations · Server-side credentials · Read-only by default</footer>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,165 +0,0 @@
|
||||
:root {
|
||||
--radius: 10px;
|
||||
--background: #fff;
|
||||
--foreground: #0a0a0a;
|
||||
--card: #fff;
|
||||
--primary: #171717;
|
||||
--primary-foreground: #fafafa;
|
||||
--secondary: #f5f5f5;
|
||||
--muted: #f5f5f5;
|
||||
--muted-foreground: #737373;
|
||||
--accent: #f5f5f5;
|
||||
--border: #e5e5e5;
|
||||
--ring: #a1a1a1;
|
||||
--destructive: #e40014;
|
||||
--success: #15803d;
|
||||
--warning: #895503;
|
||||
--font-mono: 'SF Mono', SFMono-Regular, ui-monospace, Menlo, Consolas, monospace;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root {
|
||||
--background: #0a0a0a;
|
||||
--foreground: #fafafa;
|
||||
--card: #171717;
|
||||
--primary: #e5e5e5;
|
||||
--primary-foreground: #171717;
|
||||
--secondary: #262626;
|
||||
--muted: #262626;
|
||||
--muted-foreground: #a1a1a1;
|
||||
--accent: #262626;
|
||||
--border: rgb(255 255 255 / 0.07);
|
||||
--ring: #737373;
|
||||
--destructive: #ff6568;
|
||||
--success: #4ade80;
|
||||
--warning: #fbbf24;
|
||||
}
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0;
|
||||
background: var(--background);
|
||||
color: var(--foreground);
|
||||
font-family: 'Geist', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
|
||||
font-size: 14px;
|
||||
letter-spacing: .01em;
|
||||
}
|
||||
button, select { font: inherit; }
|
||||
button:focus-visible, select:focus-visible { outline: 2px solid var(--ring); outline-offset: 2px; }
|
||||
.topbar {
|
||||
height: 64px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
padding: 0 max(24px, calc((100vw - 1440px) / 2));
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
background: color-mix(in srgb, var(--background) 92%, transparent);
|
||||
backdrop-filter: blur(16px);
|
||||
z-index: 10;
|
||||
}
|
||||
.brand { display: flex; align-items: center; gap: 10px; }
|
||||
.brand-mark {
|
||||
width: 30px; height: 30px; border-radius: 9px; background: var(--primary);
|
||||
color: var(--primary-foreground); display: grid; place-items: center; font-weight: 700;
|
||||
}
|
||||
.brand div { display: flex; align-items: baseline; gap: 7px; }
|
||||
.brand span:last-child { color: var(--muted-foreground); font-size: 12px; }
|
||||
.toolbar { display: flex; align-items: center; gap: 8px; }
|
||||
.segmented { display: flex; padding: 3px; gap: 2px; border-radius: 8px; background: var(--secondary); }
|
||||
.segmented button { border: 0; background: transparent; color: var(--muted-foreground); padding: 5px 10px; border-radius: 6px; cursor: pointer; }
|
||||
.segmented button[aria-pressed="true"] { background: var(--card); color: var(--foreground); box-shadow: 0 1px 2px rgb(0 0 0 / .08); }
|
||||
select, .button { height: 32px; border: 1px solid var(--border); border-radius: 8px; background: var(--card); color: var(--foreground); padding: 0 10px; }
|
||||
.button { cursor: pointer; font-weight: 500; }
|
||||
.button:hover { background: var(--accent); }
|
||||
.button:disabled { opacity: .5; cursor: wait; }
|
||||
main { max-width: 1440px; margin: 0 auto; padding: 42px 24px 72px; }
|
||||
.page-heading { display: flex; justify-content: space-between; align-items: end; gap: 24px; margin-bottom: 28px; }
|
||||
h1, h2, p { margin: 0; }
|
||||
h1 { font-size: 28px; line-height: 1.2; letter-spacing: -.025em; margin-top: 5px; }
|
||||
h2 { font-size: 16px; line-height: 1.25; letter-spacing: -.01em; margin-top: 3px; }
|
||||
.lede { color: var(--muted-foreground); margin-top: 8px; max-width: 680px; }
|
||||
.eyebrow { color: var(--muted-foreground); font-size: 11px; line-height: 1; font-weight: 600; text-transform: uppercase; letter-spacing: .05em; }
|
||||
.freshness { color: var(--muted-foreground); display: flex; align-items: center; gap: 8px; font-size: 12px; white-space: nowrap; }
|
||||
.status-dot { width: 7px; height: 7px; border-radius: 999px; background: var(--ring); display: inline-block; }
|
||||
.status-dot.healthy { background: var(--success); }
|
||||
.status-dot.unhealthy { background: var(--destructive); }
|
||||
.summary-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; margin-bottom: 12px; }
|
||||
.metric-card, .panel { border: 1px solid var(--border); border-radius: 14px; background: var(--card); }
|
||||
.metric-card { padding: 18px; min-height: 116px; }
|
||||
.metric-card .value { font-size: 29px; font-weight: 600; letter-spacing: -.035em; margin-top: 16px; }
|
||||
.metric-card .caption { color: var(--muted-foreground); font-size: 12px; margin-top: 3px; }
|
||||
.skeleton { background: linear-gradient(90deg, var(--card), var(--muted), var(--card)); background-size: 200% 100%; animation: shimmer 1.6s infinite; }
|
||||
@keyframes shimmer { to { background-position: -200% 0; } }
|
||||
.panel { padding: 18px; }
|
||||
.topology-panel { margin-bottom: 36px; }
|
||||
.panel-heading, .section-heading { display: flex; align-items: center; justify-content: space-between; gap: 16px; margin-bottom: 18px; }
|
||||
.section-heading { margin-top: 34px; }
|
||||
.topology { display: grid; grid-template-columns: 1fr 48px 1fr 48px 2fr; align-items: stretch; gap: 8px; }
|
||||
.topology-node { border: 1px solid var(--border); background: var(--background); border-radius: 10px; padding: 14px; min-width: 0; }
|
||||
.topology-node strong { display: block; font-size: 13px; }
|
||||
.topology-node span { display: block; color: var(--muted-foreground); font-size: 12px; margin-top: 4px; overflow: hidden; text-overflow: ellipsis; }
|
||||
.topology-cells { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 8px; }
|
||||
.connector { display: grid; place-items: center; color: var(--muted-foreground); }
|
||||
.connector::before { content: ''; width: 100%; height: 1px; background: var(--border); }
|
||||
.chart-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-bottom: 36px; }
|
||||
.chart-card { min-height: 220px; }
|
||||
.chart-head { display: flex; justify-content: space-between; align-items: start; }
|
||||
.chart-value { font-size: 22px; font-weight: 600; letter-spacing: -.03em; }
|
||||
.chart-unit { color: var(--muted-foreground); font-size: 11px; }
|
||||
.chart { width: 100%; height: 122px; margin-top: 18px; overflow: visible; }
|
||||
.chart path.area { fill: color-mix(in srgb, var(--foreground) 5%, transparent); }
|
||||
.chart path.line { fill: none; stroke: var(--foreground); stroke-width: 1.5; vector-effect: non-scaling-stroke; }
|
||||
.chart line { stroke: var(--border); stroke-width: 1; }
|
||||
.chart-empty { color: var(--muted-foreground); height: 120px; display: grid; place-items: center; font-size: 12px; }
|
||||
.two-column { display: grid; grid-template-columns: 1.7fr 1fr; gap: 12px; margin-bottom: 12px; }
|
||||
.three-column { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-bottom: 12px; }
|
||||
.two-column > *, .three-column > *, .chart-grid > * { min-width: 0; }
|
||||
.table-wrap { overflow-x: auto; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 12px; }
|
||||
th { color: var(--muted-foreground); font-weight: 500; text-align: left; padding: 0 10px 10px; }
|
||||
td { padding: 12px 10px; border-top: 1px solid var(--border); vertical-align: middle; }
|
||||
td:first-child, th:first-child { padding-left: 0; }
|
||||
td:last-child, th:last-child { padding-right: 0; }
|
||||
.mono { font-family: var(--font-mono); font-size: 11px; }
|
||||
.badge { display: inline-flex; align-items: center; border: 1px solid var(--border); border-radius: 999px; padding: 2px 7px; font-size: 11px; color: var(--muted-foreground); }
|
||||
.badge.healthy { color: var(--success); border-color: color-mix(in srgb, var(--success) 25%, var(--border)); }
|
||||
.badge.unhealthy { color: var(--destructive); border-color: color-mix(in srgb, var(--destructive) 25%, var(--border)); }
|
||||
.service-list, .list { display: grid; }
|
||||
.service-row, .list-row { padding: 11px 0; border-top: 1px solid var(--border); display: flex; align-items: center; justify-content: space-between; gap: 12px; min-width: 0; }
|
||||
.service-row:first-child, .list-row:first-child { border-top: 0; padding-top: 0; }
|
||||
.service-row:last-child, .list-row:last-child { padding-bottom: 0; }
|
||||
.row-title { font-size: 12px; font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.row-caption, .meta, .muted { color: var(--muted-foreground); font-size: 11px; }
|
||||
.panel-note { color: var(--muted-foreground); font-size: 11px; line-height: 1.45; margin: -8px 0 14px; }
|
||||
.panel-link:hover { color: var(--foreground); text-decoration: underline; }
|
||||
.row-caption { margin-top: 3px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
a { color: inherit; text-decoration: none; }
|
||||
a:hover .row-title { text-decoration: underline; }
|
||||
.cost-total { display: flex; align-items: baseline; gap: 7px; margin-bottom: 12px; }
|
||||
.cost-total strong { font-size: 28px; letter-spacing: -.035em; }
|
||||
.cost-line { display: flex; justify-content: space-between; gap: 10px; padding: 7px 0; border-top: 1px solid var(--border); font-size: 12px; }
|
||||
.cost-note { color: var(--muted-foreground); font-size: 11px; line-height: 1.5; margin-top: 12px; }
|
||||
.banner { border: 1px solid var(--border); border-radius: 10px; padding: 11px 13px; margin-bottom: 12px; font-size: 12px; }
|
||||
.banner.error { color: var(--destructive); }
|
||||
.banner.warning { color: var(--warning); }
|
||||
.hidden { display: none !important; }
|
||||
.power-actions { display: flex; align-items: center; gap: 8px; margin-top: 14px; flex-wrap: wrap; }
|
||||
footer { border-top: 1px solid var(--border); padding: 24px; color: var(--muted-foreground); font-size: 11px; text-align: center; }
|
||||
@media (max-width: 1050px) {
|
||||
.summary-grid, .chart-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
.three-column { grid-template-columns: 1fr; }
|
||||
.topology { grid-template-columns: 1fr; }
|
||||
.connector { height: 20px; }
|
||||
.connector::before { height: 100%; width: 1px; }
|
||||
}
|
||||
@media (max-width: 760px) {
|
||||
.topbar { height: auto; min-height: 64px; padding: 12px 16px; align-items: stretch; flex-direction: column; gap: 12px; }
|
||||
.brand div { display: grid; gap: 0; }
|
||||
.toolbar { flex-wrap: wrap; justify-content: flex-start; }
|
||||
main { padding: 28px 16px 56px; }
|
||||
.page-heading { align-items: flex-start; flex-direction: column; }
|
||||
.summary-grid, .chart-grid, .two-column { grid-template-columns: 1fr; }
|
||||
.topology-cells { grid-template-columns: 1fr; }
|
||||
table { min-width: 600px; }
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { buildCostModel } from './cost-model.js'
|
||||
import {
|
||||
RELAY_OPS_ENVIRONMENTS,
|
||||
relayOpsCellsFromTerraform,
|
||||
type RelayOpsCellConfig
|
||||
} from './environment-config.js'
|
||||
import type { ResourceInventory } from './resource-inventory.js'
|
||||
|
||||
const regionalCellsSource = `
|
||||
relay_gce_cells = {
|
||||
"staging-gce-c3" = {
|
||||
hostname = "c3"
|
||||
zone = "us-central1-a"
|
||||
machine_type = "e2-standard-2"
|
||||
capacity_requests = 4000
|
||||
}
|
||||
"staging-gce-c4" = {
|
||||
hostname = "c4"
|
||||
region = "asia-east2"
|
||||
zone = "asia-east2-a"
|
||||
machine_type = "e2-standard-4"
|
||||
capacity_requests = 6000
|
||||
database_pool_max = 10
|
||||
initially_enabled = false
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
function inventory(
|
||||
targetSize: number,
|
||||
activationPolicy: string,
|
||||
cells: RelayOpsCellConfig[] = RELAY_OPS_ENVIRONMENTS.staging.cells
|
||||
): ResourceInventory {
|
||||
return {
|
||||
director: null,
|
||||
auth: null,
|
||||
sql: { state: targetSize ? 'RUNNABLE' : 'STOPPED', activationPolicy, tier: 'db-custom-1-3840', availabilityType: 'ZONAL', databaseVersion: 'POSTGRES_17' },
|
||||
certificate: null,
|
||||
directorEndpoint: { health: null, ready: null, latencyMs: null },
|
||||
authEndpoint: { health: null, ready: null, latencyMs: null },
|
||||
cells: cells.map((cell) => ({
|
||||
...cell, migName: `mig-${cell.hostname}`, targetSize, runningInstances: targetSize,
|
||||
stable: true, template: 'template', imageDigest: null,
|
||||
backendHealth: targetSize ? 'healthy' : 'empty',
|
||||
endpoint: { health: null, ready: null, latencyMs: null }
|
||||
})),
|
||||
warnings: []
|
||||
}
|
||||
}
|
||||
|
||||
describe('buildCostModel', () => {
|
||||
it('shows the sleeping staging floor without VM or SQL compute', () => {
|
||||
const result = buildCostModel(RELAY_OPS_ENVIRONMENTS.staging, inventory(0, 'NEVER'))
|
||||
expect(result.kind).toBe('planning-estimate')
|
||||
expect(result.monthlyUsd).toBe(34)
|
||||
expect(result.lines.find((line) => line.label === 'Relay cell VMs')?.monthlyUsd).toBe(0)
|
||||
expect(result.actualBilling.available).toBe(false)
|
||||
expect(result.caveats[0]).toContain('not the Cloud Billing invoice')
|
||||
})
|
||||
|
||||
it('prices durable machine inventory and network floors by region', () => {
|
||||
const cells = relayOpsCellsFromTerraform({
|
||||
environment: 'staging',
|
||||
domain: 'relay-staging.onorca.dev',
|
||||
source: regionalCellsSource
|
||||
})
|
||||
const environment = { ...RELAY_OPS_ENVIRONMENTS.staging, cells }
|
||||
const result = buildCostModel(environment, inventory(1, 'NEVER', cells))
|
||||
const machines = result.lines.find((line) => line.label === 'Relay cell VMs')
|
||||
const network = result.lines.find((line) => line.label === 'Load balancer and network floor')
|
||||
|
||||
expect(machines).toEqual({
|
||||
label: 'Relay cell VMs',
|
||||
monthlyUsd: 185.79,
|
||||
basis: '2 configured VM cells at regional machine rates × 730 hours'
|
||||
})
|
||||
|
||||
expect(network).toEqual({
|
||||
label: 'Load balancer and network floor',
|
||||
monthlyUsd: 29,
|
||||
basis: 'shared HTTPS foundation plus NAT floor in 2 configured regions'
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,115 +0,0 @@
|
||||
import type {
|
||||
RelayOpsEnvironment,
|
||||
RelayOpsMachineType,
|
||||
RelayOpsRegion
|
||||
} from './environment-config.js'
|
||||
import type { ResourceInventory } from './resource-inventory.js'
|
||||
|
||||
export type CostLine = {
|
||||
label: string
|
||||
monthlyUsd: number
|
||||
basis: string
|
||||
}
|
||||
|
||||
export type CostModel = {
|
||||
kind: 'planning-estimate'
|
||||
monthlyUsd: number
|
||||
rangeUsd: [number, number]
|
||||
actualBilling: { available: false; reason: string }
|
||||
lines: CostLine[]
|
||||
caveats: string[]
|
||||
}
|
||||
|
||||
const HOURS_PER_MONTH = 730
|
||||
const MACHINE_HOURLY_USD: Record<
|
||||
RelayOpsRegion,
|
||||
Record<RelayOpsMachineType, number>
|
||||
> = {
|
||||
'us-central1': {
|
||||
'e2-standard-2': 0.06701142,
|
||||
'e2-standard-4': 0.13402284
|
||||
},
|
||||
'asia-east2': {
|
||||
'e2-standard-2': 0.0938,
|
||||
'e2-standard-4': 0.1875
|
||||
}
|
||||
}
|
||||
const SHARED_NETWORK_FLOOR_USD = 19
|
||||
const REGIONAL_NAT_FLOOR_USD = 5
|
||||
|
||||
function round(value: number): number {
|
||||
return Math.round(value * 100) / 100
|
||||
}
|
||||
|
||||
export function buildCostModel(
|
||||
environment: RelayOpsEnvironment,
|
||||
resources: ResourceInventory
|
||||
): CostModel {
|
||||
const runningCells = resources.cells.filter((cell) => (cell.targetSize ?? 0) > 0)
|
||||
const runningCellCount = runningCells.reduce((sum, cell) => sum + (cell.targetSize ?? 0), 0)
|
||||
const compute = runningCells.reduce(
|
||||
(sum, cell) =>
|
||||
sum + (cell.targetSize ?? 0) * MACHINE_HOURLY_USD[cell.region][cell.machineType],
|
||||
0
|
||||
) * HOURS_PER_MONTH
|
||||
const disks = runningCellCount * 30 * 0.1
|
||||
const sqlRunning = resources.sql?.activationPolicy === 'ALWAYS'
|
||||
const sql = sqlRunning ? (environment.id === 'production' ? 105 : 52) : 0
|
||||
const cloudRunMinimums =
|
||||
(resources.director?.minInstances ?? 0) + (resources.auth?.minInstances ?? 0)
|
||||
const cloudRun = cloudRunMinimums * 10
|
||||
const configuredRegions = new Set(
|
||||
(resources.cells.length > 0 ? resources.cells : environment.cells).map((cell) => cell.region)
|
||||
)
|
||||
const networkFoundation =
|
||||
SHARED_NETWORK_FLOOR_USD + configuredRegions.size * REGIONAL_NAT_FLOOR_USD
|
||||
const observability = environment.id === 'production' ? 12 : 5
|
||||
const lines: CostLine[] = [
|
||||
{
|
||||
label: 'Relay cell VMs',
|
||||
monthlyUsd: round(compute),
|
||||
basis: `${runningCellCount} configured VM cells at regional machine rates × 730 hours`
|
||||
},
|
||||
{
|
||||
label: 'Cell boot disks',
|
||||
monthlyUsd: round(disks),
|
||||
basis: `${runningCellCount} × 30 GB balanced persistent disk`
|
||||
},
|
||||
{
|
||||
label: 'Cloud SQL',
|
||||
monthlyUsd: sql,
|
||||
basis: sqlRunning ? `${resources.sql?.tier ?? 'configured tier'} active` : 'stopped'
|
||||
},
|
||||
{
|
||||
label: 'Cloud Run minimums',
|
||||
monthlyUsd: cloudRun,
|
||||
basis: `${cloudRunMinimums} configured minimum instances`
|
||||
},
|
||||
{
|
||||
label: 'Load balancer and network floor',
|
||||
monthlyUsd: networkFoundation,
|
||||
basis: `shared HTTPS foundation plus NAT floor in ${configuredRegions.size} configured region${configuredRegions.size === 1 ? '' : 's'}`
|
||||
},
|
||||
{
|
||||
label: 'Logs and monitoring allowance',
|
||||
monthlyUsd: observability,
|
||||
basis: 'planning allowance; varies with traffic and retention'
|
||||
}
|
||||
]
|
||||
const monthlyUsd = round(lines.reduce((sum, line) => sum + line.monthlyUsd, 0))
|
||||
return {
|
||||
kind: 'planning-estimate',
|
||||
monthlyUsd,
|
||||
rangeUsd: [round(monthlyUsd * 0.85), round(monthlyUsd * 1.3)],
|
||||
actualBilling: {
|
||||
available: false,
|
||||
reason: 'Cloud Billing export is not configured in either Relay project.'
|
||||
},
|
||||
lines,
|
||||
caveats: [
|
||||
'This is a modeled run-rate, not the Cloud Billing invoice.',
|
||||
'Network egress, actual request traffic, credits, discounts, taxes, and free tiers are excluded.',
|
||||
'Use the GCP Billing report for authoritative spend and forecasts.'
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { DashboardSnapshotCache } from './dashboard-snapshot.js'
|
||||
import type { DashboardSnapshot } from './dashboard-snapshot.js'
|
||||
import type { GcloudClient } from './gcloud-client.js'
|
||||
|
||||
const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
|
||||
|
||||
function snapshot(kind: 'good' | 'unavailable'): DashboardSnapshot {
|
||||
const good = kind === 'good'
|
||||
return {
|
||||
generatedAt: '2026-07-15T12:00:00.000Z',
|
||||
resources: {
|
||||
director: good ? {} : null,
|
||||
auth: good ? {} : null,
|
||||
sql: good ? {} : null,
|
||||
cells: [{ targetSize: good ? 1 : null }]
|
||||
},
|
||||
monitoring: {
|
||||
warnings: good
|
||||
? []
|
||||
: ['Cloud Monitoring credentials are unavailable. Run gcloud auth login.']
|
||||
},
|
||||
summary: { observedConnections: good ? 7 : 0 },
|
||||
warnings: good ? [] : ['Google Cloud credentials are unavailable. Run gcloud auth login.'],
|
||||
stale: false,
|
||||
staleReason: null
|
||||
} as unknown as DashboardSnapshot
|
||||
}
|
||||
|
||||
describe('DashboardSnapshotCache', () => {
|
||||
it('keeps the last good view when a later credential refresh fails', async () => {
|
||||
let calls = 0
|
||||
const cache = new DashboardSnapshotCache(gcloud, 0, async () => {
|
||||
calls += 1
|
||||
return snapshot(calls === 1 ? 'good' : 'unavailable')
|
||||
})
|
||||
|
||||
const first = await cache.read('production', 30)
|
||||
const second = await cache.read('production', 31)
|
||||
|
||||
expect(first.stale).toBe(false)
|
||||
expect(second.stale).toBe(true)
|
||||
expect(second.summary.observedConnections).toBe(7)
|
||||
expect(second.staleReason).toContain('credentials')
|
||||
})
|
||||
|
||||
it('keeps the last good view when a later collector throws', async () => {
|
||||
let calls = 0
|
||||
const cache = new DashboardSnapshotCache(gcloud, 0, async () => {
|
||||
calls += 1
|
||||
if (calls > 1) throw new Error('sensitive collector context')
|
||||
return snapshot('good')
|
||||
})
|
||||
|
||||
await cache.read('production', 30)
|
||||
const second = await cache.read('production', 31)
|
||||
|
||||
expect(second.stale).toBe(true)
|
||||
expect(second.summary.observedConnections).toBe(7)
|
||||
expect(JSON.stringify(second)).not.toContain('sensitive collector context')
|
||||
})
|
||||
})
|
||||
@@ -1,167 +0,0 @@
|
||||
import type { RelayOpsEnvironmentId } from './environment-config.js'
|
||||
import { relayOpsEnvironment } from './environment-config.js'
|
||||
import type { GcloudClient } from './gcloud-client.js'
|
||||
import { readRelayWorkflowRuns } from './github-runs.js'
|
||||
import { readMonitoringSnapshot } from './monitoring-snapshot.js'
|
||||
import { readResourceInventory } from './resource-inventory.js'
|
||||
import { buildCostModel } from './cost-model.js'
|
||||
|
||||
export type DashboardSnapshot = Awaited<ReturnType<typeof buildDashboardSnapshot>>
|
||||
|
||||
export async function buildDashboardSnapshot(
|
||||
environmentId: RelayOpsEnvironmentId,
|
||||
gcloud: GcloudClient,
|
||||
options: { windowMinutes?: number; fetchImpl?: typeof fetch; now?: Date } = {}
|
||||
) {
|
||||
const generatedAt = (options.now ?? new Date()).toISOString()
|
||||
const environment = relayOpsEnvironment(environmentId)
|
||||
const [monitoringResult, resourceResult, workflowResult] = await Promise.allSettled([
|
||||
readMonitoringSnapshot(environment, gcloud, {
|
||||
...(options.now === undefined ? {} : { now: options.now }),
|
||||
...(options.windowMinutes === undefined ? {} : { windowMinutes: options.windowMinutes }),
|
||||
...(options.fetchImpl === undefined ? {} : { fetchImpl: options.fetchImpl })
|
||||
}),
|
||||
readResourceInventory(environment, gcloud, options.fetchImpl),
|
||||
readRelayWorkflowRuns()
|
||||
])
|
||||
if (monitoringResult.status === 'rejected' || resourceResult.status === 'rejected') {
|
||||
const failed = [
|
||||
monitoringResult.status === 'rejected' ? 'Monitoring snapshot' : null,
|
||||
resourceResult.status === 'rejected' ? 'Resource inventory' : null
|
||||
].filter(Boolean)
|
||||
throw new Error(`${failed.join(' and ')} unavailable`)
|
||||
}
|
||||
const resources = resourceResult.value
|
||||
const monitoring = monitoringResult.value
|
||||
const warnings = [...resources.warnings, ...monitoring.warnings]
|
||||
const poweredDigests = new Set(
|
||||
resources.cells.filter((cell) => (cell.targetSize ?? 0) > 0).map((cell) => cell.imageDigest)
|
||||
)
|
||||
if (poweredDigests.has(null)) warnings.push('A powered cell image digest is unavailable.')
|
||||
if (poweredDigests.size > 1) warnings.push('Powered cells are not serving one immutable digest.')
|
||||
const expectedCertificateDomain = `*.${new URL(environment.cells[0]!.origin).hostname
|
||||
.split('.').slice(1).join('.')}`
|
||||
if (resources.certificate && !resources.certificate.domains.includes(expectedCertificateDomain)) {
|
||||
warnings.push('The Relay certificate domain does not match the configured cell domain.')
|
||||
}
|
||||
if (workflowResult.status === 'rejected') warnings.push('GitHub workflow history is unavailable.')
|
||||
const observedConnections = monitoring.metrics.total_connections.latest ?? 0
|
||||
const observedControls = monitoring.metrics.controls.latest ?? 0
|
||||
const observedSplices = monitoring.metrics.splices.latest ?? 0
|
||||
const configuredCapacity = environment.cells
|
||||
.filter((cell) => cell.configuredAdmission)
|
||||
.reduce((sum, cell) => sum + cell.capacityRequests, 0)
|
||||
const cellInventoryAvailable = resources.cells.every((cell) => cell.targetSize !== null)
|
||||
const poweredCapacity = cellInventoryAvailable
|
||||
? resources.cells
|
||||
.filter((cell) => (cell.targetSize ?? 0) > 0)
|
||||
.reduce((sum, cell) => sum + cell.capacityRequests, 0)
|
||||
: null
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
generatedAt,
|
||||
environment: {
|
||||
id: environment.id,
|
||||
label: environment.label,
|
||||
project: environment.project,
|
||||
region: environment.region,
|
||||
directorOrigin: environment.directorOrigin,
|
||||
authOrigin: environment.authOrigin,
|
||||
consoleLinks: {
|
||||
project: `https://console.cloud.google.com/home/dashboard?project=${environment.project}`,
|
||||
alerts: `https://console.cloud.google.com/monitoring/alerting?project=${environment.project}`,
|
||||
compute: `https://console.cloud.google.com/compute/instanceGroups/list?project=${environment.project}`
|
||||
}
|
||||
},
|
||||
summary: {
|
||||
observedConnections,
|
||||
observedControls,
|
||||
observedSplices,
|
||||
configuredCapacity,
|
||||
poweredCapacity,
|
||||
activeCells: cellInventoryAvailable
|
||||
? resources.cells.filter(
|
||||
(cell) => (cell.targetSize ?? 0) > 0 && cell.backendHealth === 'healthy'
|
||||
).length
|
||||
: null,
|
||||
totalCells: resources.cells.length
|
||||
},
|
||||
resources,
|
||||
monitoring,
|
||||
workflows: workflowResult.status === 'fulfilled' ? workflowResult.value : [],
|
||||
cost: buildCostModel(environment, resources),
|
||||
warnings,
|
||||
stale: false,
|
||||
staleReason: null as string | null
|
||||
}
|
||||
}
|
||||
|
||||
type SnapshotCacheEntry = { snapshot: DashboardSnapshot; expiresAt: number }
|
||||
type SnapshotBuilder = (
|
||||
environment: RelayOpsEnvironmentId,
|
||||
gcloud: GcloudClient,
|
||||
options: { windowMinutes?: number }
|
||||
) => Promise<DashboardSnapshot>
|
||||
|
||||
export class DashboardSnapshotCache {
|
||||
private readonly entries = new Map<string, SnapshotCacheEntry>()
|
||||
private readonly pending = new Map<string, Promise<DashboardSnapshot>>()
|
||||
private readonly lastGood = new Map<RelayOpsEnvironmentId, DashboardSnapshot>()
|
||||
|
||||
constructor(
|
||||
private readonly gcloud: GcloudClient,
|
||||
private readonly ttlMs = 30_000,
|
||||
private readonly builder: SnapshotBuilder = buildDashboardSnapshot
|
||||
) {}
|
||||
|
||||
async read(environment: RelayOpsEnvironmentId, windowMinutes: number): Promise<DashboardSnapshot> {
|
||||
const key = `${environment}:${windowMinutes}`
|
||||
const cached = this.entries.get(key)
|
||||
if (cached && cached.expiresAt > Date.now()) return cached.snapshot
|
||||
const existing = this.pending.get(key)
|
||||
if (existing) return await existing
|
||||
const request = this.builder(environment, this.gcloud, { windowMinutes })
|
||||
.then((snapshot) => {
|
||||
const coreInventoryUnavailable =
|
||||
snapshot.resources.director === null &&
|
||||
snapshot.resources.auth === null &&
|
||||
snapshot.resources.sql === null &&
|
||||
snapshot.resources.cells.every((cell) => cell.targetSize === null)
|
||||
const monitoringCredentialsUnavailable = snapshot.monitoring.warnings.some(
|
||||
(warning) => warning.includes('credentials are unavailable')
|
||||
)
|
||||
const lastGood = this.lastGood.get(environment)
|
||||
const result = (coreInventoryUnavailable || monitoringCredentialsUnavailable) && lastGood
|
||||
? {
|
||||
...lastGood,
|
||||
stale: true,
|
||||
staleReason: 'Local Google Cloud credentials are temporarily unavailable.',
|
||||
warnings: [...new Set([...lastGood.warnings, ...snapshot.warnings])]
|
||||
}
|
||||
: snapshot
|
||||
if (!coreInventoryUnavailable && !monitoringCredentialsUnavailable) {
|
||||
this.lastGood.set(environment, snapshot)
|
||||
}
|
||||
this.entries.set(key, { snapshot: result, expiresAt: Date.now() + this.ttlMs })
|
||||
return result
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
const lastGood = this.lastGood.get(environment)
|
||||
if (!lastGood) throw error
|
||||
const stale = {
|
||||
...lastGood,
|
||||
stale: true,
|
||||
staleReason: 'The latest operations refresh failed; showing the last good snapshot.',
|
||||
warnings: [...new Set([
|
||||
...lastGood.warnings,
|
||||
'The latest operations refresh failed before a complete snapshot was available.'
|
||||
])]
|
||||
}
|
||||
this.entries.set(key, { snapshot: stale, expiresAt: Date.now() + this.ttlMs })
|
||||
return stale
|
||||
})
|
||||
.finally(() => this.pending.delete(key))
|
||||
this.pending.set(key, request)
|
||||
return await request
|
||||
}
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
RELAY_OPS_ENVIRONMENTS,
|
||||
relayOpsCellsFromTerraform
|
||||
} from './environment-config.js'
|
||||
|
||||
const durableUsCell = `
|
||||
"production-gce-c26" = {
|
||||
hostname = "c26"
|
||||
zone = "us-central1-a"
|
||||
machine_type = "e2-standard-4"
|
||||
capacity_requests = 4000
|
||||
initially_enabled = false
|
||||
}
|
||||
`
|
||||
|
||||
const durableAsiaCells = `
|
||||
"production-gce-c27" = {
|
||||
hostname = "c27"
|
||||
region = "asia-east2"
|
||||
zone = "asia-east2-a"
|
||||
machine_type = "e2-standard-4"
|
||||
capacity_requests = 6000
|
||||
database_pool_max = 10
|
||||
initially_enabled = false
|
||||
}
|
||||
"production-gce-c28" = {
|
||||
hostname = "c28"
|
||||
region = "asia-east2"
|
||||
zone = "asia-east2-b"
|
||||
machine_type = "e2-standard-4"
|
||||
capacity_requests = 6000
|
||||
database_pool_max = 10
|
||||
initially_enabled = false
|
||||
}
|
||||
"production-gce-c29" = {
|
||||
hostname = "c29"
|
||||
region = "asia-east2"
|
||||
zone = "asia-east2-c"
|
||||
machine_type = "e2-standard-4"
|
||||
capacity_requests = 6000
|
||||
database_pool_max = 10
|
||||
initially_enabled = false
|
||||
}
|
||||
`
|
||||
|
||||
const durableAsiaSource = `
|
||||
relay_gce_cells = {${durableUsCell}${durableAsiaCells}}
|
||||
`
|
||||
|
||||
const durableUsOnlySource = `
|
||||
relay_gce_cells = {${durableUsCell}}
|
||||
`
|
||||
|
||||
// Why: relay-ops sat at 18 cells for four days after C19-C22 shipped, which threw
|
||||
// `selector membership must contain every configured cell exactly once` and blocked
|
||||
// every production mutation. Reading Terraform here makes that drift fail the build.
|
||||
function terraformCells(environment: 'production' | 'staging'): Array<{
|
||||
cellId: string
|
||||
region: string
|
||||
zone: string
|
||||
machineType: string
|
||||
capacityRequests: number
|
||||
databasePoolMax: number
|
||||
connectionHardCap: number | null
|
||||
}> {
|
||||
const tfvars = readFileSync(
|
||||
fileURLToPath(new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url)),
|
||||
'utf8'
|
||||
)
|
||||
const block = /relay_gce_cells\s*=\s*\{([\s\S]*)\n\}/.exec(tfvars)?.[1] ?? ''
|
||||
return [...block.matchAll(/"([a-z]+-gce-c\d+)"\s*=\s*\{([\s\S]*?)\n {2}\}/g)]
|
||||
.map((match) => ({
|
||||
cellId: match[1] ?? '',
|
||||
region: /region\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? 'us-central1',
|
||||
zone: /zone\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '',
|
||||
machineType: /machine_type\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '',
|
||||
capacityRequests: Number(/capacity_requests\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1]),
|
||||
databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10),
|
||||
connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))(
|
||||
/connection_hard_cap\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1]
|
||||
)
|
||||
}))
|
||||
.sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId))
|
||||
}
|
||||
|
||||
const cellOrdinal = (cellId: string): number => Number(/c(\d+)$/.exec(cellId)?.[1] ?? 0)
|
||||
|
||||
describe('relay operations environment config', () => {
|
||||
it.each(['production', 'staging'] as const)(
|
||||
'matches the %s cells Terraform actually provisions',
|
||||
(environment) => {
|
||||
const expected = terraformCells(environment)
|
||||
expect(expected.length).toBeGreaterThan(0)
|
||||
expect(
|
||||
RELAY_OPS_ENVIRONMENTS[environment].cells.map((cell) => ({
|
||||
cellId: cell.cellId,
|
||||
region: cell.region,
|
||||
zone: cell.zone,
|
||||
machineType: cell.machineType,
|
||||
capacityRequests: cell.capacityRequests,
|
||||
databasePoolMax: cell.databasePoolMax,
|
||||
connectionHardCap: cell.connectionHardCap
|
||||
}))
|
||||
).toEqual(expected)
|
||||
}
|
||||
)
|
||||
|
||||
it('derives hostname and origin from the cell ordinal', () => {
|
||||
const cells = RELAY_OPS_ENVIRONMENTS.production.cells
|
||||
expect(cells.at(-1)).toMatchObject({
|
||||
hostname: `c${cells.length}`,
|
||||
origin: `https://c${cells.length}.relay.onorca.dev`
|
||||
})
|
||||
})
|
||||
|
||||
it('inventories Asia cells only when they exist in durable Terraform', () => {
|
||||
const cells = relayOpsCellsFromTerraform({
|
||||
environment: 'production',
|
||||
domain: 'relay.onorca.dev',
|
||||
source: durableAsiaSource
|
||||
})
|
||||
|
||||
expect(cells.slice(1)).toEqual([
|
||||
{
|
||||
cellId: 'production-gce-c27', hostname: 'c27', origin: 'https://c27.relay.onorca.dev',
|
||||
region: 'asia-east2', zone: 'asia-east2-a', machineType: 'e2-standard-4',
|
||||
capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null,
|
||||
configuredAdmission: false
|
||||
},
|
||||
{
|
||||
cellId: 'production-gce-c28', hostname: 'c28', origin: 'https://c28.relay.onorca.dev',
|
||||
region: 'asia-east2', zone: 'asia-east2-b', machineType: 'e2-standard-4',
|
||||
capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null,
|
||||
configuredAdmission: false
|
||||
},
|
||||
{
|
||||
cellId: 'production-gce-c29', hostname: 'c29', origin: 'https://c29.relay.onorca.dev',
|
||||
region: 'asia-east2', zone: 'asia-east2-c', machineType: 'e2-standard-4',
|
||||
capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null,
|
||||
configuredAdmission: false
|
||||
}
|
||||
])
|
||||
|
||||
const usOnlyCells = relayOpsCellsFromTerraform({
|
||||
environment: 'production',
|
||||
domain: 'relay.onorca.dev',
|
||||
source: durableUsOnlySource
|
||||
})
|
||||
expect(usOnlyCells).toHaveLength(1)
|
||||
expect(usOnlyCells.some((cell) => cell.region === 'asia-east2')).toBe(false)
|
||||
expect(usOnlyCells.some((cell) => cell.cellId === 'production-gce-c27')).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -1,130 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { z } from 'zod'
|
||||
|
||||
export type RelayOpsEnvironmentId = 'production' | 'staging'
|
||||
export type RelayOpsRegion = 'us-central1' | 'asia-east2'
|
||||
export type RelayOpsMachineType = 'e2-standard-2' | 'e2-standard-4'
|
||||
|
||||
export type RelayOpsCellConfig = {
|
||||
cellId: string
|
||||
hostname: string
|
||||
origin: string
|
||||
region: RelayOpsRegion
|
||||
zone: string
|
||||
machineType: RelayOpsMachineType
|
||||
capacityRequests: number
|
||||
databasePoolMax: number
|
||||
// The cell's rendered connectionHardCap; null when Terraform leaves it unset.
|
||||
connectionHardCap: number | null
|
||||
configuredAdmission: boolean
|
||||
}
|
||||
|
||||
export type RelayOpsEnvironment = {
|
||||
id: RelayOpsEnvironmentId
|
||||
label: string
|
||||
project: string
|
||||
region: string
|
||||
directorOrigin: string
|
||||
authOrigin: string
|
||||
directorService: string
|
||||
authService: string
|
||||
sqlInstance: string
|
||||
migPrefix: string
|
||||
certificateName: string
|
||||
cells: RelayOpsCellConfig[]
|
||||
}
|
||||
|
||||
const RegionSchema = z.enum(['us-central1', 'asia-east2'])
|
||||
const MachineTypeSchema = z.enum(['e2-standard-2', 'e2-standard-4'])
|
||||
const EnvironmentSchema = z.enum(['production', 'staging'])
|
||||
|
||||
function cellOrdinal(cellId: string): number {
|
||||
return Number(/c(\d+)$/.exec(cellId)?.[1] ?? 0)
|
||||
}
|
||||
|
||||
function required(body: string, pattern: RegExp, label: string): string {
|
||||
const value = pattern.exec(body)?.[1]
|
||||
if (!value) throw new Error(`Relay Ops could not read ${label} from durable Terraform config`)
|
||||
return value
|
||||
}
|
||||
|
||||
export function relayOpsCellsFromTerraform(input: {
|
||||
environment: RelayOpsEnvironmentId
|
||||
domain: string
|
||||
source: string
|
||||
}): RelayOpsCellConfig[] {
|
||||
const block = /relay_gce_cells\s*=\s*\{([\s\S]*)\n\}/.exec(input.source)?.[1]
|
||||
if (!block) throw new Error('Relay Ops could not read durable Relay cells')
|
||||
return [...block.matchAll(/"([a-z]+-gce-c\d+)"\s*=\s*\{([\s\S]*?)\n {2}\}/g)]
|
||||
.map((match) => {
|
||||
const cellId = match[1]!
|
||||
const body = match[2]!
|
||||
const hostname = required(body, /\bhostname\s*=\s*"([^"]+)"/, `${cellId} hostname`)
|
||||
const configuredAdmission = /\binitially_enabled\s*=\s*(true|false)/.exec(body)?.[1]
|
||||
return {
|
||||
cellId,
|
||||
hostname,
|
||||
origin: `https://${hostname}.${input.domain}`,
|
||||
region: RegionSchema.parse(
|
||||
/\bregion\s*=\s*"([^"]+)"/.exec(body)?.[1] ?? 'us-central1'
|
||||
),
|
||||
zone: required(body, /\bzone\s*=\s*"([^"]+)"/, `${cellId} zone`),
|
||||
machineType: MachineTypeSchema.parse(
|
||||
required(body, /\bmachine_type\s*=\s*"([^"]+)"/, `${cellId} machine type`)
|
||||
),
|
||||
capacityRequests: Number(
|
||||
required(body, /\bcapacity_requests\s*=\s*(\d+)/, `${cellId} capacity`)
|
||||
),
|
||||
databasePoolMax: Number(/\bdatabase_pool_max\s*=\s*(\d+)/.exec(body)?.[1] ?? 10),
|
||||
connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))(
|
||||
/\bconnection_hard_cap\s*=\s*(\d+)/.exec(body)?.[1]
|
||||
),
|
||||
configuredAdmission: configuredAdmission === undefined || configuredAdmission === 'true'
|
||||
}
|
||||
})
|
||||
.sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId))
|
||||
}
|
||||
|
||||
function durableCells(environment: RelayOpsEnvironmentId, domain: string): RelayOpsCellConfig[] {
|
||||
const source = readFileSync(
|
||||
// Repository root; infra/terraform moves with this tree, so the relative depth holds.
|
||||
new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
return relayOpsCellsFromTerraform({ environment, domain, source })
|
||||
}
|
||||
|
||||
export const RELAY_OPS_ENVIRONMENTS: Record<RelayOpsEnvironmentId, RelayOpsEnvironment> = {
|
||||
production: {
|
||||
id: 'production',
|
||||
label: 'Production',
|
||||
project: 'onorca-cloud',
|
||||
region: 'us-central1',
|
||||
directorOrigin: 'https://relay.onorca.dev',
|
||||
authOrigin: 'https://login.onorca.dev',
|
||||
directorService: 'orca-cloud-relay',
|
||||
authService: 'orca-cloud-auth',
|
||||
sqlInstance: 'orca-cloud-auth-db',
|
||||
migPrefix: 'orca-cloud-relay-gce-',
|
||||
certificateName: 'orca-cloud-relay-gce',
|
||||
cells: durableCells('production', 'relay.onorca.dev')
|
||||
},
|
||||
staging: {
|
||||
id: 'staging',
|
||||
label: 'Staging',
|
||||
project: 'onorca-cloud-staging',
|
||||
region: 'us-central1',
|
||||
directorOrigin: 'https://relay-staging.onorca.dev',
|
||||
authOrigin: 'https://auth-staging.onorca.dev',
|
||||
directorService: 'orca-cloud-relay-staging',
|
||||
authService: 'orca-cloud-auth-staging',
|
||||
sqlInstance: 'orca-cloud-staging-auth-db',
|
||||
migPrefix: 'orca-cloud-staging-relay-gce-',
|
||||
certificateName: 'orca-cloud-staging-relay-gce',
|
||||
cells: durableCells('staging', 'relay-staging.onorca.dev')
|
||||
}
|
||||
}
|
||||
|
||||
export function relayOpsEnvironment(value: unknown): RelayOpsEnvironment {
|
||||
return RELAY_OPS_ENVIRONMENTS[EnvironmentSchema.parse(value)]
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createGcloudClient } from './gcloud-client.js'
|
||||
|
||||
describe('createGcloudClient', () => {
|
||||
it('shares and caches one credential refresh across concurrent readers', async () => {
|
||||
let calls = 0
|
||||
const token = 'a'.repeat(40)
|
||||
const client = createGcloudClient(async () => {
|
||||
calls += 1
|
||||
await Promise.resolve()
|
||||
return token
|
||||
})
|
||||
|
||||
const values = await Promise.all([
|
||||
client.accessToken(),
|
||||
client.accessToken(),
|
||||
client.accessToken()
|
||||
])
|
||||
|
||||
expect(values).toEqual([token, token, token])
|
||||
expect(await client.accessToken()).toBe(token)
|
||||
expect(calls).toBe(1)
|
||||
})
|
||||
|
||||
it('caches bounded identity tokens by audience', async () => {
|
||||
const commands: string[][] = []
|
||||
const token = 'aaa.bbb.ccc'
|
||||
const client = createGcloudClient(async (args) => {
|
||||
commands.push(args)
|
||||
return token
|
||||
})
|
||||
await expect(client.identityToken?.('https://relay.example/admin')).resolves.toBe(token)
|
||||
await expect(client.identityToken?.('https://relay.example/admin')).resolves.toBe(token)
|
||||
expect(commands).toEqual([
|
||||
[
|
||||
'auth',
|
||||
'print-identity-token',
|
||||
'--audiences=https://relay.example/admin',
|
||||
'--include-email'
|
||||
]
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -1,77 +0,0 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import { promisify } from 'node:util'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const TOKEN_PATTERN = /^[A-Za-z0-9._~+\/-]{32,8192}$/
|
||||
|
||||
export type GcloudClient = {
|
||||
accessToken(): Promise<string>
|
||||
identityToken?(audience: string): Promise<string>
|
||||
}
|
||||
|
||||
export class GcloudCommandError extends Error {
|
||||
constructor(readonly operation: string) {
|
||||
super(`${operation} is unavailable`)
|
||||
}
|
||||
}
|
||||
|
||||
async function runGcloud(args: string[]): Promise<string> {
|
||||
try {
|
||||
const result = await execFileAsync('gcloud', args, {
|
||||
encoding: 'utf8',
|
||||
timeout: 90_000,
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
env: {
|
||||
...process.env,
|
||||
CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK: '1',
|
||||
CLOUDSDK_CORE_DISABLE_PROMPTS: '1',
|
||||
CLOUDSDK_CORE_DISABLE_USAGE_REPORTING: '1'
|
||||
}
|
||||
})
|
||||
return result.stdout.trim()
|
||||
} catch {
|
||||
// Gcloud stderr can echo command context; keep dashboard errors intentionally non-sensitive.
|
||||
throw new GcloudCommandError(`gcloud ${args.slice(0, 3).join(' ')}`)
|
||||
}
|
||||
}
|
||||
|
||||
export function createGcloudClient(
|
||||
tokenCommand: (args: string[]) => Promise<string> = runGcloud
|
||||
): GcloudClient {
|
||||
let cachedToken: { value: string; expiresAt: number } | null = null
|
||||
const identityTokens = new Map<string, { value: string; expiresAt: number }>()
|
||||
let pendingToken: Promise<string> | null = null
|
||||
return {
|
||||
async accessToken(): Promise<string> {
|
||||
if (cachedToken && cachedToken.expiresAt > Date.now()) return cachedToken.value
|
||||
if (pendingToken) return await pendingToken
|
||||
// One refresh avoids concurrent gcloud processes contending on the local credential store.
|
||||
pendingToken = tokenCommand(['auth', 'print-access-token'])
|
||||
.then((token) => {
|
||||
if (!TOKEN_PATTERN.test(token)) throw new GcloudCommandError('gcloud access token')
|
||||
cachedToken = { value: token, expiresAt: Date.now() + 5 * 60_000 }
|
||||
return token
|
||||
})
|
||||
.finally(() => { pendingToken = null })
|
||||
return await pendingToken
|
||||
},
|
||||
async identityToken(audience: string): Promise<string> {
|
||||
const cached = identityTokens.get(audience)
|
||||
if (cached && cached.expiresAt > Date.now()) return cached.value
|
||||
const token = await tokenCommand([
|
||||
'auth',
|
||||
'print-identity-token',
|
||||
`--audiences=${audience}`,
|
||||
'--include-email'
|
||||
])
|
||||
if (
|
||||
token.length > 8_192 ||
|
||||
!/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)
|
||||
) {
|
||||
throw new GcloudCommandError('gcloud identity token')
|
||||
}
|
||||
identityTokens.set(audience, { value: token, expiresAt: Date.now() + 5 * 60_000 })
|
||||
return token
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import { promisify } from 'node:util'
|
||||
import { z } from 'zod'
|
||||
import { relayRepositoryApiPath } from './relay-repository.js'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
|
||||
const WorkflowRunSchema = z.object({
|
||||
id: z.number().int().positive(),
|
||||
name: z.string(),
|
||||
event: z.string(),
|
||||
status: z.string(),
|
||||
conclusion: z.string().nullable(),
|
||||
head_sha: z.string(),
|
||||
html_url: z.string().url(),
|
||||
created_at: z.string(),
|
||||
updated_at: z.string()
|
||||
})
|
||||
|
||||
const WorkflowRunsSchema = z.object({
|
||||
workflow_runs: z.array(WorkflowRunSchema)
|
||||
})
|
||||
|
||||
export type RelayWorkflowRun = {
|
||||
id: number
|
||||
name: string
|
||||
status: string
|
||||
conclusion: string | null
|
||||
headSha: string
|
||||
url: string
|
||||
createdAt: string
|
||||
updatedAt: string
|
||||
}
|
||||
|
||||
const RELAY_WORKFLOW_PATTERN = /(Relay|Auth|Power)/i
|
||||
|
||||
export async function readRelayWorkflowRuns(): Promise<RelayWorkflowRun[]> {
|
||||
let stdout: string
|
||||
try {
|
||||
const result = await execFileAsync(
|
||||
'gh',
|
||||
[
|
||||
'api',
|
||||
'--method',
|
||||
'GET',
|
||||
relayRepositoryApiPath('actions/runs'),
|
||||
'-f',
|
||||
'per_page=100'
|
||||
],
|
||||
{ encoding: 'utf8', timeout: 30_000, maxBuffer: 4 * 1024 * 1024 }
|
||||
)
|
||||
stdout = result.stdout
|
||||
} catch {
|
||||
throw new Error('GitHub workflow history is unavailable')
|
||||
}
|
||||
const runs = WorkflowRunsSchema.parse(JSON.parse(stdout) as unknown).workflow_runs
|
||||
const counts = new Map<string, number>()
|
||||
return runs
|
||||
.filter((run) => {
|
||||
if (!RELAY_WORKFLOW_PATTERN.test(run.name)) return false
|
||||
const count = counts.get(run.name) ?? 0
|
||||
if (count >= 2) return false
|
||||
counts.set(run.name, count + 1)
|
||||
return true
|
||||
})
|
||||
.slice(0, 12)
|
||||
.map((run) => ({
|
||||
id: run.id,
|
||||
name: run.name,
|
||||
status: run.status,
|
||||
conclusion: run.conclusion,
|
||||
headSha: run.head_sha.slice(0, 8),
|
||||
url: run.html_url,
|
||||
createdAt: run.created_at,
|
||||
updatedAt: run.updated_at
|
||||
}))
|
||||
}
|
||||
@@ -1,874 +0,0 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
livePreflightGcloud,
|
||||
runIncidentLivePreflight
|
||||
} from './incident-live-preflight-cli.js'
|
||||
import {
|
||||
INCIDENT_MONITOR_THRESHOLDS,
|
||||
type IncidentSample
|
||||
} from './incident-monitor.js'
|
||||
import { relayOpsEnvironment } from './environment-config.js'
|
||||
import type { AdmissionSelector } from './incident-selector.js'
|
||||
|
||||
const directories: string[] = []
|
||||
const now = Date.parse('2026-07-28T12:00:00.000Z')
|
||||
const selector = {
|
||||
generation: 1,
|
||||
membership: {
|
||||
existingOnly: ['production-gce-c1'],
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
}
|
||||
|
||||
function stateFile(
|
||||
migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition' = 'strict',
|
||||
overrides: Record<string, unknown> = {}
|
||||
): string {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-live-preflight-'))
|
||||
directories.push(directory)
|
||||
const path = join(directory, 'state.json')
|
||||
const expectedSelector = migrationPolicy === 'capacity-transition'
|
||||
? {
|
||||
generation: 1,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: [],
|
||||
general: ['production-gce-c1']
|
||||
}
|
||||
}
|
||||
: selector
|
||||
writeFileSync(path, JSON.stringify({
|
||||
schemaVersion: 4,
|
||||
environment: 'production',
|
||||
expectedSelector,
|
||||
migrationPolicy,
|
||||
recoverySourceCellId:
|
||||
migrationPolicy === 'recover-forward' ? 'production-gce-c1' : null,
|
||||
capacityCellId:
|
||||
migrationPolicy === 'capacity-transition' ? 'production-gce-c1' : null,
|
||||
preDrainDryRun: true,
|
||||
startedAt: new Date(now - 17 * 60_000).toISOString(),
|
||||
windowStartedAt: new Date(now - 16 * 60_000).toISOString(),
|
||||
durationMinutes: 15,
|
||||
intervalMs: 60_000,
|
||||
sampleCount: 16,
|
||||
lastSampleAt: new Date(now - 60_007).toISOString(),
|
||||
frozenAt: null,
|
||||
completedAt: new Date(now - 60_000).toISOString(),
|
||||
...overrides
|
||||
}))
|
||||
return path
|
||||
}
|
||||
|
||||
// Every configured production cell, in the lexicographic order
|
||||
// normalizeSelectorMembership canonicalises to. Derived from the same durable
|
||||
// Terraform config the override path reads, so a new cell cannot strand these.
|
||||
const configuredCellIds = relayOpsEnvironment('production').cells.map(
|
||||
(cell) => cell.cellId
|
||||
)
|
||||
const canonicalCellIds = [...configuredCellIds].sort()
|
||||
const canonicalMembership = {
|
||||
existingOnly: canonicalCellIds,
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
|
||||
// What the director reports: a normalised selector, never an echo of what the
|
||||
// caller expected. An order-sensitive comparison only holds if the override path
|
||||
// canonicalises its own input the same way.
|
||||
function canonicalSample(generation = 1): IncidentSample {
|
||||
const next = sample()
|
||||
next.selector = { generation, membership: canonicalMembership }
|
||||
return next
|
||||
}
|
||||
|
||||
function membershipFile(
|
||||
membership: Record<string, string[]> = canonicalMembership
|
||||
): string {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-live-preflight-selector-'))
|
||||
directories.push(directory)
|
||||
const path = join(directory, 'selector.json')
|
||||
writeFileSync(path, JSON.stringify(membership))
|
||||
return path
|
||||
}
|
||||
|
||||
function sample(): IncidentSample {
|
||||
const observedAt = new Date(now).toISOString()
|
||||
const signal = (value: number) => ({ value, observedAt })
|
||||
return {
|
||||
collectedAt: observedAt,
|
||||
selector,
|
||||
expectedSelector: selector,
|
||||
cells: [{
|
||||
cellId: 'production-gce-c1',
|
||||
region: 'us-central1',
|
||||
runtimeKnown: true,
|
||||
powered: true,
|
||||
expectedAdmissionState: 'existing-only'
|
||||
}],
|
||||
sources: {
|
||||
'active-probe': {
|
||||
observedAt,
|
||||
signals: {
|
||||
'director.health': signal(1),
|
||||
'director.ready': signal(1),
|
||||
'director.latency_ms': signal(1),
|
||||
'auth.health': signal(1),
|
||||
'auth.ready': signal(1),
|
||||
'auth.latency_ms': signal(1),
|
||||
'cell.production-gce-c1.health': signal(1),
|
||||
'cell.production-gce-c1.ready': signal(1),
|
||||
'cell.production-gce-c1.latency_ms': signal(1)
|
||||
}
|
||||
},
|
||||
'cloud-monitoring': {
|
||||
observedAt,
|
||||
signals: {
|
||||
'cloud_sql.cpu': signal(0.1),
|
||||
'cloud_sql.memory': signal(0.1),
|
||||
'cloud_sql.backends': signal(1),
|
||||
'cloud_sql.lock_waits': signal(0),
|
||||
'cloud_sql.deadlocks': signal(0),
|
||||
'director.instances': signal(5),
|
||||
'director.cpu': signal(0.1),
|
||||
'director.memory': signal(0.1),
|
||||
'director.concurrency': signal(1),
|
||||
'director.errors': signal(0),
|
||||
'auth.errors': signal(0)
|
||||
}
|
||||
},
|
||||
'relay-logs': {
|
||||
observedAt,
|
||||
signals: {
|
||||
'relay.pool_waiting': signal(0),
|
||||
'relay.pool_wait_ms': signal(0),
|
||||
'relay.postgres_retries': signal(0),
|
||||
'relay.postgres_retry_exhausted': signal(0),
|
||||
'cell.production-gce-c1.connections': signal(1),
|
||||
'cell.production-gce-c1.queued_bytes': signal(0)
|
||||
}
|
||||
},
|
||||
'director-admin': {
|
||||
observedAt,
|
||||
signals: {
|
||||
'cell.production-gce-c1.admission_state': signal(0),
|
||||
'cell.production-gce-c1.heartbeat_fresh': signal(1),
|
||||
'cell.production-gce-c1.heartbeat_age_ms': signal(1),
|
||||
'cell.production-gce-c1.migration_blocked': signal(0),
|
||||
'cell.production-gce-c1.migration_target_inactive': signal(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const directory of directories.splice(0)) {
|
||||
rmSync(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
describe('relay incident live preflight', () => {
|
||||
it('accepts the package-manager argument separator', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--', '--state-file', stateFile()],
|
||||
{ now: () => now, collect: async () => sample() }
|
||||
)).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
it('accepts one complete fresh green sample', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: async () => sample() }
|
||||
)).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
it('rejects monitor evidence beyond the 35-minute lineage bound', async () => {
|
||||
const path = stateFile('strict', {
|
||||
startedAt: new Date(now - 36 * 60_000 - 1).toISOString()
|
||||
})
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', path],
|
||||
{ now: () => now, collect: async () => sample() }
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
})
|
||||
|
||||
it('scales the evidence age bound by same-cap wave index', async () => {
|
||||
const agedState = (ageMs: number) => stateFile('strict', {
|
||||
startedAt: new Date(now - ageMs - 17 * 60_000).toISOString(),
|
||||
windowStartedAt: new Date(now - ageMs - 16 * 60_000).toISOString(),
|
||||
lastSampleAt: new Date(now - ageMs - 7).toISOString(),
|
||||
completedAt: new Date(now - ageMs).toISOString()
|
||||
})
|
||||
const deps = { now: () => now, collect: async () => sample() }
|
||||
// One predecessor cell roll (~16 min) exceeds wave 0 but fits wave 1.
|
||||
const oneRollOld = agedState(17 * 60_000)
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', oneRollOld], deps
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', oneRollOld, '--wave-index', '0'], deps
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', oneRollOld, '--wave-index', '1'], deps
|
||||
)).resolves.toBeUndefined()
|
||||
// Wave 0 edges: the 10-minute bound covers same-cap job start-up latency.
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(10 * 60_000), '--wave-index', '0'], deps
|
||||
)).resolves.toBeUndefined()
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(10 * 60_000 + 1), '--wave-index', '0'], deps
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
// Both edges of one predecessor job timeout: 10min + 75min exactly.
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(85 * 60_000), '--wave-index', '1'], deps
|
||||
)).resolves.toBeUndefined()
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(85 * 60_000 + 1), '--wave-index', '1'], deps
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(160 * 60_000), '--wave-index', '2'], deps
|
||||
)).resolves.toBeUndefined()
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(160 * 60_000 + 1), '--wave-index', '2'], deps
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(235 * 60_000), '--wave-index', '3'], deps
|
||||
)).resolves.toBeUndefined()
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(235 * 60_000 + 1), '--wave-index', '3'], deps
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
// The last cell of a ten-cell same-cap batch: 10min + 9 * 75min exactly.
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(685 * 60_000), '--wave-index', '9'], deps
|
||||
)).resolves.toBeUndefined()
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(685 * 60_000 + 1), '--wave-index', '9'], deps
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
// The wave index is a strict single-use 0-9 argument.
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile(), '--wave-index', '10'], deps
|
||||
)).rejects.toThrow('usage:')
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile(), '--wave-index', ''], deps
|
||||
)).rejects.toThrow('usage:')
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile(), '--wave-index', '1', '--wave-index', '1'],
|
||||
deps
|
||||
)).rejects.toThrow('usage:')
|
||||
})
|
||||
|
||||
it('expects the wave-adjusted live selector generation', async () => {
|
||||
const agedPath = stateFile('strict', {
|
||||
startedAt: new Date(now - 34 * 60_000).toISOString(),
|
||||
windowStartedAt: new Date(now - 33 * 60_000).toISOString(),
|
||||
lastSampleAt: new Date(now - 17 * 60_000 - 7).toISOString(),
|
||||
completedAt: new Date(now - 17 * 60_000).toISOString()
|
||||
})
|
||||
const liveAt = (generation: number) =>
|
||||
async (expectedSelector: AdmissionSelector) => ({
|
||||
...sample(),
|
||||
selector: { ...selector, generation },
|
||||
expectedSelector
|
||||
})
|
||||
// One predecessor roll advanced the live selector by exactly 2.
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedPath, '--wave-index', '1'],
|
||||
{ now: () => now, collect: liveAt(selector.generation + 2) }
|
||||
)).resolves.toBeUndefined()
|
||||
// The sealed pre-roll generation must no longer satisfy wave 1.
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedPath, '--wave-index', '1'],
|
||||
{ now: () => now, collect: liveAt(selector.generation) }
|
||||
)).rejects.toThrow('director-admin/selector_mismatch')
|
||||
// Wave 0 still expects the sealed generation itself.
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: liveAt(selector.generation) }
|
||||
)).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
it('fails closed on a live threshold breach', async () => {
|
||||
const unhealthy = sample()
|
||||
unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: async () => unhealthy }
|
||||
)).rejects.toThrow('cloud-monitoring/threshold_max')
|
||||
})
|
||||
|
||||
// Why: a frozen wave has to name what froze it without re-reading the sample.
|
||||
it('names the signal and its numbers in the failure message', async () => {
|
||||
const slowCell = sample()
|
||||
slowCell.sources['active-probe']!.signals[
|
||||
'cell.production-gce-c1.latency_ms'
|
||||
]!.value = 2_568
|
||||
// A cell probe now re-samples before it fails a wave, so the wait is injected;
|
||||
// this cell stays slow on every sample and still names what stopped it.
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: async () => slowCell, wait: async () => {} }
|
||||
)).rejects.toThrow(
|
||||
'relay live preflight failed: active-probe/threshold_max cell.production-gce-c1.latency_ms observed=2568 threshold=2000'
|
||||
)
|
||||
|
||||
// A failure with no signal keeps the source/code token and drops the rest.
|
||||
const stale = sample()
|
||||
stale.sources['active-probe']!.observedAt = new Date(now - 60_001).toISOString()
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: async () => stale }
|
||||
)).rejects.toThrow(
|
||||
'relay live preflight failed: active-probe/source_stale observed=60001 threshold=60000'
|
||||
)
|
||||
})
|
||||
|
||||
// Why: this one sample decides a mutating wave, so an Asia cell's ~30 s
|
||||
// "no healthy upstream" window could still fail a wave here even after the
|
||||
// 15-minute gate learned to ride it out.
|
||||
describe('cell probe tolerance', () => {
|
||||
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
|
||||
// Serves `badSamples` unhealthy cell readings, then healthy ones.
|
||||
const downThen = (badSamples: number) => {
|
||||
let index = 0
|
||||
return async () => {
|
||||
const next = sample()
|
||||
if (index++ < badSamples) {
|
||||
next.sources['active-probe']!.signals['cell.production-gce-c1.health']!
|
||||
.value = 0
|
||||
next.sources['active-probe']!.signals['cell.production-gce-c1.ready']!
|
||||
.value = 0
|
||||
}
|
||||
return next
|
||||
}
|
||||
}
|
||||
|
||||
it('re-samples through a probe outage within the tolerance', async () => {
|
||||
const waits: number[] = []
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{
|
||||
now: () => now,
|
||||
collect: downThen(tolerance),
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)).resolves.toBeUndefined()
|
||||
expect(waits).toHaveLength(tolerance)
|
||||
})
|
||||
|
||||
it('fails the wave once the probe outage outlasts the tolerance', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{
|
||||
now: () => now,
|
||||
collect: downThen(tolerance + 1),
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow('active-probe/threshold_equal cell.production-gce-c1.health')
|
||||
})
|
||||
|
||||
it('does not re-sample a director probe failure', async () => {
|
||||
let samples = 0
|
||||
const down = async () => {
|
||||
samples++
|
||||
const next = sample()
|
||||
next.sources['active-probe']!.signals['director.health']!.value = 0
|
||||
return next
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: down, wait: async () => {} }
|
||||
)).rejects.toThrow('active-probe/threshold_equal director.health')
|
||||
expect(samples).toBe(1)
|
||||
})
|
||||
|
||||
it('does not re-sample a non-probe threshold failure', async () => {
|
||||
let samples = 0
|
||||
const hot = async () => {
|
||||
samples++
|
||||
const next = sample()
|
||||
next.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9
|
||||
return next
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: hot, wait: async () => {} }
|
||||
)).rejects.toThrow('cloud-monitoring/threshold_max')
|
||||
expect(samples).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
// Why: on 2026-09-17 a canary wave died because one director admin read
|
||||
// returned 404 for a 2 s Cloud SQL pool timeout. Collecting the sample is not
|
||||
// a health verdict, so a thrown collector spends an attempt instead.
|
||||
describe('collector failures', () => {
|
||||
it('re-samples after a thrown collector and then passes', async () => {
|
||||
const waits: number[] = []
|
||||
let attempts = 0
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{
|
||||
now: () => now,
|
||||
collect: async () => {
|
||||
attempts++
|
||||
if (attempts === 1) throw new Error('Relay admin telemetry returned 404')
|
||||
return sample()
|
||||
},
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)).resolves.toBeUndefined()
|
||||
expect(attempts).toBe(2)
|
||||
expect(waits).toEqual([15_000])
|
||||
})
|
||||
|
||||
it('fails the wave when every attempt throws, naming the collector', async () => {
|
||||
let attempts = 0
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{
|
||||
now: () => now,
|
||||
collect: async () => {
|
||||
attempts++
|
||||
throw new Error('Relay admin telemetry returned 404')
|
||||
},
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow(
|
||||
'relay live preflight failed: collector: Relay admin telemetry returned 404'
|
||||
)
|
||||
expect(attempts).toBe(1 + INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples)
|
||||
})
|
||||
|
||||
it('does not re-sample a collector failure past the evidence-age budget', async () => {
|
||||
const agedPath = stateFile('strict', {
|
||||
startedAt: new Date(now - 26 * 60_000).toISOString(),
|
||||
windowStartedAt: new Date(now - 25 * 60_000).toISOString(),
|
||||
lastSampleAt: new Date(now - 10 * 60_000 + 7).toISOString(),
|
||||
completedAt: new Date(now - 10 * 60_000 + 14).toISOString()
|
||||
})
|
||||
let attempts = 0
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedPath],
|
||||
{
|
||||
now: () => now,
|
||||
collect: async () => {
|
||||
attempts++
|
||||
throw new Error('Relay admin telemetry returned 404')
|
||||
},
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow('relay live preflight failed: collector:')
|
||||
expect(attempts).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
it('enforces the signed migration policy', async () => {
|
||||
const inactiveTarget = sample()
|
||||
inactiveTarget.sources['director-admin']!.signals[
|
||||
'cell.production-gce-c1.migration_target_inactive'
|
||||
]!.value = 30
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: async () => inactiveTarget }
|
||||
)).rejects.toThrow('director-admin/threshold_max')
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile('recover-forward')],
|
||||
{ now: () => now, collect: async () => inactiveTarget }
|
||||
)).resolves.toBeUndefined()
|
||||
inactiveTarget.sources['director-admin']!.signals[
|
||||
'cell.production-gce-c1.migration_blocked'
|
||||
]!.value = 1
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile('recover-forward')],
|
||||
{ now: () => now, collect: async () => inactiveTarget }
|
||||
)).rejects.toThrow('director-admin/threshold_max')
|
||||
})
|
||||
|
||||
it('binds capacity-transition evidence to its general cell', async () => {
|
||||
const capacitySample = sample()
|
||||
const capacitySelector = {
|
||||
generation: 1,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: [],
|
||||
general: ['production-gce-c1']
|
||||
}
|
||||
}
|
||||
capacitySample.selector = capacitySelector
|
||||
capacitySample.expectedSelector = capacitySelector
|
||||
capacitySample.cells[0]!.expectedAdmissionState = 'general'
|
||||
capacitySample.sources['director-admin']!.signals[
|
||||
'cell.production-gce-c1.admission_state'
|
||||
]!.value = 2
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile('capacity-transition')],
|
||||
{ now: () => now, collect: async () => capacitySample }
|
||||
)).resolves.toBeUndefined()
|
||||
capacitySample.sources['director-admin']!.signals[
|
||||
'cell.production-gce-c1.migration_target_inactive'
|
||||
]!.value = 1
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile('capacity-transition')],
|
||||
{ now: () => now, collect: async () => capacitySample }
|
||||
)).rejects.toThrow('director-admin/threshold_max')
|
||||
})
|
||||
|
||||
it('rejects stale live evidence', async () => {
|
||||
const stale = sample()
|
||||
stale.sources['active-probe']!.observedAt = new Date(now - 60_001).toISOString()
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: async () => stale }
|
||||
)).rejects.toThrow('active-probe/source_stale')
|
||||
})
|
||||
|
||||
it('retries freshness-only failures when explicitly requested', async () => {
|
||||
const stale = sample()
|
||||
stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
|
||||
new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
|
||||
const missing = sample()
|
||||
delete missing.sources['relay-logs']
|
||||
const collect = vi.fn()
|
||||
.mockResolvedValueOnce(stale)
|
||||
.mockResolvedValueOnce(missing)
|
||||
.mockResolvedValueOnce(sample())
|
||||
const wait = vi.fn(async () => undefined)
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile(), '--retry-freshness'],
|
||||
{ now: () => now, collect, wait }
|
||||
)).resolves.toBeUndefined()
|
||||
expect(collect).toHaveBeenCalledTimes(3)
|
||||
expect(wait).toHaveBeenCalledTimes(2)
|
||||
expect(wait).toHaveBeenNthCalledWith(1, 15_000)
|
||||
expect(wait).toHaveBeenNthCalledWith(2, 15_000)
|
||||
})
|
||||
|
||||
it('retries a first-wave stale sample and passes on the fresh one', async () => {
|
||||
const stale = sample()
|
||||
stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
|
||||
new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
|
||||
const collect = vi.fn().mockResolvedValueOnce(stale).mockResolvedValueOnce(sample())
|
||||
const wait = vi.fn(async () => undefined)
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile(), '--wave-index', '0', '--retry-freshness'],
|
||||
{ now: () => now, collect, wait }
|
||||
)).resolves.toBeUndefined()
|
||||
expect(collect).toHaveBeenCalledTimes(2)
|
||||
expect(wait).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('stops retrying when the next wait would exceed the evidence-age bound', async () => {
|
||||
const completedAt = now - 590_000
|
||||
const stale = sample()
|
||||
stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
|
||||
const collect = vi.fn(async () => stale)
|
||||
const wait = vi.fn(async () => undefined)
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile('strict', {
|
||||
startedAt: new Date(completedAt - 17 * 60_000).toISOString(),
|
||||
windowStartedAt: new Date(completedAt - 16 * 60_000).toISOString(),
|
||||
lastSampleAt: new Date(completedAt - 30_000).toISOString(),
|
||||
completedAt: new Date(completedAt).toISOString()
|
||||
}), '--retry-freshness'],
|
||||
{ now: () => now, collect, wait }
|
||||
)).rejects.toThrow('cloud-monitoring/source_stale')
|
||||
expect(collect).toHaveBeenCalledOnce()
|
||||
expect(wait).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not retry a threshold failure', async () => {
|
||||
const unhealthy = sample()
|
||||
unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9
|
||||
unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
|
||||
new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
|
||||
const collect = vi.fn(async () => unhealthy)
|
||||
const wait = vi.fn(async () => undefined)
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile(), '--retry-freshness'],
|
||||
{ now: () => now, collect, wait }
|
||||
)).rejects.toThrow('cloud-monitoring/threshold_max')
|
||||
expect(collect).toHaveBeenCalledOnce()
|
||||
expect(wait).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('fails closed after the bounded freshness retry window', async () => {
|
||||
const stale = sample()
|
||||
stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
|
||||
const collect = vi.fn(async () => stale)
|
||||
const wait = vi.fn(async () => undefined)
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile(), '--retry-freshness'],
|
||||
{ now: () => now, collect, wait }
|
||||
)).rejects.toThrow('cloud-monitoring/source_stale')
|
||||
expect(collect).toHaveBeenCalledTimes(5)
|
||||
expect(wait).toHaveBeenCalledTimes(4)
|
||||
})
|
||||
|
||||
|
||||
// Why: a same-cap wave has no sealed 15-minute evidence, so this live recheck
|
||||
// and the pre-drain sample after it are what stand between the dispatch and a
|
||||
// mutation. It must judge the fleet exactly as it does with evidence, and it
|
||||
// must never accept a half-specified selector.
|
||||
describe('dispatch selector without monitor state', () => {
|
||||
const overrideArgs = (extra: string[] = [], membership = canonicalMembership) => [
|
||||
'--no-monitor-state',
|
||||
'--expected-selector-generation', '1',
|
||||
'--selector-membership-file', membershipFile(membership),
|
||||
...extra
|
||||
]
|
||||
|
||||
// The director's reading, plus whatever the override path decided to expect.
|
||||
const liveCollect = (
|
||||
mutate: (next: IncidentSample) => IncidentSample = (next) => next
|
||||
) => async (expected: AdmissionSelector) => {
|
||||
const next = canonicalSample(expected.generation)
|
||||
next.expectedSelector = expected
|
||||
return mutate(next)
|
||||
}
|
||||
|
||||
it('accepts one complete fresh green sample with no sealed evidence', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(),
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
// Why: the live selector is normalised and the comparison is an ordered
|
||||
// stringify, so an operator's unsorted membership must canonicalise here or
|
||||
// every override wave reads as selector drift on a healthy fleet.
|
||||
it('canonicalises an unsorted operator membership', async () => {
|
||||
const shuffled = {
|
||||
existingOnly: [...canonicalCellIds].reverse(),
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
expect(shuffled.existingOnly).not.toEqual(canonicalCellIds)
|
||||
const seen: AdmissionSelector[] = []
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs([], shuffled),
|
||||
{
|
||||
now: () => now,
|
||||
collect: async (expected) => {
|
||||
seen.push(expected)
|
||||
const next = canonicalSample(expected.generation)
|
||||
next.expectedSelector = expected
|
||||
return next
|
||||
}
|
||||
}
|
||||
)).resolves.toBeUndefined()
|
||||
expect(seen[0]!.membership.existingOnly).toEqual(canonicalCellIds)
|
||||
})
|
||||
|
||||
// Why: normalising is also what enforces every configured cell exactly once,
|
||||
// which a bare schema parse would have dropped.
|
||||
it('rejects a membership that is not every configured cell exactly once', async () => {
|
||||
const duplicated = {
|
||||
existingOnly: [...canonicalCellIds, canonicalCellIds[0] as string],
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs([], duplicated),
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow('every configured cell exactly once')
|
||||
const missing = {
|
||||
existingOnly: canonicalCellIds.slice(1),
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs([], missing),
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow('every configured cell exactly once')
|
||||
const unknown = {
|
||||
existingOnly: [...canonicalCellIds.slice(1), 'production-gce-c999'],
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs([], unknown),
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow('every configured cell exactly once')
|
||||
})
|
||||
|
||||
it('fails closed on a live threshold breach', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(),
|
||||
{
|
||||
now: () => now,
|
||||
collect: liveCollect((next) => {
|
||||
next.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.99
|
||||
return next
|
||||
}),
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow('cloud-monitoring/threshold_max cloud_sql.cpu')
|
||||
})
|
||||
|
||||
it('fails closed on a live selector mismatch', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(),
|
||||
{
|
||||
now: () => now,
|
||||
collect: liveCollect((next) => {
|
||||
next.selector = { ...next.selector, generation: 7 }
|
||||
return next
|
||||
}),
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow('selector_mismatch')
|
||||
})
|
||||
|
||||
it('expects the wave-adjusted live selector generation', async () => {
|
||||
const seen: AdmissionSelector[] = []
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(['--wave-index', '2']),
|
||||
{
|
||||
now: () => now,
|
||||
collect: async (expected) => {
|
||||
seen.push(expected)
|
||||
const next = canonicalSample(expected.generation)
|
||||
next.expectedSelector = expected
|
||||
return next
|
||||
}
|
||||
}
|
||||
)).resolves.toBeUndefined()
|
||||
expect(seen[0]!.generation).toBe(5)
|
||||
})
|
||||
|
||||
it('offsets by the wave delta the cell class declares', async () => {
|
||||
const generationFor = async (args: string[]) => {
|
||||
const seen: AdmissionSelector[] = []
|
||||
await expect(runIncidentLivePreflight(args, {
|
||||
now: () => now,
|
||||
collect: async (expected) => {
|
||||
seen.push(expected)
|
||||
const next = canonicalSample(expected.generation)
|
||||
next.expectedSelector = expected
|
||||
return next
|
||||
}
|
||||
})).resolves.toBeUndefined()
|
||||
return seen[0]!.generation
|
||||
}
|
||||
// A migration-only cell's wave isolates and restores nothing, so no predecessor moved it.
|
||||
expect(await generationFor(
|
||||
overrideArgs(['--wave-index', '2', '--selector-wave-delta', '0'])
|
||||
)).toBe(1)
|
||||
expect(await generationFor(
|
||||
overrideArgs(['--wave-index', '2', '--selector-wave-delta', '2'])
|
||||
)).toBe(5)
|
||||
})
|
||||
|
||||
it('rejects a selector wave delta no cell class produces', async () => {
|
||||
for (const delta of ['1', '3', '4', '', '-0', '02']) {
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(['--selector-wave-delta', delta]),
|
||||
{ now: () => now }
|
||||
)).rejects.toThrow('usage:')
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(['--selector-wave-delta', '0', '--selector-wave-delta', '0']),
|
||||
{ now: () => now }
|
||||
)).rejects.toThrow('usage:')
|
||||
})
|
||||
|
||||
it('pins the strictest migration policy', async () => {
|
||||
// An inactive migration target is tolerable only under recover-forward,
|
||||
// and an override cannot elect that policy, so this must still fail.
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(),
|
||||
{
|
||||
now: () => now,
|
||||
collect: liveCollect((next) => {
|
||||
next.sources['director-admin']!.signals[
|
||||
'cell.production-gce-c1.migration_target_inactive'
|
||||
]!.value = 30
|
||||
return next
|
||||
}),
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow('director-admin/threshold_max')
|
||||
})
|
||||
|
||||
it('rejects a half-specified override', async () => {
|
||||
const cases: string[][] = [
|
||||
['--no-monitor-state'],
|
||||
['--no-monitor-state', '--expected-selector-generation', '1'],
|
||||
['--no-monitor-state', '--selector-membership-file', membershipFile()],
|
||||
// Mixing the two sources would let a caller pass sealed evidence it
|
||||
// never wants read.
|
||||
[
|
||||
'--no-monitor-state',
|
||||
'--expected-selector-generation', '1',
|
||||
'--selector-membership-file', membershipFile(),
|
||||
'--state-file', stateFile()
|
||||
],
|
||||
// Override arguments without the flag must not be silently ignored.
|
||||
['--state-file', stateFile(), '--expected-selector-generation', '1'],
|
||||
['--no-monitor-state', '--no-monitor-state'],
|
||||
['--expected-selector-generation', '1']
|
||||
]
|
||||
for (const args of cases) {
|
||||
await expect(runIncidentLivePreflight(
|
||||
args,
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow('usage:')
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects an unknown option and a negative generation', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(['--skip-everything']),
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow('usage:')
|
||||
await expect(runIncidentLivePreflight(
|
||||
[
|
||||
'--no-monitor-state',
|
||||
'--expected-selector-generation', '-1',
|
||||
'--selector-membership-file', membershipFile()
|
||||
],
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow()
|
||||
})
|
||||
|
||||
it('re-samples a tolerable failure and then passes', async () => {
|
||||
let samples = 0
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(),
|
||||
{
|
||||
now: () => now,
|
||||
collect: liveCollect((next) => {
|
||||
samples++
|
||||
if (samples === 1) {
|
||||
next.sources['cloud-monitoring']!.signals['director.instances']!.value = 2
|
||||
}
|
||||
return next
|
||||
}),
|
||||
wait: async () => {}
|
||||
}
|
||||
)).resolves.toBeUndefined()
|
||||
expect(samples).toBe(2)
|
||||
})
|
||||
})
|
||||
|
||||
it('uses the supplied admin token without minting through gcloud', async () => {
|
||||
const identityToken = vi.fn(async () => 'minted.token.value')
|
||||
const gcloud = livePreflightGcloud(
|
||||
{ accessToken: async () => 'access-token', identityToken },
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'supplied.token.value' }
|
||||
)
|
||||
await expect(gcloud.identityToken!('audience')).resolves.toBe(
|
||||
'supplied.token.value'
|
||||
)
|
||||
expect(identityToken).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -1,360 +0,0 @@
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { z } from 'zod'
|
||||
import { relayOpsEnvironment } from './environment-config.js'
|
||||
import { createGcloudClient } from './gcloud-client.js'
|
||||
import { suppliedIdentityToken } from './incident-monitor-cli.js'
|
||||
import {
|
||||
AdmissionSelectorSchema,
|
||||
normalizeSelectorMembership,
|
||||
SelectorMembershipSchema,
|
||||
type AdmissionSelector
|
||||
} from './incident-selector.js'
|
||||
import {
|
||||
describeIncidentFailure,
|
||||
evaluateIncidentSample,
|
||||
FRESHNESS_FAILURE_CODES,
|
||||
INCIDENT_MONITOR_THRESHOLDS,
|
||||
preDrainDryRunPassed,
|
||||
toleratedStreakKey,
|
||||
type IncidentSample
|
||||
} from './incident-monitor.js'
|
||||
import { createIncidentSampleCollector } from './incident-monitor-sources.js'
|
||||
|
||||
const FRESHNESS_RETRY_ATTEMPTS = 5
|
||||
const FRESHNESS_RETRY_INTERVAL_MS = 15_000
|
||||
// 10 min, not 5: a wave job reaches this check ~5 min after the monitor completes
|
||||
// (runner queue ~2 min, gate job ~80 s, checkout ~60 s); on 2026-09-17 a green
|
||||
// gate died at 302 s. The live samples below hold every wave to now.
|
||||
const MONITOR_EVIDENCE_MAX_AGE_MS = 10 * 60_000
|
||||
// Matches the wave job timeout-minutes; bounds each predecessor wave.
|
||||
const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
|
||||
// Widest any wave chain declares (same-cap's cell_1..cell_10); each job workflow
|
||||
// pins its own narrower range.
|
||||
const WAVE_INDEX_PATTERN = /^[0-9]$/
|
||||
// 2 for a general cell's isolate-and-restore wave, 0 for a migration-only cell's no-op pair.
|
||||
const SELECTOR_WAVE_DELTA_PATTERN = /^[02]$/
|
||||
|
||||
export function livePreflightGcloud(
|
||||
gcloud: ReturnType<typeof createGcloudClient>,
|
||||
environment: NodeJS.ProcessEnv = process.env
|
||||
): ReturnType<typeof createGcloudClient> {
|
||||
const token = suppliedIdentityToken(environment.ORCA_RELAY_ADMIN_ID_TOKEN)
|
||||
return token ? { ...gcloud, identityToken: async () => token } : gcloud
|
||||
}
|
||||
|
||||
const PreflightStateSchema = z.object({
|
||||
schemaVersion: z.literal(4),
|
||||
environment: z.literal('production'),
|
||||
expectedSelector: AdmissionSelectorSchema,
|
||||
migrationPolicy: z.enum(['strict', 'recover-forward', 'capacity-transition']),
|
||||
recoverySourceCellId: z.string().nullable(),
|
||||
capacityCellId: z.string().nullable(),
|
||||
preDrainDryRun: z.literal(true),
|
||||
startedAt: z.string(),
|
||||
windowStartedAt: z.string(),
|
||||
durationMinutes: z.literal(15),
|
||||
intervalMs: z.literal(60_000),
|
||||
sampleCount: z.number().int().min(16),
|
||||
lastSampleAt: z.string(),
|
||||
frozenAt: z.null(),
|
||||
completedAt: z.string()
|
||||
}).superRefine((state, context) => {
|
||||
const validRecovery =
|
||||
state.migrationPolicy === 'recover-forward' &&
|
||||
state.capacityCellId === null &&
|
||||
state.recoverySourceCellId !== null &&
|
||||
state.expectedSelector.membership.existingOnly.includes(
|
||||
state.recoverySourceCellId
|
||||
)
|
||||
const validStrict =
|
||||
state.migrationPolicy === 'strict' &&
|
||||
state.recoverySourceCellId === null &&
|
||||
state.capacityCellId === null
|
||||
const validCapacity =
|
||||
state.migrationPolicy === 'capacity-transition' &&
|
||||
state.recoverySourceCellId === null &&
|
||||
state.capacityCellId !== null &&
|
||||
state.expectedSelector.membership.general.includes(state.capacityCellId)
|
||||
if (!validRecovery && !validStrict && !validCapacity) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: 'relay live preflight migration policy is invalid'
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
const PREFLIGHT_USAGE =
|
||||
'usage: --state-file <verified-monitor-state> [--wave-index <0-3>]' +
|
||||
' [--selector-wave-delta <0|2>] [--retry-freshness]' +
|
||||
' | --no-monitor-state --expected-selector-generation <n>' +
|
||||
' --selector-membership-file <json> [--wave-index <0-3>]' +
|
||||
' [--selector-wave-delta <0|2>]'
|
||||
|
||||
const VALUE_OPTIONS = new Set([
|
||||
'--state-file',
|
||||
'--wave-index',
|
||||
'--selector-wave-delta',
|
||||
'--expected-selector-generation',
|
||||
'--selector-membership-file'
|
||||
])
|
||||
const FLAG_OPTIONS = new Set(['--retry-freshness', '--no-monitor-state'])
|
||||
|
||||
// Rejects an unknown option and a repeated one, so a typo can never silently
|
||||
// widen what this check accepts.
|
||||
export function parsePreflightArgs(argv: string[]): {
|
||||
options: Map<string, string>
|
||||
flags: Set<string>
|
||||
} {
|
||||
const args = argv[0] === '--' ? argv.slice(1) : argv
|
||||
const options = new Map<string, string>()
|
||||
const flags = new Set<string>()
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index] as string
|
||||
if (FLAG_OPTIONS.has(arg)) {
|
||||
if (flags.has(arg)) throw new Error(PREFLIGHT_USAGE)
|
||||
flags.add(arg)
|
||||
continue
|
||||
}
|
||||
const value = args[index + 1]
|
||||
if (!VALUE_OPTIONS.has(arg) || options.has(arg) || !value) {
|
||||
throw new Error(PREFLIGHT_USAGE)
|
||||
}
|
||||
options.set(arg, value)
|
||||
index += 1
|
||||
}
|
||||
return { options, flags }
|
||||
}
|
||||
|
||||
// What the live recheck measures the fleet against. Either source supplies the
|
||||
// exact same fields; only where they come from and how they can go stale differs.
|
||||
type PreflightPlan = {
|
||||
environment: 'production'
|
||||
expectedSelector: AdmissionSelector
|
||||
migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition'
|
||||
recoverySourceCellId: string | null
|
||||
capacityCellId: string | null
|
||||
// The instant the live samples age from. Monitor evidence ages from the moment
|
||||
// the 15-minute window closed; a dispatch plan has no evidence to age, so its
|
||||
// retry budget starts when this process does.
|
||||
evidenceAnchorMs: number
|
||||
}
|
||||
|
||||
async function monitorEvidencePreflightPlan(
|
||||
options: Map<string, string>,
|
||||
nowMs: number,
|
||||
waveIndex: string
|
||||
): Promise<PreflightPlan> {
|
||||
const stateFile = options.get('--state-file')
|
||||
if (
|
||||
!stateFile ||
|
||||
options.has('--expected-selector-generation') ||
|
||||
options.has('--selector-membership-file')
|
||||
) throw new Error(PREFLIGHT_USAGE)
|
||||
const state = PreflightStateSchema.parse(
|
||||
JSON.parse(await readFile(resolve(stateFile), 'utf8'))
|
||||
)
|
||||
const completedAt = Date.parse(state.completedAt)
|
||||
const windowStartedAt = Date.parse(state.windowStartedAt)
|
||||
const lastSampleAt = Date.parse(state.lastSampleAt)
|
||||
const evidenceAgeMs = nowMs - completedAt
|
||||
// Later waves start after sequential predecessor cell rolls, so the
|
||||
// freshness bound grows by one cell-job timeout per predecessor; the live
|
||||
// samples collected below still hold every wave to current health.
|
||||
const maxEvidenceAgeMs =
|
||||
MONITOR_EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS
|
||||
if (
|
||||
!preDrainDryRunPassed(state) ||
|
||||
!Number.isFinite(windowStartedAt) ||
|
||||
completedAt - windowStartedAt < 15 * 60_000 ||
|
||||
!Number.isFinite(lastSampleAt) ||
|
||||
lastSampleAt > completedAt ||
|
||||
completedAt - lastSampleAt > state.intervalMs ||
|
||||
!Number.isFinite(completedAt) ||
|
||||
evidenceAgeMs < 0 ||
|
||||
evidenceAgeMs > maxEvidenceAgeMs
|
||||
) {
|
||||
throw new Error('relay live preflight monitor evidence is incomplete or stale')
|
||||
}
|
||||
return {
|
||||
environment: state.environment,
|
||||
expectedSelector: state.expectedSelector,
|
||||
migrationPolicy: state.migrationPolicy,
|
||||
recoverySourceCellId: state.recoverySourceCellId,
|
||||
capacityCellId: state.capacityCellId,
|
||||
evidenceAnchorMs: completedAt
|
||||
}
|
||||
}
|
||||
|
||||
// The expected selector exactly as a same-cap dispatch states it, canonicalised as the monitor CLI
|
||||
// does when it seals evidence. The live selector read from the director is normalised too and the
|
||||
// comparison is an ordered stringify, so unsorted operator input would read as selector drift on a
|
||||
// healthy fleet; normalising is also what enforces every configured cell exactly once.
|
||||
export async function readDispatchSelector(
|
||||
generation: string,
|
||||
membershipFile: string
|
||||
): Promise<AdmissionSelector> {
|
||||
const membership = normalizeSelectorMembership(
|
||||
SelectorMembershipSchema.parse(
|
||||
JSON.parse(await readFile(resolve(membershipFile), 'utf8'))
|
||||
),
|
||||
new Set(relayOpsEnvironment('production').cells.map((cell) => cell.cellId))
|
||||
)
|
||||
return AdmissionSelectorSchema.parse({ generation: Number(generation), membership })
|
||||
}
|
||||
|
||||
// Each predecessor same-cap apply wave reversibly isolates and restores its cell with membership
|
||||
// unchanged (rollback is single-cell, so it never reaches a later wave), so the live selector
|
||||
// comparison must expect the wave-adjusted generation. A general cell advances it by 2; a
|
||||
// migration-only cell is already isolated and stays that way, so its wave advances it by 0. A wave
|
||||
// is never mixed, so one delta covers every predecessor.
|
||||
export function waveAdjustedSelector(
|
||||
selector: AdmissionSelector,
|
||||
waveIndex: string,
|
||||
selectorWaveDelta: string
|
||||
): AdmissionSelector {
|
||||
if (!WAVE_INDEX_PATTERN.test(waveIndex) || !SELECTOR_WAVE_DELTA_PATTERN.test(selectorWaveDelta)) {
|
||||
throw new Error('relay wave index or selector wave delta is invalid')
|
||||
}
|
||||
return {
|
||||
...selector,
|
||||
generation: selector.generation + Number(selectorWaveDelta) * Number(waveIndex)
|
||||
}
|
||||
}
|
||||
|
||||
// No monitor evidence: the expected selector comes straight from the dispatch inputs, which the
|
||||
// rehome inspect after this verifies against the live director. The policy is pinned to strict --
|
||||
// the only one the same-cap rollout ever verifies -- and every threshold is unchanged.
|
||||
async function dispatchPreflightPlan(
|
||||
options: Map<string, string>,
|
||||
nowMs: number
|
||||
): Promise<PreflightPlan> {
|
||||
const generation = options.get('--expected-selector-generation')
|
||||
const membershipFile = options.get('--selector-membership-file')
|
||||
if (!generation || !membershipFile || options.has('--state-file')) {
|
||||
throw new Error(PREFLIGHT_USAGE)
|
||||
}
|
||||
return {
|
||||
environment: 'production',
|
||||
expectedSelector: await readDispatchSelector(generation, membershipFile),
|
||||
migrationPolicy: 'strict',
|
||||
recoverySourceCellId: null,
|
||||
capacityCellId: null,
|
||||
evidenceAnchorMs: nowMs
|
||||
}
|
||||
}
|
||||
|
||||
export async function runIncidentLivePreflight(
|
||||
argv: string[],
|
||||
dependencies: {
|
||||
now?: () => number
|
||||
wait?: (ms: number) => Promise<void>
|
||||
collect?: (expectedSelector: AdmissionSelector) => Promise<IncidentSample>
|
||||
gcloud?: ReturnType<typeof createGcloudClient>
|
||||
environment?: NodeJS.ProcessEnv
|
||||
} = {}
|
||||
): Promise<void> {
|
||||
const parsed = parsePreflightArgs(argv)
|
||||
const waveIndex = parsed.options.get('--wave-index') ?? '0'
|
||||
if (!WAVE_INDEX_PATTERN.test(waveIndex)) throw new Error(PREFLIGHT_USAGE)
|
||||
const selectorWaveDelta = parsed.options.get('--selector-wave-delta') ?? '2'
|
||||
if (!SELECTOR_WAVE_DELTA_PATTERN.test(selectorWaveDelta)) throw new Error(PREFLIGHT_USAGE)
|
||||
const now = dependencies.now ?? Date.now
|
||||
const plan = parsed.flags.has('--no-monitor-state')
|
||||
? await dispatchPreflightPlan(parsed.options, now())
|
||||
: await monitorEvidencePreflightPlan(parsed.options, now(), waveIndex)
|
||||
const maxEvidenceAgeMs =
|
||||
MONITOR_EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS
|
||||
const gcloud = livePreflightGcloud(
|
||||
dependencies.gcloud ?? createGcloudClient(),
|
||||
dependencies.environment
|
||||
)
|
||||
const collectOptions = {
|
||||
environment: plan.environment,
|
||||
expectedSelector: waveAdjustedSelector(plan.expectedSelector, waveIndex, selectorWaveDelta),
|
||||
...(dependencies.now ? { now: dependencies.now } : {})
|
||||
}
|
||||
const injected = dependencies.collect
|
||||
const collect = injected
|
||||
? () => injected(collectOptions.expectedSelector)
|
||||
: createIncidentSampleCollector(gcloud, collectOptions)
|
||||
const wait = dependencies.wait ?? ((ms: number) => new Promise<void>((resolveWait) => {
|
||||
setTimeout(resolveWait, ms)
|
||||
}))
|
||||
const freshnessAttempts = parsed.flags.has('--retry-freshness') ? FRESHNESS_RETRY_ATTEMPTS : 1
|
||||
// Why: this single sample decides a mutating wave, so an Asia cell's ~30 s
|
||||
// "no healthy upstream" window, or a one-minute director instance-replacement
|
||||
// dip, could fail a wave here even after the 15-minute gate learned to ride it
|
||||
// out. Hold the two to the same tolerance. Unlike the freshness retry this
|
||||
// needs no flag, because a tolerated breach is never the operator's call to
|
||||
// waive.
|
||||
const cellProbeAttempts = 1 + INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
const attempts = Math.max(freshnessAttempts, cellProbeAttempts)
|
||||
let freshnessRetries = freshnessAttempts - 1
|
||||
let cellProbeRetries = cellProbeAttempts - 1
|
||||
// Waiting must never carry the mutation past the same evidence-age bound the
|
||||
// entry check enforces, so the wave budget also caps the retry window.
|
||||
const budgetExhausted = (): boolean =>
|
||||
now() + FRESHNESS_RETRY_INTERVAL_MS - plan.evidenceAnchorMs > maxEvidenceAgeMs
|
||||
for (let attempt = 1; attempt <= attempts; attempt++) {
|
||||
// A director admin read can fail on its own (its handler maps a Cloud SQL
|
||||
// pool timeout onto 404), which says nothing about relay health; spend an
|
||||
// attempt on it rather than failing the wave on one unlucky sample.
|
||||
let sample: IncidentSample
|
||||
try {
|
||||
sample = await collect()
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'sample collection failed'
|
||||
if (attempt === attempts || budgetExhausted()) {
|
||||
throw new Error(`relay live preflight failed: collector: ${message}`)
|
||||
}
|
||||
console.warn(
|
||||
`relay live preflight re-sampling after collector failure (${attempt}/${attempts - 1})`
|
||||
)
|
||||
await wait(FRESHNESS_RETRY_INTERVAL_MS)
|
||||
continue
|
||||
}
|
||||
const evaluation = evaluateIncidentSample(
|
||||
sample,
|
||||
now(),
|
||||
plan.migrationPolicy,
|
||||
plan.recoverySourceCellId,
|
||||
plan.capacityCellId
|
||||
)
|
||||
if (evaluation.status === 'green') return
|
||||
const freshnessFailures = evaluation.failures.filter((failure) =>
|
||||
FRESHNESS_FAILURE_CODES.has(failure.code)
|
||||
)
|
||||
// Tolerated readings only (per-cell probes and the director instance count).
|
||||
// The director and auth health probes are absent here on purpose and fail
|
||||
// the wave on their first bad sample.
|
||||
const cellProbeFailures = evaluation.failures.filter((failure) =>
|
||||
!FRESHNESS_FAILURE_CODES.has(failure.code) && toleratedStreakKey(failure) !== null
|
||||
)
|
||||
const retryable =
|
||||
freshnessFailures.length + cellProbeFailures.length === evaluation.failures.length &&
|
||||
(freshnessFailures.length === 0 || freshnessRetries > 0) &&
|
||||
(cellProbeFailures.length === 0 || cellProbeRetries > 0)
|
||||
if (!retryable || attempt === attempts || budgetExhausted()) {
|
||||
throw new Error(
|
||||
`relay live preflight failed: ${evaluation.failures
|
||||
.map(describeIncidentFailure)
|
||||
.join(',')}`
|
||||
)
|
||||
}
|
||||
if (freshnessFailures.length > 0) freshnessRetries--
|
||||
if (cellProbeFailures.length > 0) cellProbeRetries--
|
||||
console.warn(
|
||||
`relay live preflight re-sampling after tolerable failure (${attempt}/${attempts - 1})`
|
||||
)
|
||||
await wait(FRESHNESS_RETRY_INTERVAL_MS)
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
runIncidentLivePreflight(process.argv.slice(2)).catch((error: unknown) => {
|
||||
console.error(error instanceof Error ? error.message : 'relay live preflight failed')
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,455 +0,0 @@
|
||||
import { mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
parseIncidentMonitorArguments,
|
||||
runIncidentMonitorCli
|
||||
} from './incident-monitor-cli.js'
|
||||
import type { IncidentSample } from './incident-monitor.js'
|
||||
import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js'
|
||||
|
||||
const directories: string[] = []
|
||||
const startedAt = Date.parse('2026-07-28T00:00:00.000Z')
|
||||
const productionCells = RELAY_OPS_ENVIRONMENTS.production.cells.map((cell) => cell.cellId)
|
||||
const selector = {
|
||||
generation: 1,
|
||||
membership: {
|
||||
existingOnly: productionCells.slice(1),
|
||||
migrationOnly: [],
|
||||
general: [productionCells[0]!]
|
||||
}
|
||||
}
|
||||
const selectorArguments = [
|
||||
'--expected-selector-generation',
|
||||
'1',
|
||||
'--expected-existing-only-cells',
|
||||
productionCells.slice(1).join(','),
|
||||
'--expected-migration-only-cells',
|
||||
'none',
|
||||
'--expected-general-cells',
|
||||
productionCells[0]!
|
||||
]
|
||||
const signal = (value: number, at: number) => ({
|
||||
value,
|
||||
observedAt: new Date(at).toISOString()
|
||||
})
|
||||
|
||||
function sample(at: number): IncidentSample {
|
||||
const observedAt = new Date(at).toISOString()
|
||||
const cellId = 'production-gce-c1'
|
||||
return {
|
||||
collectedAt: observedAt,
|
||||
selector,
|
||||
expectedSelector: selector,
|
||||
cells: [{
|
||||
cellId,
|
||||
region: 'us-central1',
|
||||
runtimeKnown: true,
|
||||
powered: true,
|
||||
expectedAdmissionState: 'general'
|
||||
}],
|
||||
sources: {
|
||||
'active-probe': {
|
||||
observedAt,
|
||||
signals: {
|
||||
'director.health': signal(1, at),
|
||||
'director.ready': signal(1, at),
|
||||
'director.latency_ms': signal(1, at),
|
||||
'auth.health': signal(1, at),
|
||||
'auth.ready': signal(1, at),
|
||||
'auth.latency_ms': signal(1, at),
|
||||
[`cell.${cellId}.health`]: signal(1, at),
|
||||
[`cell.${cellId}.ready`]: signal(1, at),
|
||||
[`cell.${cellId}.latency_ms`]: signal(1, at)
|
||||
}
|
||||
},
|
||||
'cloud-monitoring': {
|
||||
observedAt,
|
||||
signals: {
|
||||
'cloud_sql.cpu': signal(0.1, at),
|
||||
'cloud_sql.memory': signal(0.1, at),
|
||||
'cloud_sql.backends': signal(1, at),
|
||||
'cloud_sql.lock_waits': signal(0, at),
|
||||
'cloud_sql.deadlocks': signal(0, at),
|
||||
'director.instances': signal(5, at),
|
||||
'director.cpu': signal(0.1, at),
|
||||
'director.memory': signal(0.1, at),
|
||||
'director.concurrency': signal(1, at),
|
||||
'director.errors': signal(0, at),
|
||||
'auth.errors': signal(0, at)
|
||||
}
|
||||
},
|
||||
'relay-logs': {
|
||||
observedAt,
|
||||
signals: {
|
||||
'relay.pool_waiting': signal(0, at),
|
||||
'relay.pool_wait_ms': signal(0, at),
|
||||
'relay.postgres_retries': signal(0, at),
|
||||
'relay.postgres_retry_exhausted': signal(0, at),
|
||||
[`cell.${cellId}.connections`]: signal(1, at),
|
||||
[`cell.${cellId}.queued_bytes`]: signal(0, at)
|
||||
}
|
||||
},
|
||||
'director-admin': {
|
||||
observedAt,
|
||||
signals: {
|
||||
[`cell.${cellId}.admission_state`]: signal(2, at),
|
||||
[`cell.${cellId}.heartbeat_fresh`]: signal(1, at),
|
||||
[`cell.${cellId}.heartbeat_age_ms`]: signal(1, at),
|
||||
[`cell.${cellId}.migration_blocked`]: signal(0, at),
|
||||
[`cell.${cellId}.migration_target_inactive`]: signal(0, at)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const directory of directories.splice(0)) {
|
||||
rmSync(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
describe('incident monitor CLI', () => {
|
||||
it('requires an exact selector and rejects invalid membership', () => {
|
||||
expect(() => parseIncidentMonitorArguments([])).toThrow(
|
||||
'--expected-selector-generation'
|
||||
)
|
||||
expect(() =>
|
||||
parseIncidentMonitorArguments([
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
'--expected-selector-generation',
|
||||
'1',
|
||||
'--expected-existing-only-cells',
|
||||
productionCells.slice(1).join(','),
|
||||
'--expected-migration-only-cells',
|
||||
'none',
|
||||
'--expected-general-cells',
|
||||
'production-gce-c99'
|
||||
])
|
||||
).toThrow('every configured cell exactly once')
|
||||
expect(() =>
|
||||
parseIncidentMonitorArguments([
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
...selectorArguments,
|
||||
'--interval-seconds',
|
||||
'61'
|
||||
])
|
||||
).toThrow('between 1 and 60')
|
||||
expect(() =>
|
||||
parseIncidentMonitorArguments([
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
...selectorArguments,
|
||||
'--duration-minutes',
|
||||
'14'
|
||||
])
|
||||
).toThrow('between 15 and 90')
|
||||
expect(() =>
|
||||
parseIncidentMonitorArguments([
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
'--expected-selector-generation',
|
||||
'0',
|
||||
'--expected-existing-only-cells',
|
||||
productionCells.slice(1).join(','),
|
||||
'--expected-migration-only-cells',
|
||||
productionCells[0]!,
|
||||
'--expected-general-cells',
|
||||
'none'
|
||||
])
|
||||
).toThrow('generation 0 cannot represent migration-only')
|
||||
expect(() =>
|
||||
parseIncidentMonitorArguments([
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
...selectorArguments,
|
||||
'--migration-policy',
|
||||
'recover-forward',
|
||||
'--pre-drain-dry-run'
|
||||
])
|
||||
).toThrow('requires an existing-only --recovery-source-cell-id')
|
||||
expect(() =>
|
||||
parseIncidentMonitorArguments([
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
...selectorArguments,
|
||||
'--migration-policy',
|
||||
'capacity-transition',
|
||||
'--pre-drain-dry-run'
|
||||
])
|
||||
).toThrow('requires a general --capacity-cell-id')
|
||||
expect(
|
||||
parseIncidentMonitorArguments([
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
...selectorArguments,
|
||||
'--migration-policy',
|
||||
'capacity-transition',
|
||||
'--capacity-cell-id',
|
||||
productionCells[0]!,
|
||||
'--pre-drain-dry-run'
|
||||
])
|
||||
).toMatchObject({
|
||||
migrationPolicy: 'capacity-transition',
|
||||
capacityCellId: productionCells[0]!,
|
||||
recoverySourceCellId: null
|
||||
})
|
||||
})
|
||||
|
||||
it('writes private durable checkpoints for a green pre-drain dry run', async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-'))
|
||||
directories.push(directory)
|
||||
let now = startedAt
|
||||
const output: string[] = []
|
||||
const code = await runIncidentMonitorCli(
|
||||
[
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
...selectorArguments,
|
||||
'--pre-drain-dry-run',
|
||||
'--output-directory',
|
||||
directory
|
||||
],
|
||||
{
|
||||
cwd: directory,
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => sample(now),
|
||||
writeOutput: (value) => output.push(value)
|
||||
}
|
||||
)
|
||||
expect(code).toBe(0)
|
||||
const statePath = join(directory, 'incident-1.state.json')
|
||||
const summaryPath = join(directory, 'incident-1.summaries.jsonl')
|
||||
const markdownPath = join(directory, 'incident-1.summary.md')
|
||||
expect(statSync(statePath).mode & 0o077).toBe(0)
|
||||
expect(statSync(summaryPath).mode & 0o077).toBe(0)
|
||||
expect(statSync(markdownPath).mode & 0o077).toBe(0)
|
||||
const summaries = readFileSync(summaryPath, 'utf8')
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((line) => JSON.parse(line))
|
||||
expect(summaries.map((entry) => entry.checkpointMinute)).toEqual([0, 5, 15])
|
||||
expect(output.join('')).not.toContain('token')
|
||||
expect(readFileSync(markdownPath, 'utf8')).toContain(
|
||||
'| 0 | 15 | green | 16 | none |'
|
||||
)
|
||||
expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({
|
||||
completedAt: new Date(startedAt + 15 * 60_000).toISOString(),
|
||||
frozenAt: null,
|
||||
migrationPolicy: 'strict',
|
||||
recoverySourceCellId: null,
|
||||
capacityCellId: null,
|
||||
sampleCount: 16
|
||||
})
|
||||
})
|
||||
|
||||
it('runs a recovery dry run without masking blocked migrations', async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-'))
|
||||
directories.push(directory)
|
||||
let now = startedAt
|
||||
const recoverySelector = {
|
||||
generation: 1,
|
||||
membership: {
|
||||
existingOnly: ['production-gce-c1'],
|
||||
migrationOnly: [],
|
||||
general: productionCells.slice(1)
|
||||
}
|
||||
}
|
||||
const recoverySample = (): IncidentSample => {
|
||||
const current = sample(now)
|
||||
current.selector = recoverySelector
|
||||
current.expectedSelector = recoverySelector
|
||||
current.cells[0]!.expectedAdmissionState = 'existing-only'
|
||||
current.sources['director-admin']!.signals[
|
||||
'cell.production-gce-c1.admission_state'
|
||||
] = signal(0, now)
|
||||
current.sources['director-admin']!.signals[
|
||||
'cell.production-gce-c1.migration_target_inactive'
|
||||
] = signal(30, now)
|
||||
return current
|
||||
}
|
||||
const args = [
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
'--expected-selector-generation',
|
||||
'1',
|
||||
'--expected-existing-only-cells',
|
||||
'production-gce-c1',
|
||||
'--expected-migration-only-cells',
|
||||
'none',
|
||||
'--expected-general-cells',
|
||||
productionCells.slice(1).join(','),
|
||||
'--pre-drain-dry-run',
|
||||
'--migration-policy',
|
||||
'recover-forward',
|
||||
'--recovery-source-cell-id',
|
||||
'production-gce-c1',
|
||||
'--output-directory',
|
||||
directory
|
||||
]
|
||||
const dependencies = {
|
||||
cwd: directory,
|
||||
now: () => now,
|
||||
wait: async (ms: number) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => recoverySample(),
|
||||
writeOutput: () => {}
|
||||
}
|
||||
await expect(runIncidentMonitorCli(args, dependencies)).resolves.toBe(0)
|
||||
expect(
|
||||
JSON.parse(readFileSync(join(directory, 'incident-1.state.json'), 'utf8'))
|
||||
).toMatchObject({
|
||||
migrationPolicy: 'recover-forward',
|
||||
recoverySourceCellId: 'production-gce-c1',
|
||||
capacityCellId: null,
|
||||
frozenAt: null
|
||||
})
|
||||
})
|
||||
|
||||
it('fails a frozen pre-drain gate after its first sample', async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-'))
|
||||
directories.push(directory)
|
||||
let now = startedAt
|
||||
let collections = 0
|
||||
let waits = 0
|
||||
const code = await runIncidentMonitorCli(
|
||||
[
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
...selectorArguments,
|
||||
'--pre-drain-dry-run',
|
||||
'--output-directory',
|
||||
directory
|
||||
],
|
||||
{
|
||||
cwd: directory,
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
waits++
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
collections++
|
||||
const unhealthy = sample(now)
|
||||
unhealthy.sources['relay-logs']!.signals['relay.pool_waiting'] =
|
||||
signal(801, now)
|
||||
return unhealthy
|
||||
},
|
||||
writeOutput: () => {}
|
||||
}
|
||||
)
|
||||
expect(code).toBe(2)
|
||||
expect(collections).toBe(1)
|
||||
expect(waits).toBe(0)
|
||||
expect(
|
||||
JSON.parse(readFileSync(join(directory, 'incident-1.state.json'), 'utf8'))
|
||||
).toMatchObject({
|
||||
completedAt: new Date(startedAt).toISOString(),
|
||||
frozenAt: new Date(startedAt).toISOString(),
|
||||
sampleCount: 1
|
||||
})
|
||||
})
|
||||
|
||||
it('requires --restart and preserves a latched freeze', async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-'))
|
||||
directories.push(directory)
|
||||
let now = startedAt
|
||||
const args = [
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
...selectorArguments,
|
||||
'--duration-minutes',
|
||||
'15',
|
||||
'--output-directory',
|
||||
directory
|
||||
]
|
||||
await runIncidentMonitorCli(args, {
|
||||
cwd: directory,
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
const unhealthy = sample(now)
|
||||
unhealthy.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(801, now)
|
||||
return unhealthy
|
||||
},
|
||||
writeOutput: () => {}
|
||||
})
|
||||
await expect(
|
||||
runIncidentMonitorCli(args, {
|
||||
cwd: directory,
|
||||
now: () => now,
|
||||
collect: async () => sample(now),
|
||||
writeOutput: () => {}
|
||||
})
|
||||
).rejects.toThrow('pass --restart')
|
||||
const changedAdmission = [...args]
|
||||
changedAdmission[changedAdmission.indexOf('--expected-selector-generation') + 1] = '2'
|
||||
await expect(
|
||||
runIncidentMonitorCli([...changedAdmission, '--restart'], {
|
||||
cwd: directory,
|
||||
now: () => now,
|
||||
collect: async () => sample(now),
|
||||
writeOutput: () => {}
|
||||
})
|
||||
).rejects.toThrow('do not match')
|
||||
await expect(
|
||||
runIncidentMonitorCli([...args, '--restart'], {
|
||||
cwd: directory,
|
||||
now: () => now,
|
||||
collect: async () => sample(now),
|
||||
writeOutput: () => {}
|
||||
})
|
||||
).resolves.toBe(2)
|
||||
})
|
||||
|
||||
it('resumes a gracefully segmented monitor without resetting continuity', async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-'))
|
||||
directories.push(directory)
|
||||
let now = startedAt
|
||||
const args = [
|
||||
'--incident-id',
|
||||
'incident-1',
|
||||
...selectorArguments,
|
||||
'--duration-minutes',
|
||||
'15',
|
||||
'--output-directory',
|
||||
directory
|
||||
]
|
||||
const dependencies = {
|
||||
cwd: directory,
|
||||
now: () => now,
|
||||
wait: async (ms: number) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => sample(now),
|
||||
writeOutput: () => {}
|
||||
}
|
||||
await expect(
|
||||
runIncidentMonitorCli([...args, '--max-samples-this-run', '2'], dependencies)
|
||||
).resolves.toBe(0)
|
||||
const statePath = join(directory, 'incident-1.state.json')
|
||||
expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({
|
||||
completedAt: null,
|
||||
sampleCount: 2,
|
||||
lastSampleAt: new Date(startedAt + 60_000).toISOString()
|
||||
})
|
||||
await expect(
|
||||
runIncidentMonitorCli([...args, '--restart'], dependencies)
|
||||
).resolves.toBe(0)
|
||||
expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({
|
||||
completedAt: new Date(startedAt + 15 * 60_000).toISOString(),
|
||||
windowSequence: 0,
|
||||
sampleCount: 17
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,510 +0,0 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import {
|
||||
appendFile,
|
||||
chmod,
|
||||
mkdir,
|
||||
open,
|
||||
readFile,
|
||||
rename,
|
||||
stat,
|
||||
writeFile
|
||||
} from 'node:fs/promises'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { z } from 'zod'
|
||||
import { relayOpsEnvironment } from './environment-config.js'
|
||||
import { createGcloudClient } from './gcloud-client.js'
|
||||
import {
|
||||
AdmissionSelectorSchema,
|
||||
normalizeSelectorMembership,
|
||||
type AdmissionSelector
|
||||
} from './incident-selector.js'
|
||||
import {
|
||||
initialIncidentMonitorState,
|
||||
preDrainDryRunPassed,
|
||||
runIncidentMonitor,
|
||||
type IncidentCheckpoint,
|
||||
type IncidentSample,
|
||||
type IncidentMonitorState
|
||||
} from './incident-monitor.js'
|
||||
import { createIncidentSampleCollector } from './incident-monitor-sources.js'
|
||||
|
||||
const StateSchema = z.object({
|
||||
schemaVersion: z.literal(4),
|
||||
incidentId: z.string(),
|
||||
environment: z.enum(['production', 'staging']),
|
||||
expectedSelector: AdmissionSelectorSchema,
|
||||
preDrainDryRun: z.boolean(),
|
||||
migrationPolicy: z.enum(['strict', 'recover-forward', 'capacity-transition']),
|
||||
recoverySourceCellId: z.string().nullable(),
|
||||
capacityCellId: z.string().nullable(),
|
||||
startedAt: z.string(),
|
||||
windowStartedAt: z.string().nullable(),
|
||||
windowSequence: z.number().int().nonnegative(),
|
||||
durationMinutes: z.number().int(),
|
||||
intervalMs: z.number().int(),
|
||||
nextCheckpointIndex: z.number().int().nonnegative(),
|
||||
sampleCount: z.number().int().nonnegative(),
|
||||
totalSampleCount: z.number().int().nonnegative(),
|
||||
lastSampleAt: z.string().nullable(),
|
||||
continuityEvents: z.array(z.object({
|
||||
recordedAt: z.string(),
|
||||
windowSequence: z.number().int().nonnegative(),
|
||||
// Pre-2026-09-05 state files predate tolerated freshness gaps.
|
||||
tolerated: z.boolean().default(false),
|
||||
failures: z.array(z.object({
|
||||
code: z.string(),
|
||||
source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']),
|
||||
signal: z.string().optional(),
|
||||
observed: z.number().optional(),
|
||||
threshold: z.number().optional()
|
||||
}))
|
||||
})),
|
||||
frozenAt: z.string().nullable(),
|
||||
failures: z.array(z.object({
|
||||
code: z.string(),
|
||||
source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']),
|
||||
signal: z.string().optional(),
|
||||
observed: z.number().optional(),
|
||||
threshold: z.number().optional()
|
||||
})),
|
||||
// Pre-2026-09-17 state files predate cell-probe tolerance; a resumed run that
|
||||
// carries no streak is one that also restarts its window, so it earns nothing.
|
||||
probeStreaks: z.record(z.string(), z.number().int().nonnegative()).default({}),
|
||||
toleratedProbeEvents: z.array(z.object({
|
||||
recordedAt: z.string(),
|
||||
windowSequence: z.number().int().nonnegative(),
|
||||
failures: z.array(z.object({
|
||||
code: z.string(),
|
||||
source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']),
|
||||
signal: z.string().optional(),
|
||||
observed: z.number().optional(),
|
||||
threshold: z.number().optional()
|
||||
}))
|
||||
})).default([]),
|
||||
completedAt: z.string().nullable()
|
||||
})
|
||||
|
||||
type CliOptions = {
|
||||
environment: 'production' | 'staging'
|
||||
incidentId: string
|
||||
durationMinutes: number
|
||||
intervalMs: number
|
||||
expectedSelector: AdmissionSelector
|
||||
stateFile: string
|
||||
summaryFile: string
|
||||
markdownFile: string
|
||||
restart: boolean
|
||||
preDrainDryRun: boolean
|
||||
migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition'
|
||||
recoverySourceCellId: string | null
|
||||
capacityCellId: string | null
|
||||
maxSamplesThisRun: number | null
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value: string | undefined, name: string): number {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isSafeInteger(parsed) || parsed <= 0) {
|
||||
throw new Error(`${name} must be a positive integer`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
export function parseIncidentMonitorArguments(argv: string[], cwd = process.cwd()): CliOptions {
|
||||
const flags = new Set(['restart', 'pre-drain-dry-run'])
|
||||
const valueArguments = new Set([
|
||||
'duration-minutes',
|
||||
'environment',
|
||||
'expected-selector-generation',
|
||||
'expected-existing-only-cells',
|
||||
'expected-migration-only-cells',
|
||||
'expected-general-cells',
|
||||
'incident-id',
|
||||
'interval-seconds',
|
||||
'max-samples-this-run',
|
||||
'migration-policy',
|
||||
'output-directory',
|
||||
'recovery-source-cell-id',
|
||||
'capacity-cell-id'
|
||||
])
|
||||
const values: Record<string, string> = {}
|
||||
const enabledFlags = new Set<string>()
|
||||
for (let index = 0; index < argv.length; index++) {
|
||||
const argument = argv[index]
|
||||
if (!argument?.startsWith('--')) throw new Error(`invalid argument ${argument ?? ''}`)
|
||||
const name = argument.slice(2)
|
||||
if (flags.has(name)) {
|
||||
enabledFlags.add(name)
|
||||
continue
|
||||
}
|
||||
if (!valueArguments.has(name)) throw new Error(`unknown argument --${name}`)
|
||||
const value = argv[++index]
|
||||
if (!value || value.startsWith('--')) throw new Error(`missing --${name} value`)
|
||||
values[name] = value
|
||||
}
|
||||
const environment = z.enum(['production', 'staging']).parse(
|
||||
values.environment ?? 'production'
|
||||
)
|
||||
const incidentId = values['incident-id'] ?? randomUUID()
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{7,127}$/.test(incidentId)) {
|
||||
throw new Error('--incident-id must be 8-128 safe characters')
|
||||
}
|
||||
const selectorGeneration = Number(values['expected-selector-generation'])
|
||||
if (!Number.isSafeInteger(selectorGeneration) || selectorGeneration < 0) {
|
||||
throw new Error('--expected-selector-generation must be a nonnegative integer')
|
||||
}
|
||||
const configuredCells = new Set(
|
||||
relayOpsEnvironment(environment).cells.map((cell) => cell.cellId)
|
||||
)
|
||||
const cellList = (name: string): string[] => {
|
||||
const value = values[name]
|
||||
if (value === undefined) throw new Error(`--${name} is required; use none for an empty set`)
|
||||
return value === 'none' ? [] : value.split(',').map((cellId) => cellId.trim())
|
||||
}
|
||||
const expectedSelector = {
|
||||
generation: selectorGeneration,
|
||||
membership: normalizeSelectorMembership(
|
||||
{
|
||||
existingOnly: cellList('expected-existing-only-cells'),
|
||||
migrationOnly: cellList('expected-migration-only-cells'),
|
||||
general: cellList('expected-general-cells')
|
||||
},
|
||||
configuredCells
|
||||
)
|
||||
}
|
||||
if (selectorGeneration === 0 && expectedSelector.membership.migrationOnly.length > 0) {
|
||||
throw new Error('generation 0 cannot represent migration-only admission')
|
||||
}
|
||||
const preDrainDryRun = enabledFlags.has('pre-drain-dry-run')
|
||||
const migrationPolicy = z.enum([
|
||||
'strict',
|
||||
'recover-forward',
|
||||
'capacity-transition'
|
||||
]).parse(
|
||||
values['migration-policy'] ?? 'strict'
|
||||
)
|
||||
const recoverySourceCellId =
|
||||
values['recovery-source-cell-id'] === undefined ||
|
||||
values['recovery-source-cell-id'] === 'none'
|
||||
? null
|
||||
: values['recovery-source-cell-id']
|
||||
const capacityCellId =
|
||||
values['capacity-cell-id'] === undefined || values['capacity-cell-id'] === 'none'
|
||||
? null
|
||||
: values['capacity-cell-id']
|
||||
const durationMinutes = parsePositiveInteger(
|
||||
values['duration-minutes'] ?? (preDrainDryRun ? '15' : '90'),
|
||||
'--duration-minutes'
|
||||
)
|
||||
const intervalMs =
|
||||
parsePositiveInteger(values['interval-seconds'] ?? '60', '--interval-seconds') * 1_000
|
||||
if (durationMinutes < 15 || durationMinutes > 90) {
|
||||
throw new Error('--duration-minutes must be between 15 and 90')
|
||||
}
|
||||
if (intervalMs > 60_000) {
|
||||
throw new Error('--interval-seconds must be between 1 and 60')
|
||||
}
|
||||
if (preDrainDryRun && durationMinutes !== 15) {
|
||||
throw new Error('--pre-drain-dry-run requires --duration-minutes 15')
|
||||
}
|
||||
if (migrationPolicy !== 'strict' && !preDrainDryRun) {
|
||||
throw new Error(`--migration-policy ${migrationPolicy} requires --pre-drain-dry-run`)
|
||||
}
|
||||
if (
|
||||
migrationPolicy === 'recover-forward' &&
|
||||
(
|
||||
recoverySourceCellId === null ||
|
||||
!configuredCells.has(recoverySourceCellId) ||
|
||||
!expectedSelector.membership.existingOnly.includes(recoverySourceCellId)
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
'--migration-policy recover-forward requires an existing-only --recovery-source-cell-id'
|
||||
)
|
||||
}
|
||||
if (migrationPolicy !== 'recover-forward' && recoverySourceCellId !== null) {
|
||||
throw new Error('--recovery-source-cell-id requires --migration-policy recover-forward')
|
||||
}
|
||||
if (
|
||||
migrationPolicy === 'capacity-transition' &&
|
||||
(
|
||||
capacityCellId === null ||
|
||||
!configuredCells.has(capacityCellId) ||
|
||||
!expectedSelector.membership.general.includes(capacityCellId)
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
'--migration-policy capacity-transition requires a general --capacity-cell-id'
|
||||
)
|
||||
}
|
||||
if (migrationPolicy !== 'capacity-transition' && capacityCellId !== null) {
|
||||
throw new Error('--capacity-cell-id requires --migration-policy capacity-transition')
|
||||
}
|
||||
const directory = resolve(cwd, values['output-directory'] ?? '.relay-incidents')
|
||||
const maxSamplesThisRun = values['max-samples-this-run']
|
||||
? parsePositiveInteger(values['max-samples-this-run'], '--max-samples-this-run')
|
||||
: null
|
||||
return {
|
||||
environment,
|
||||
incidentId,
|
||||
durationMinutes,
|
||||
intervalMs,
|
||||
expectedSelector,
|
||||
stateFile: resolve(directory, `${incidentId}.state.json`),
|
||||
summaryFile: resolve(directory, `${incidentId}.summaries.jsonl`),
|
||||
markdownFile: resolve(directory, `${incidentId}.summary.md`),
|
||||
restart: enabledFlags.has('restart'),
|
||||
preDrainDryRun,
|
||||
migrationPolicy,
|
||||
recoverySourceCellId,
|
||||
capacityCellId,
|
||||
maxSamplesThisRun
|
||||
}
|
||||
}
|
||||
|
||||
async function fileExists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await stat(path)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function syncFile(path: string): Promise<void> {
|
||||
const handle = await open(path, 'r')
|
||||
try {
|
||||
await handle.sync()
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
async function persistState(path: string, state: IncidentMonitorState): Promise<void> {
|
||||
await mkdir(dirname(path), { recursive: true, mode: 0o700 })
|
||||
await chmod(dirname(path), 0o700)
|
||||
const temporaryPath = `${path}.tmp`
|
||||
await writeFile(temporaryPath, `${JSON.stringify(state)}\n`, { mode: 0o600 })
|
||||
await chmod(temporaryPath, 0o600)
|
||||
await syncFile(temporaryPath)
|
||||
await rename(temporaryPath, path)
|
||||
await syncFile(path)
|
||||
}
|
||||
|
||||
async function appendCheckpoint(path: string, checkpoint: IncidentCheckpoint): Promise<void> {
|
||||
await mkdir(dirname(path), { recursive: true, mode: 0o700 })
|
||||
await chmod(dirname(path), 0o700)
|
||||
if (await fileExists(path)) {
|
||||
const existing = (await readFile(path, 'utf8'))
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line) as IncidentCheckpoint)
|
||||
if (
|
||||
existing.some((entry) =>
|
||||
entry.windowSequence === checkpoint.windowSequence &&
|
||||
entry.checkpointMinute === checkpoint.checkpointMinute
|
||||
)
|
||||
) return
|
||||
}
|
||||
await appendFile(path, `${JSON.stringify(checkpoint)}\n`, { mode: 0o600 })
|
||||
await chmod(path, 0o600)
|
||||
await syncFile(path)
|
||||
}
|
||||
|
||||
function markdownFailure(checkpoint: IncidentCheckpoint): string {
|
||||
if (checkpoint.failures.length === 0) return 'none'
|
||||
return checkpoint.failures
|
||||
.map((failure) => {
|
||||
const signal = failure.signal ? `/${failure.signal}` : ''
|
||||
return `${failure.source}/${failure.code}${signal}`
|
||||
})
|
||||
.join(', ')
|
||||
}
|
||||
|
||||
async function writeMarkdownSummary(
|
||||
path: string,
|
||||
summaryPath: string,
|
||||
incidentId: string,
|
||||
environment: string
|
||||
): Promise<void> {
|
||||
const checkpoints = (await readFile(summaryPath, 'utf8'))
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line) as IncidentCheckpoint)
|
||||
const rows = checkpoints.map((checkpoint) => [
|
||||
`| ${checkpoint.windowSequence}`,
|
||||
checkpoint.checkpointMinute,
|
||||
checkpoint.status,
|
||||
checkpoint.sampleCount,
|
||||
`${markdownFailure(checkpoint)} |`
|
||||
].join(' | '))
|
||||
const markdown = [
|
||||
'# Relay incident monitor',
|
||||
'',
|
||||
`Incident: \`${incidentId}\``,
|
||||
'',
|
||||
`Environment: \`${environment}\``,
|
||||
'',
|
||||
`Expected selector: \`${JSON.stringify(checkpoints[0]?.expectedSelector ?? null)}\``,
|
||||
'',
|
||||
`Migration policy: \`${checkpoints[0]?.migrationPolicy ?? 'unknown'}\``,
|
||||
'',
|
||||
`Recovery source: \`${checkpoints[0]?.recoverySourceCellId ?? 'none'}\``,
|
||||
'',
|
||||
`Capacity cell: \`${checkpoints[0]?.capacityCellId ?? 'none'}\``,
|
||||
'',
|
||||
'| Window | Minute | Status | Samples | Failures |',
|
||||
'| ---: | ---: | --- | ---: | --- |',
|
||||
...rows,
|
||||
''
|
||||
].join('\n')
|
||||
const temporaryPath = `${path}.tmp`
|
||||
await writeFile(temporaryPath, markdown, { mode: 0o600 })
|
||||
await chmod(temporaryPath, 0o600)
|
||||
await syncFile(temporaryPath)
|
||||
await rename(temporaryPath, path)
|
||||
await syncFile(path)
|
||||
}
|
||||
|
||||
export function suppliedIdentityToken(value: string | undefined): string | null {
|
||||
if (value === undefined) return null
|
||||
if (
|
||||
value.length > 8_192 ||
|
||||
!/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(value)
|
||||
) {
|
||||
throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is invalid')
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
async function readInitialState(
|
||||
options: CliOptions,
|
||||
now: () => number = Date.now
|
||||
): Promise<IncidentMonitorState> {
|
||||
const exists = await fileExists(options.stateFile)
|
||||
if (exists && !options.restart) {
|
||||
throw new Error('incident state already exists; pass --restart to resume it')
|
||||
}
|
||||
if (!exists && options.restart) throw new Error('no incident state exists to restart')
|
||||
if (!exists) {
|
||||
return initialIncidentMonitorState({
|
||||
incidentId: options.incidentId,
|
||||
environment: options.environment,
|
||||
expectedSelector: options.expectedSelector,
|
||||
preDrainDryRun: options.preDrainDryRun,
|
||||
migrationPolicy: options.migrationPolicy,
|
||||
recoverySourceCellId: options.recoverySourceCellId,
|
||||
capacityCellId: options.capacityCellId,
|
||||
startedAt: new Date(now()).toISOString(),
|
||||
durationMinutes: options.durationMinutes,
|
||||
intervalMs: options.intervalMs
|
||||
})
|
||||
}
|
||||
const state = StateSchema.parse(
|
||||
JSON.parse(await readFile(options.stateFile, 'utf8'))
|
||||
) as IncidentMonitorState
|
||||
if (
|
||||
state.incidentId !== options.incidentId ||
|
||||
state.environment !== options.environment ||
|
||||
JSON.stringify(state.expectedSelector) !== JSON.stringify(options.expectedSelector) ||
|
||||
state.preDrainDryRun !== options.preDrainDryRun ||
|
||||
state.migrationPolicy !== options.migrationPolicy ||
|
||||
state.recoverySourceCellId !== options.recoverySourceCellId ||
|
||||
state.capacityCellId !== options.capacityCellId ||
|
||||
state.durationMinutes !== options.durationMinutes ||
|
||||
state.intervalMs !== options.intervalMs
|
||||
) {
|
||||
throw new Error('restart arguments do not match durable incident state')
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
export async function runIncidentMonitorCli(
|
||||
argv: string[],
|
||||
dependencies: {
|
||||
cwd?: string
|
||||
now?: () => number
|
||||
wait?: (ms: number) => Promise<void>
|
||||
gcloud?: ReturnType<typeof createGcloudClient>
|
||||
collect?: () => Promise<IncidentSample>
|
||||
writeOutput?: (value: string) => void
|
||||
environment?: NodeJS.ProcessEnv
|
||||
} = {}
|
||||
): Promise<number> {
|
||||
const options = parseIncidentMonitorArguments(argv, dependencies.cwd)
|
||||
const state = await readInitialState(options, dependencies.now)
|
||||
const baseGcloud = dependencies.gcloud ?? createGcloudClient()
|
||||
const token = suppliedIdentityToken(
|
||||
(dependencies.environment ?? process.env).ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
)
|
||||
await persistState(options.stateFile, state)
|
||||
const gcloud = token
|
||||
? { ...baseGcloud, identityToken: async () => token }
|
||||
: baseGcloud
|
||||
const collect =
|
||||
dependencies.collect ??
|
||||
createIncidentSampleCollector(gcloud, {
|
||||
environment: options.environment,
|
||||
expectedSelector: options.expectedSelector,
|
||||
...(dependencies.now ? { now: dependencies.now } : {})
|
||||
})
|
||||
let samplesThisRun = 0
|
||||
const segmentedCollect = async (): Promise<IncidentSample> => {
|
||||
try {
|
||||
return await collect()
|
||||
} finally {
|
||||
samplesThisRun++
|
||||
}
|
||||
}
|
||||
const wait =
|
||||
dependencies.wait ??
|
||||
((ms: number) => new Promise((resolvePromise) => setTimeout(resolvePromise, ms)))
|
||||
const segmentComplete = Symbol('segment-complete')
|
||||
const output = dependencies.writeOutput ?? ((value) => process.stdout.write(`${value}\n`))
|
||||
let result: IncidentMonitorState
|
||||
try {
|
||||
result = await runIncidentMonitor(state, {
|
||||
now: dependencies.now ?? Date.now,
|
||||
wait: async (ms) => {
|
||||
if (
|
||||
options.maxSamplesThisRun !== null &&
|
||||
samplesThisRun >= options.maxSamplesThisRun
|
||||
) {
|
||||
throw segmentComplete
|
||||
}
|
||||
await wait(ms)
|
||||
},
|
||||
collect: segmentedCollect,
|
||||
warn: (message) => console.warn(message),
|
||||
persist: async (nextState) => await persistState(options.stateFile, nextState),
|
||||
checkpoint: async (checkpoint) => {
|
||||
await appendCheckpoint(options.summaryFile, checkpoint)
|
||||
await writeMarkdownSummary(
|
||||
options.markdownFile,
|
||||
options.summaryFile,
|
||||
options.incidentId,
|
||||
options.environment
|
||||
)
|
||||
output(JSON.stringify(checkpoint))
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
if (error !== segmentComplete) throw error
|
||||
return 0
|
||||
}
|
||||
if (options.preDrainDryRun && !preDrainDryRunPassed(result)) return 2
|
||||
return result.frozenAt ? 2 : 0
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
runIncidentMonitorCli(process.argv.slice(2))
|
||||
.then((code) => {
|
||||
process.exitCode = code
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
console.error(error instanceof Error ? error.message : 'incident monitor failed')
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,655 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js'
|
||||
import type { GcloudClient } from './gcloud-client.js'
|
||||
import { effectiveAdmissionState } from './incident-selector.js'
|
||||
import { INCIDENT_MONITOR_THRESHOLDS } from './incident-monitor.js'
|
||||
import {
|
||||
directorSignals,
|
||||
GOOGLE_METRICS,
|
||||
readGoogleMetric,
|
||||
readGoogleMetricWithEmptyRetry,
|
||||
relayFiveMinuteDeltaSignal
|
||||
} from './incident-monitor-sources.js'
|
||||
|
||||
const now = Date.parse('2026-07-28T10:00:00.000Z')
|
||||
const startAt = new Date(now - 5 * 60_000).toISOString()
|
||||
const endAt = new Date(now).toISOString()
|
||||
const productionCells = RELAY_OPS_ENVIRONMENTS.production.cells.map(
|
||||
({ cellId }) => cellId
|
||||
)
|
||||
|
||||
describe('incident monitor sources', () => {
|
||||
it('uses legacy booleans only at selector generation zero', () => {
|
||||
const membership = {
|
||||
existingOnly: ['c1'],
|
||||
migrationOnly: [],
|
||||
general: ['c2']
|
||||
}
|
||||
expect(
|
||||
effectiveAdmissionState({ generation: 0, membership }, true, 'c1')
|
||||
).toBe('general')
|
||||
expect(
|
||||
effectiveAdmissionState({ generation: 1, membership }, true, 'c1')
|
||||
).toBe('existing-only')
|
||||
})
|
||||
|
||||
it('sums DELTA metrics across the window and scopes every target exactly', async () => {
|
||||
const filters: string[] = []
|
||||
const fetchImpl: typeof fetch = async (input) => {
|
||||
const url = new URL(String(input))
|
||||
filters.push(url.searchParams.get('filter') ?? '')
|
||||
return Response.json({
|
||||
timeSeries: [
|
||||
{
|
||||
points: [
|
||||
{
|
||||
interval: { endTime: new Date(now - 120_000).toISOString() },
|
||||
value: { int64Value: '2' }
|
||||
},
|
||||
{
|
||||
interval: { endTime: new Date(now - 60_000).toISOString() },
|
||||
value: { int64Value: '3' }
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
const environment = RELAY_OPS_ENVIRONMENTS.production
|
||||
const directorErrors = GOOGLE_METRICS.find(
|
||||
(definition) => definition.signal === 'director.errors'
|
||||
)!
|
||||
const deadlocks = GOOGLE_METRICS.find(
|
||||
(definition) => definition.signal === 'cloud_sql.deadlocks'
|
||||
)!
|
||||
await expect(
|
||||
readGoogleMetric(
|
||||
environment,
|
||||
directorErrors,
|
||||
'secret-access-token',
|
||||
startAt,
|
||||
endAt,
|
||||
fetchImpl
|
||||
)
|
||||
).resolves.toEqual({
|
||||
value: 5,
|
||||
observedAt: new Date(now - 60_000).toISOString()
|
||||
})
|
||||
await readGoogleMetric(
|
||||
environment,
|
||||
deadlocks,
|
||||
'secret-access-token',
|
||||
startAt,
|
||||
endAt,
|
||||
fetchImpl
|
||||
)
|
||||
expect(filters[0]).toContain(
|
||||
'resource.label."service_name"="orca-cloud-relay"'
|
||||
)
|
||||
expect(filters[0]).toContain('metric.label."response_code"!="503"')
|
||||
expect(filters[1]).toContain(
|
||||
'resource.label."database_id"="onorca-cloud:orca-cloud-auth-db"'
|
||||
)
|
||||
})
|
||||
|
||||
it('zero-fills an expired sparse lock-wait point', async () => {
|
||||
let pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
|
||||
const fetchImpl: typeof fetch = async () => Response.json({
|
||||
timeSeries: [{
|
||||
points: [{
|
||||
interval: { endTime: new Date(pointAt).toISOString() },
|
||||
value: { int64Value: '1' }
|
||||
}]
|
||||
}]
|
||||
})
|
||||
const definition = GOOGLE_METRICS.find(
|
||||
({ signal }) => signal === 'cloud_sql.lock_waits'
|
||||
)!
|
||||
await expect(readGoogleMetric(
|
||||
RELAY_OPS_ENVIRONMENTS.production,
|
||||
definition,
|
||||
'secret-access-token',
|
||||
startAt,
|
||||
endAt,
|
||||
fetchImpl
|
||||
)).resolves.toEqual({
|
||||
value: 1,
|
||||
observedAt: new Date(pointAt).toISOString()
|
||||
})
|
||||
await expect(readGoogleMetric(
|
||||
RELAY_OPS_ENVIRONMENTS.production,
|
||||
definition,
|
||||
'secret-access-token',
|
||||
startAt,
|
||||
endAt,
|
||||
fetchImpl,
|
||||
() => now + 3_000
|
||||
)).resolves.toEqual({
|
||||
value: 1,
|
||||
observedAt: new Date(pointAt).toISOString()
|
||||
})
|
||||
pointAt--
|
||||
await expect(readGoogleMetric(
|
||||
RELAY_OPS_ENVIRONMENTS.production,
|
||||
definition,
|
||||
'secret-access-token',
|
||||
startAt,
|
||||
endAt,
|
||||
fetchImpl
|
||||
)).resolves.toEqual({ value: 0, observedAt: endAt })
|
||||
})
|
||||
|
||||
it('freshens a sparse zero without masking a recent nonzero lock wait', async () => {
|
||||
let value = 0
|
||||
const pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
|
||||
const readAt = now + 11_879
|
||||
const fetchImpl: typeof fetch = async () => Response.json({
|
||||
timeSeries: [{
|
||||
points: [{
|
||||
interval: { endTime: new Date(pointAt).toISOString() },
|
||||
value: { int64Value: String(value) }
|
||||
}]
|
||||
}]
|
||||
})
|
||||
const definition = GOOGLE_METRICS.find(
|
||||
({ signal }) => signal === 'cloud_sql.lock_waits'
|
||||
)!
|
||||
|
||||
const sparseZero = await readGoogleMetric(
|
||||
RELAY_OPS_ENVIRONMENTS.production,
|
||||
definition,
|
||||
'secret-access-token',
|
||||
startAt,
|
||||
endAt,
|
||||
fetchImpl,
|
||||
() => readAt
|
||||
)
|
||||
expect(sparseZero).toEqual({
|
||||
value: 0,
|
||||
observedAt: new Date(readAt).toISOString()
|
||||
})
|
||||
expect(readAt - pointAt).toBe(191_879)
|
||||
expect(readAt - Date.parse(sparseZero!.observedAt)).toBe(0)
|
||||
|
||||
value = 20
|
||||
await expect(readGoogleMetric(
|
||||
RELAY_OPS_ENVIRONMENTS.production,
|
||||
definition,
|
||||
'secret-access-token',
|
||||
startAt,
|
||||
endAt,
|
||||
fetchImpl,
|
||||
() => readAt
|
||||
)).resolves.toEqual({
|
||||
value: 20,
|
||||
observedAt: new Date(pointAt).toISOString()
|
||||
})
|
||||
})
|
||||
|
||||
it('preserves a recent nonzero lock wait across staggered series', async () => {
|
||||
const nonzeroAt = now - 179_000
|
||||
const zeroAt = now - 178_000
|
||||
const definition = GOOGLE_METRICS.find(
|
||||
({ signal }) => signal === 'cloud_sql.lock_waits'
|
||||
)!
|
||||
const metric = await readGoogleMetric(
|
||||
RELAY_OPS_ENVIRONMENTS.production,
|
||||
definition,
|
||||
'secret-access-token',
|
||||
startAt,
|
||||
endAt,
|
||||
async () => Response.json({
|
||||
timeSeries: [
|
||||
{
|
||||
points: [{
|
||||
interval: { endTime: new Date(nonzeroAt).toISOString() },
|
||||
value: { int64Value: '7' }
|
||||
}]
|
||||
},
|
||||
{
|
||||
points: [{
|
||||
interval: { endTime: new Date(zeroAt).toISOString() },
|
||||
value: { int64Value: '0' }
|
||||
}]
|
||||
}
|
||||
]
|
||||
})
|
||||
)
|
||||
|
||||
expect(metric).toEqual({
|
||||
value: 7,
|
||||
observedAt: new Date(nonzeroAt).toISOString()
|
||||
})
|
||||
|
||||
await expect(readGoogleMetric(
|
||||
RELAY_OPS_ENVIRONMENTS.production,
|
||||
definition,
|
||||
'secret-access-token',
|
||||
startAt,
|
||||
endAt,
|
||||
async () => Response.json({
|
||||
timeSeries: [{
|
||||
points: [
|
||||
{
|
||||
interval: { endTime: new Date(nonzeroAt).toISOString() },
|
||||
value: { int64Value: '7' }
|
||||
},
|
||||
{
|
||||
interval: { endTime: new Date(zeroAt).toISOString() },
|
||||
value: { int64Value: '0' }
|
||||
}
|
||||
]
|
||||
}]
|
||||
})
|
||||
)).resolves.toEqual({ value: 0, observedAt: endAt })
|
||||
})
|
||||
|
||||
it('retries an empty required metric without weakening its value', async () => {
|
||||
let calls = 0
|
||||
const waits: number[] = []
|
||||
const definition = GOOGLE_METRICS.find(
|
||||
({ signal }) => signal === 'cloud_sql.cpu'
|
||||
)!
|
||||
await expect(readGoogleMetricWithEmptyRetry(
|
||||
RELAY_OPS_ENVIRONMENTS.production,
|
||||
definition,
|
||||
'secret-access-token',
|
||||
startAt,
|
||||
endAt,
|
||||
async () => Response.json(calls++ === 0
|
||||
? { timeSeries: [] }
|
||||
: {
|
||||
timeSeries: [{
|
||||
points: [{
|
||||
interval: { endTime: new Date(now - 60_000).toISOString() },
|
||||
value: { doubleValue: 0.81 }
|
||||
}]
|
||||
}]
|
||||
}),
|
||||
() => now,
|
||||
async (ms) => { waits.push(ms) }
|
||||
)).resolves.toEqual({
|
||||
value: 0.81,
|
||||
observedAt: new Date(now - 60_000).toISOString()
|
||||
})
|
||||
expect(calls).toBe(2)
|
||||
expect(waits).toEqual([2_000])
|
||||
})
|
||||
|
||||
it('still reports a required metric missing after bounded retries', async () => {
|
||||
let calls = 0
|
||||
const waits: number[] = []
|
||||
const definition = GOOGLE_METRICS.find(
|
||||
({ signal }) => signal === 'director.concurrency'
|
||||
)!
|
||||
await expect(readGoogleMetricWithEmptyRetry(
|
||||
RELAY_OPS_ENVIRONMENTS.production,
|
||||
definition,
|
||||
'secret-access-token',
|
||||
startAt,
|
||||
endAt,
|
||||
async () => {
|
||||
calls++
|
||||
return Response.json({ timeSeries: [] })
|
||||
},
|
||||
() => now,
|
||||
async (ms) => { waits.push(ms) }
|
||||
)).resolves.toBeNull()
|
||||
expect(calls).toBe(3)
|
||||
expect(waits).toEqual([2_000, 2_000])
|
||||
})
|
||||
|
||||
it('timestamps sparse retry aggregates at query completion', () => {
|
||||
expect(relayFiveMinuteDeltaSignal({
|
||||
available: true,
|
||||
points: [
|
||||
{ at: new Date(now - 22 * 60_000).toISOString(), value: 7 },
|
||||
{ at: new Date(now - 4 * 60_000).toISOString(), value: 2 }
|
||||
]
|
||||
}, endAt)).toEqual({ value: 2, observedAt: endAt })
|
||||
expect(relayFiveMinuteDeltaSignal({
|
||||
available: true,
|
||||
points: [{ at: new Date(now - 22 * 60_000).toISOString(), value: 7 }]
|
||||
}, endAt)).toEqual({ value: 0, observedAt: endAt })
|
||||
expect(relayFiveMinuteDeltaSignal({
|
||||
available: false,
|
||||
points: []
|
||||
}, endAt)).toBeNull()
|
||||
})
|
||||
|
||||
// Why: the per-region cell latency bar is only correct if the tfvars region
|
||||
// reaches the evaluator on every cell expectation.
|
||||
it('carries the configured region onto every cell expectation', async () => {
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => 'unused',
|
||||
identityToken: async () => 'unused'
|
||||
}
|
||||
const selector = {
|
||||
generation: 1,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: [],
|
||||
general: productionCells
|
||||
}
|
||||
}
|
||||
const fetchImpl: typeof fetch = async (_input, init) => {
|
||||
const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
|
||||
if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
|
||||
if (body.cellId) {
|
||||
return Response.json({
|
||||
status: {
|
||||
enabled: true,
|
||||
connectionCapacity: { hardCap: 600 },
|
||||
runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true }
|
||||
}
|
||||
})
|
||||
}
|
||||
return Response.json({
|
||||
blocked: 0,
|
||||
blockedExpiredUnregistered: 0,
|
||||
registeredTargetInactive: 0
|
||||
})
|
||||
}
|
||||
const result = await directorSignals('production', selector, gcloud, now, fetchImpl)
|
||||
const regionById = new Map(result.cells.map((cell) => [cell.cellId, cell.region]))
|
||||
expect(regionById.get('production-gce-c1')).toBe('us-central1')
|
||||
expect(regionById.get('production-gce-c27')).toBe('asia-east2')
|
||||
expect(result.cells).toHaveLength(productionCells.length)
|
||||
for (const cell of RELAY_OPS_ENVIRONMENTS.production.cells) {
|
||||
expect(regionById.get(cell.cellId)).toBe(cell.region)
|
||||
}
|
||||
})
|
||||
|
||||
it('aggregates admin state without returning tokens or response identities', async () => {
|
||||
const identityToken = 'secret.header.signature'
|
||||
const sensitiveIdentity = 'user@example.test'
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => 'unused',
|
||||
identityToken: async () => identityToken
|
||||
}
|
||||
let activeRequests = 0
|
||||
let maximumActiveRequests = 0
|
||||
let requestCount = 0
|
||||
const fetchImpl: typeof fetch = async (_input, init) => {
|
||||
requestCount++
|
||||
activeRequests++
|
||||
maximumActiveRequests = Math.max(maximumActiveRequests, activeRequests)
|
||||
expect(new Headers(init?.headers).get('authorization')).toBe(
|
||||
`Bearer ${identityToken}`
|
||||
)
|
||||
const body = JSON.parse(String(init?.body)) as {
|
||||
cellId?: string
|
||||
sourceCellId?: string
|
||||
targetCellId?: string
|
||||
}
|
||||
await Promise.resolve()
|
||||
activeRequests--
|
||||
if (!body.cellId && !body.sourceCellId) {
|
||||
return Response.json({
|
||||
selector: {
|
||||
generation: 1,
|
||||
membership: {
|
||||
existingOnly: productionCells.slice(2),
|
||||
migrationOnly: ['production-gce-c2'],
|
||||
general: ['production-gce-c1']
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if (body.cellId) {
|
||||
return Response.json({
|
||||
status: {
|
||||
// Selector-era monitoring must ignore this legacy compatibility bit.
|
||||
enabled: body.cellId !== 'production-gce-c1',
|
||||
connectionCapacity: {
|
||||
hardCap: body.cellId === 'production-gce-c1' ? 1_000 : 600
|
||||
},
|
||||
runtime: {
|
||||
lastHeartbeatAt: now - 1_000,
|
||||
heartbeatFresh: true
|
||||
},
|
||||
userId: sensitiveIdentity
|
||||
}
|
||||
})
|
||||
}
|
||||
return Response.json({
|
||||
blocked: 0,
|
||||
blockedExpiredUnregistered:
|
||||
body.sourceCellId === 'production-gce-c1' &&
|
||||
body.targetCellId === 'production-gce-c2'
|
||||
? 1
|
||||
: 0,
|
||||
registeredTargetInactive: 0,
|
||||
userId: sensitiveIdentity
|
||||
})
|
||||
}
|
||||
const result = await directorSignals(
|
||||
'production',
|
||||
{
|
||||
generation: 1,
|
||||
membership: {
|
||||
existingOnly: productionCells.slice(2),
|
||||
migrationOnly: ['production-gce-c2'],
|
||||
general: ['production-gce-c1']
|
||||
}
|
||||
},
|
||||
gcloud,
|
||||
now,
|
||||
fetchImpl
|
||||
)
|
||||
expect(requestCount).toBe(productionCells.length * 2)
|
||||
expect(maximumActiveRequests).toBe(1)
|
||||
expect(
|
||||
result.source.signals['cell.production-gce-c1.migration_blocked']
|
||||
).toMatchObject({ value: 1 })
|
||||
expect(result.cells.find((cell) => cell.cellId === 'production-gce-c1')).toMatchObject({
|
||||
expectedAdmissionState: 'general'
|
||||
})
|
||||
expect(
|
||||
result.source.signals['cell.production-gce-c1.admission_state']
|
||||
).toMatchObject({ value: 2 })
|
||||
expect(
|
||||
result.source.signals['cell.production-gce-c1.connection_hard_cap']
|
||||
).toMatchObject({ value: 1_000 })
|
||||
expect(
|
||||
result.source.signals['cell.production-gce-c2.admission_state']
|
||||
).toMatchObject({ value: 1 })
|
||||
const serialized = JSON.stringify(result)
|
||||
expect(serialized).not.toContain(identityToken)
|
||||
expect(serialized).not.toContain(sensitiveIdentity)
|
||||
})
|
||||
|
||||
// Why: the director maps a Cloud SQL pool connect timeout in cell-status onto
|
||||
// a 404, so without this the whole sample dies on one 2 s database stall.
|
||||
it('retries a transient director admin failure and then reports the cell', async () => {
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => 'unused',
|
||||
identityToken: async () => 'unused'
|
||||
}
|
||||
const selector = {
|
||||
generation: 1,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: productionCells }
|
||||
}
|
||||
const waits: number[] = []
|
||||
let cellStatusCalls = 0
|
||||
const fetchImpl: typeof fetch = async (_input, init) => {
|
||||
const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
|
||||
if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
|
||||
if (body.cellId) {
|
||||
cellStatusCalls++
|
||||
if (cellStatusCalls === 1) {
|
||||
return Response.json(
|
||||
{ error: 'timeout exceeded when trying to connect' },
|
||||
{ status: 404 }
|
||||
)
|
||||
}
|
||||
return Response.json({
|
||||
status: {
|
||||
enabled: true,
|
||||
connectionCapacity: { hardCap: 600 },
|
||||
runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true }
|
||||
}
|
||||
})
|
||||
}
|
||||
return Response.json({
|
||||
blocked: 0,
|
||||
blockedExpiredUnregistered: 0,
|
||||
registeredTargetInactive: 0
|
||||
})
|
||||
}
|
||||
const result = await directorSignals(
|
||||
'production',
|
||||
selector,
|
||||
gcloud,
|
||||
now,
|
||||
fetchImpl,
|
||||
async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
)
|
||||
expect(waits).toEqual([5_000])
|
||||
expect(cellStatusCalls).toBe(productionCells.length + 1)
|
||||
expect(result.cells).toHaveLength(productionCells.length)
|
||||
expect(
|
||||
result.source.signals['cell.production-gce-c1.connection_hard_cap']
|
||||
).toMatchObject({ value: 600 })
|
||||
})
|
||||
|
||||
// A rejected admin token is a decision, not weather: retrying it only burns
|
||||
// the sample budget and hides the misconfiguration.
|
||||
it('fails immediately on an unauthorized director admin response', async () => {
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => 'unused',
|
||||
identityToken: async () => 'unused'
|
||||
}
|
||||
const selector = {
|
||||
generation: 1,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: productionCells }
|
||||
}
|
||||
const waits: number[] = []
|
||||
let requestCount = 0
|
||||
const fetchImpl: typeof fetch = async (_input, init) => {
|
||||
requestCount++
|
||||
const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
|
||||
if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
|
||||
return Response.json({ error: 'invalid_token' }, { status: 401 })
|
||||
}
|
||||
await expect(
|
||||
directorSignals('production', selector, gcloud, now, fetchImpl, async (ms) => {
|
||||
waits.push(ms)
|
||||
})
|
||||
).rejects.toThrow('Relay admin telemetry returned 401')
|
||||
expect(requestCount).toBe(2)
|
||||
expect(waits).toEqual([])
|
||||
})
|
||||
|
||||
// Why: on 2026-10-19 cell-status refused reserve production-gce-c7 with a 404 on every sample.
|
||||
it('marks a reserve cell unprobed and skips its refused migration reads', async () => {
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => 'unused',
|
||||
identityToken: async () => 'unused'
|
||||
}
|
||||
const reserveCellId = 'production-gce-c7'
|
||||
const selector = {
|
||||
generation: 1,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: ['production-gce-c2'],
|
||||
general: productionCells.filter((cellId) => cellId !== 'production-gce-c2')
|
||||
}
|
||||
}
|
||||
const waits: number[] = []
|
||||
const fetchImpl: typeof fetch = async (input, init) => {
|
||||
const path = new URL(String(input)).pathname
|
||||
const body = JSON.parse(String(init?.body)) as Record<string, unknown>
|
||||
if (path === '/v1/admin/admission-selector/status') return Response.json({ selector })
|
||||
if (path === '/v1/admin/cell-status') {
|
||||
if (body.cellId === reserveCellId) {
|
||||
return Response.json({ error: 'cell_in_reserve_mode' }, { status: 404 })
|
||||
}
|
||||
return Response.json({
|
||||
status: {
|
||||
enabled: true,
|
||||
connectionCapacity: { hardCap: 600 },
|
||||
runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true }
|
||||
}
|
||||
})
|
||||
}
|
||||
expect(path).toBe('/v1/admin/evacuation-status')
|
||||
expect(body.sourceCellId).not.toBe(reserveCellId)
|
||||
return Response.json({
|
||||
blocked: 0,
|
||||
blockedExpiredUnregistered: 0,
|
||||
registeredTargetInactive: 0
|
||||
})
|
||||
}
|
||||
const result = await directorSignals('production', selector, gcloud, now, fetchImpl, async (ms) => {
|
||||
waits.push(ms)
|
||||
})
|
||||
expect(waits).toEqual([])
|
||||
expect(result.cells.find((cell) => cell.cellId === reserveCellId)).toMatchObject({
|
||||
reserve: true
|
||||
})
|
||||
expect(result.cells.find((cell) => cell.cellId === 'production-gce-c1')).not.toHaveProperty(
|
||||
'reserve'
|
||||
)
|
||||
const signals = result.source.signals
|
||||
expect(signals[`cell.${reserveCellId}.admission_state`]).toMatchObject({ value: 2 })
|
||||
// Why: 10-19 run 39225932592 froze reserve c7 at 731 connections against the 500 default.
|
||||
const configuredCap = RELAY_OPS_ENVIRONMENTS.production.cells.find(
|
||||
(cell) => cell.cellId === reserveCellId
|
||||
)?.connectionHardCap
|
||||
expect(configuredCap).toBe(1_000)
|
||||
expect(signals[`cell.${reserveCellId}.connection_hard_cap`]).toMatchObject({
|
||||
value: configuredCap
|
||||
})
|
||||
for (const suffix of ['heartbeat_fresh', 'heartbeat_age_ms', 'migration_blocked']) {
|
||||
expect(signals[`cell.${reserveCellId}.${suffix}`]).toBeUndefined()
|
||||
}
|
||||
expect(signals['cell.production-gce-c1.heartbeat_age_ms']).toMatchObject({ value: 1_000 })
|
||||
})
|
||||
|
||||
it('still fails a cell-status 404 that is not the reserve refusal', async () => {
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => 'unused',
|
||||
identityToken: async () => 'unused'
|
||||
}
|
||||
const selector = {
|
||||
generation: 1,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: productionCells }
|
||||
}
|
||||
const collect = (error: string) =>
|
||||
directorSignals('production', selector, gcloud, now, async (input) => {
|
||||
const path = new URL(String(input)).pathname
|
||||
if (path === '/v1/admin/admission-selector/status') return Response.json({ selector })
|
||||
return Response.json({ error }, { status: 404 })
|
||||
}, async () => {})
|
||||
for (const error of ['cell_not_found', 'cell_in_reserve_mode_soon', '']) {
|
||||
await expect(collect(error)).rejects.toThrow('Relay admin telemetry returned 404')
|
||||
}
|
||||
})
|
||||
|
||||
// A 404 the director means (wrong role) must not be retried either.
|
||||
it('does not retry a director-only 404', async () => {
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => 'unused',
|
||||
identityToken: async () => 'unused'
|
||||
}
|
||||
let requestCount = 0
|
||||
const fetchImpl: typeof fetch = async () => {
|
||||
requestCount++
|
||||
return Response.json({ error: 'director_only' }, { status: 404 })
|
||||
}
|
||||
await expect(
|
||||
directorSignals(
|
||||
'production',
|
||||
{ generation: 1, membership: { existingOnly: [], migrationOnly: [], general: productionCells } },
|
||||
gcloud,
|
||||
now,
|
||||
fetchImpl,
|
||||
async () => {}
|
||||
)
|
||||
).rejects.toThrow('Relay admin telemetry returned 404')
|
||||
expect(requestCount).toBe(1)
|
||||
})
|
||||
})
|
||||
@@ -1,783 +0,0 @@
|
||||
import { z } from 'zod'
|
||||
import { buildDashboardSnapshot } from './dashboard-snapshot.js'
|
||||
import type {
|
||||
RelayOpsEnvironment,
|
||||
RelayOpsEnvironmentId
|
||||
} from './environment-config.js'
|
||||
import { relayOpsEnvironment } from './environment-config.js'
|
||||
import type { GcloudClient } from './gcloud-client.js'
|
||||
import type { RelayMetricSnapshot } from './monitoring-snapshot.js'
|
||||
import {
|
||||
AdmissionSelectorSchema,
|
||||
effectiveAdmissionState,
|
||||
normalizeSelectorMembership,
|
||||
selectorCellState,
|
||||
type AdmissionSelector,
|
||||
} from './incident-selector.js'
|
||||
import {
|
||||
INCIDENT_MONITOR_THRESHOLDS,
|
||||
type IncidentSample,
|
||||
type IncidentSignal,
|
||||
type IncidentSource
|
||||
} from './incident-monitor.js'
|
||||
|
||||
const NumericSchema = z.union([z.number(), z.string()])
|
||||
.transform(Number)
|
||||
.pipe(z.number().finite())
|
||||
export const MonitoringPointSchema = z.object({
|
||||
interval: z.object({ endTime: z.string() }),
|
||||
value: z.object({
|
||||
doubleValue: NumericSchema.optional(),
|
||||
int64Value: NumericSchema.optional(),
|
||||
distributionValue: z.object({
|
||||
mean: NumericSchema.optional(),
|
||||
range: z.object({ max: NumericSchema.optional() }).optional()
|
||||
}).optional()
|
||||
})
|
||||
})
|
||||
export const MonitoringResponseSchema = z.object({
|
||||
timeSeries: z.array(z.object({ points: z.array(MonitoringPointSchema) })).default([]),
|
||||
nextPageToken: z.string().optional()
|
||||
})
|
||||
const CellStatusSchema = z.object({
|
||||
status: z.object({
|
||||
enabled: z.boolean(),
|
||||
connectionCapacity: z
|
||||
.object({ hardCap: z.number().int().positive() })
|
||||
.nullable(),
|
||||
runtime: z.object({
|
||||
lastHeartbeatAt: z.number(),
|
||||
heartbeatFresh: z.boolean()
|
||||
}).nullable()
|
||||
})
|
||||
})
|
||||
const SelectorStatusSchema = z.object({
|
||||
selector: AdmissionSelectorSchema
|
||||
})
|
||||
const MigrationStatusSchema = z.object({
|
||||
blocked: z.number().int().nonnegative(),
|
||||
registeredTargetInactive: z.number().int().nonnegative(),
|
||||
blockedExpiredUnregistered: z.number().int().nonnegative()
|
||||
})
|
||||
|
||||
export type GoogleMetricDefinition = {
|
||||
signal: string
|
||||
type: string
|
||||
resourceFilter: string
|
||||
aggregation: 'latest-max' | 'latest-sum' | 'window-sum'
|
||||
emptyIsZero?: boolean
|
||||
zeroAfterMs?: number
|
||||
}
|
||||
|
||||
export const GOOGLE_METRICS: GoogleMetricDefinition[] = [
|
||||
{
|
||||
signal: 'cloud_sql.cpu',
|
||||
type: 'cloudsql.googleapis.com/database/cpu/utilization',
|
||||
resourceFilter: 'resource.type="cloudsql_database"',
|
||||
aggregation: 'latest-max'
|
||||
},
|
||||
{
|
||||
signal: 'cloud_sql.memory',
|
||||
type: 'cloudsql.googleapis.com/database/memory/utilization',
|
||||
resourceFilter: 'resource.type="cloudsql_database"',
|
||||
aggregation: 'latest-max'
|
||||
},
|
||||
{
|
||||
signal: 'cloud_sql.backends',
|
||||
type: 'cloudsql.googleapis.com/database/postgresql/num_backends',
|
||||
resourceFilter: 'resource.type="cloudsql_database"',
|
||||
aggregation: 'latest-sum'
|
||||
},
|
||||
{
|
||||
signal: 'cloud_sql.lock_waits',
|
||||
type: 'cloudsql.googleapis.com/database/postgresql/backends_in_wait',
|
||||
resourceFilter:
|
||||
'resource.type="cloudsql_database" AND metric.label."wait_event_type"="Lock"',
|
||||
aggregation: 'latest-max',
|
||||
emptyIsZero: true,
|
||||
zeroAfterMs: INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
|
||||
},
|
||||
{
|
||||
signal: 'cloud_sql.deadlocks',
|
||||
type: 'cloudsql.googleapis.com/database/postgresql/deadlock_count',
|
||||
resourceFilter: 'resource.type="cloudsql_database"',
|
||||
aggregation: 'window-sum',
|
||||
emptyIsZero: true
|
||||
},
|
||||
{
|
||||
signal: 'director.instances',
|
||||
type: 'run.googleapis.com/container/instance_count',
|
||||
resourceFilter: 'resource.type="cloud_run_revision"',
|
||||
aggregation: 'latest-sum'
|
||||
},
|
||||
{
|
||||
signal: 'director.cpu',
|
||||
type: 'run.googleapis.com/container/cpu/utilizations',
|
||||
resourceFilter: 'resource.type="cloud_run_revision"',
|
||||
aggregation: 'latest-max'
|
||||
},
|
||||
{
|
||||
signal: 'director.memory',
|
||||
type: 'run.googleapis.com/container/memory/utilizations',
|
||||
resourceFilter: 'resource.type="cloud_run_revision"',
|
||||
aggregation: 'latest-max'
|
||||
},
|
||||
{
|
||||
signal: 'director.concurrency',
|
||||
type: 'run.googleapis.com/container/max_request_concurrencies',
|
||||
resourceFilter:
|
||||
'resource.type="cloud_run_revision" AND metric.label."state"="active"',
|
||||
aggregation: 'latest-max'
|
||||
},
|
||||
{
|
||||
signal: 'director.errors',
|
||||
type: 'run.googleapis.com/request_count',
|
||||
resourceFilter:
|
||||
'resource.type="cloud_run_revision" AND metric.label."response_code_class"="5xx" AND metric.label."response_code"!="503"',
|
||||
aggregation: 'window-sum',
|
||||
emptyIsZero: true
|
||||
},
|
||||
{
|
||||
signal: 'auth.errors',
|
||||
type: 'run.googleapis.com/request_count',
|
||||
resourceFilter:
|
||||
'resource.type="cloud_run_revision" AND metric.label."response_code_class"="5xx"',
|
||||
aggregation: 'window-sum',
|
||||
emptyIsZero: true
|
||||
},
|
||||
{
|
||||
signal: 'auth.requests',
|
||||
type: 'run.googleapis.com/request_count',
|
||||
resourceFilter: 'resource.type="cloud_run_revision"',
|
||||
aggregation: 'window-sum',
|
||||
emptyIsZero: true
|
||||
}
|
||||
]
|
||||
|
||||
export function pointValue(point: z.infer<typeof MonitoringPointSchema>): number {
|
||||
return (
|
||||
point.value.doubleValue ??
|
||||
point.value.int64Value ??
|
||||
point.value.distributionValue?.range?.max ??
|
||||
point.value.distributionValue?.mean ??
|
||||
0
|
||||
)
|
||||
}
|
||||
|
||||
function serviceFilter(signal: string, directorService: string, authService: string): string {
|
||||
if (signal.startsWith('director.')) {
|
||||
return `resource.label."service_name"="${directorService}"`
|
||||
}
|
||||
if (signal.startsWith('auth.')) {
|
||||
return `resource.label."service_name"="${authService}"`
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function targetFilter(
|
||||
definition: GoogleMetricDefinition,
|
||||
environment: RelayOpsEnvironment
|
||||
): string {
|
||||
if (definition.signal.startsWith('cloud_sql.')) {
|
||||
return `resource.label."database_id"="${environment.project}:${environment.sqlInstance}"`
|
||||
}
|
||||
return serviceFilter(
|
||||
definition.signal,
|
||||
environment.directorService,
|
||||
environment.authService
|
||||
)
|
||||
}
|
||||
|
||||
export async function googleJson(
|
||||
fetchImpl: typeof fetch,
|
||||
token: string,
|
||||
url: URL | string,
|
||||
init: RequestInit = {}
|
||||
): Promise<unknown> {
|
||||
const response = await fetchImpl(url, {
|
||||
...init,
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
...(init.body ? { 'content-type': 'application/json' } : {})
|
||||
},
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
if (!response.ok) throw new Error(`Google telemetry returned ${response.status}`)
|
||||
return await response.json()
|
||||
}
|
||||
|
||||
export async function readGoogleMetric(
|
||||
environment: RelayOpsEnvironment,
|
||||
definition: GoogleMetricDefinition,
|
||||
token: string,
|
||||
startAt: string,
|
||||
endAt: string,
|
||||
fetchImpl: typeof fetch,
|
||||
now: () => number = () => Date.parse(endAt)
|
||||
): Promise<IncidentSignal | null> {
|
||||
const url = new URL(
|
||||
`https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries`
|
||||
)
|
||||
const filters = [
|
||||
`metric.type="${definition.type}"`,
|
||||
definition.resourceFilter,
|
||||
targetFilter(definition, environment)
|
||||
].filter(Boolean)
|
||||
url.searchParams.set('filter', filters.join(' AND '))
|
||||
url.searchParams.set('interval.startTime', startAt)
|
||||
url.searchParams.set('interval.endTime', endAt)
|
||||
url.searchParams.set('view', 'FULL')
|
||||
url.searchParams.set('pageSize', '1000')
|
||||
const parsed = MonitoringResponseSchema.parse(
|
||||
await googleJson(fetchImpl, token, url)
|
||||
)
|
||||
if (parsed.nextPageToken) throw new Error('Google metric pagination is incomplete')
|
||||
const queryEndMs = Date.parse(endAt)
|
||||
const readAtMs = Math.max(queryEndMs, now())
|
||||
const readAt = new Date(readAtMs).toISOString()
|
||||
const points = parsed.timeSeries.flatMap((series) => series.points)
|
||||
if (points.length === 0) {
|
||||
return definition.emptyIsZero ? { value: 0, observedAt: readAt } : null
|
||||
}
|
||||
const zeroAfterMs = definition.zeroAfterMs
|
||||
if (definition.emptyIsZero && zeroAfterMs !== undefined) {
|
||||
const latestSeriesPoints = parsed.timeSeries.flatMap((series) => {
|
||||
const seriesNewestAt = Math.max(
|
||||
...series.points.map((point) => Date.parse(point.interval.endTime))
|
||||
)
|
||||
return series.points.filter(
|
||||
(point) => Date.parse(point.interval.endTime) === seriesNewestAt
|
||||
)
|
||||
})
|
||||
const futurePoints = latestSeriesPoints.filter(
|
||||
(point) => Date.parse(point.interval.endTime) > queryEndMs
|
||||
)
|
||||
if (futurePoints.length > 0) {
|
||||
return {
|
||||
value: Math.max(...futurePoints.map(pointValue)),
|
||||
observedAt: new Date(Math.max(
|
||||
...futurePoints.map((point) => Date.parse(point.interval.endTime))
|
||||
)).toISOString()
|
||||
}
|
||||
}
|
||||
const recentNonzero = latestSeriesPoints.filter((point) => {
|
||||
const pointAt = Date.parse(point.interval.endTime)
|
||||
return pointValue(point) > 0 && queryEndMs - pointAt <= zeroAfterMs
|
||||
})
|
||||
if (recentNonzero.length === 0) return { value: 0, observedAt: readAt }
|
||||
return {
|
||||
value: Math.max(...recentNonzero.map(pointValue)),
|
||||
observedAt: new Date(Math.min(
|
||||
...recentNonzero.map((point) => Date.parse(point.interval.endTime))
|
||||
)).toISOString()
|
||||
}
|
||||
}
|
||||
const newestAt = Math.max(...points.map((point) => Date.parse(point.interval.endTime)))
|
||||
const selected = definition.aggregation === 'window-sum'
|
||||
? points
|
||||
: points.filter((point) => Date.parse(point.interval.endTime) === newestAt)
|
||||
const values = selected.map(pointValue)
|
||||
const value =
|
||||
definition.aggregation !== 'latest-max'
|
||||
? values.reduce((total, entry) => total + entry, 0)
|
||||
: Math.max(...values)
|
||||
return {
|
||||
value,
|
||||
observedAt: new Date(newestAt).toISOString()
|
||||
}
|
||||
}
|
||||
|
||||
export async function readGoogleMetricWithEmptyRetry(
|
||||
environment: RelayOpsEnvironment,
|
||||
definition: GoogleMetricDefinition,
|
||||
token: string,
|
||||
startAt: string,
|
||||
endAt: string,
|
||||
fetchImpl: typeof fetch,
|
||||
now: () => number = () => Date.parse(endAt),
|
||||
wait: (ms: number) => Promise<void> = async (ms) =>
|
||||
await new Promise((resolve) => setTimeout(resolve, ms))
|
||||
): Promise<IncidentSignal | null> {
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
const signal = await readGoogleMetric(
|
||||
environment,
|
||||
definition,
|
||||
token,
|
||||
startAt,
|
||||
endAt,
|
||||
fetchImpl,
|
||||
now
|
||||
)
|
||||
if (signal !== null || definition.emptyIsZero) return signal
|
||||
if (attempt < 2) await wait(2_000)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function addSignal(
|
||||
signals: Record<string, IncidentSignal>,
|
||||
name: string,
|
||||
value: number | null,
|
||||
observedAt: string | null
|
||||
): void {
|
||||
if (value === null || observedAt === null) return
|
||||
signals[name] = { value, observedAt }
|
||||
}
|
||||
|
||||
function endpointSignals(
|
||||
snapshot: Awaited<ReturnType<typeof buildDashboardSnapshot>>,
|
||||
nowAt: string
|
||||
): IncidentSource {
|
||||
const signals: Record<string, IncidentSignal> = {}
|
||||
const addEndpoint = (
|
||||
prefix: string,
|
||||
endpoint: { health: boolean | null; ready: boolean | null; latencyMs: number | null },
|
||||
unavailableIsZero = false
|
||||
) => {
|
||||
addSignal(
|
||||
signals,
|
||||
`${prefix}.health`,
|
||||
endpoint.health === null ? (unavailableIsZero ? 0 : null) : Number(endpoint.health),
|
||||
nowAt
|
||||
)
|
||||
addSignal(
|
||||
signals,
|
||||
`${prefix}.ready`,
|
||||
endpoint.ready === null ? (unavailableIsZero ? 0 : null) : Number(endpoint.ready),
|
||||
nowAt
|
||||
)
|
||||
addSignal(signals, `${prefix}.latency_ms`, endpoint.latencyMs, nowAt)
|
||||
}
|
||||
addEndpoint('director', snapshot.resources.directorEndpoint)
|
||||
addEndpoint('auth', snapshot.resources.authEndpoint)
|
||||
for (const cell of snapshot.resources.cells) {
|
||||
addEndpoint(`cell.${cell.cellId}`, cell.endpoint, true)
|
||||
}
|
||||
return { observedAt: nowAt, signals }
|
||||
}
|
||||
|
||||
export function relayFiveMinuteDeltaSignal(
|
||||
metric: Pick<RelayMetricSnapshot, 'available' | 'points'>,
|
||||
endAt: string
|
||||
): IncidentSignal | null {
|
||||
if (!metric.available) return null
|
||||
const endMs = Date.parse(endAt)
|
||||
if (!Number.isFinite(endMs)) throw new Error('Relay telemetry end time is invalid')
|
||||
return {
|
||||
value: metric.points
|
||||
.filter((point) => {
|
||||
const pointMs = Date.parse(point.at)
|
||||
return pointMs >= endMs - 300_000 && pointMs <= endMs
|
||||
})
|
||||
.reduce((total, point) => total + point.value, 0),
|
||||
observedAt: endAt
|
||||
}
|
||||
}
|
||||
|
||||
function relaySignals(
|
||||
snapshot: Awaited<ReturnType<typeof buildDashboardSnapshot>>
|
||||
): IncidentSource {
|
||||
const metrics = snapshot.monitoring.metrics
|
||||
const signals: Record<string, IncidentSignal> = {}
|
||||
addSignal(
|
||||
signals,
|
||||
'relay.pool_waiting',
|
||||
metrics.db_waiters_max.latest,
|
||||
metrics.db_waiters_max.latestAt
|
||||
)
|
||||
addSignal(
|
||||
signals,
|
||||
'relay.pool_wait_ms',
|
||||
metrics.db_wait_ms_max.latest,
|
||||
metrics.db_wait_ms_max.latestAt
|
||||
)
|
||||
const retries = relayFiveMinuteDeltaSignal(
|
||||
metrics.postgres_retries,
|
||||
snapshot.monitoring.endAt
|
||||
)
|
||||
if (retries) signals['relay.postgres_retries'] = retries
|
||||
const retryExhausted = relayFiveMinuteDeltaSignal(
|
||||
metrics.postgres_retry_exhausted,
|
||||
snapshot.monitoring.endAt
|
||||
)
|
||||
if (retryExhausted) signals['relay.postgres_retry_exhausted'] = retryExhausted
|
||||
for (const cell of snapshot.resources.cells) {
|
||||
addSignal(
|
||||
signals,
|
||||
`cell.${cell.cellId}.connections`,
|
||||
metrics.total_connections.latestByCell[cell.cellId] ?? null,
|
||||
metrics.total_connections.latestAt
|
||||
)
|
||||
addSignal(
|
||||
signals,
|
||||
`cell.${cell.cellId}.queued_bytes`,
|
||||
metrics.queued_bytes.latestByCell[cell.cellId] ?? null,
|
||||
metrics.queued_bytes.latestAt
|
||||
)
|
||||
}
|
||||
const observedTimes = Object.values(signals).map((entry) => Date.parse(entry.observedAt))
|
||||
if (observedTimes.length === 0) throw new Error('Relay telemetry is unavailable')
|
||||
const observedAt = new Date(Math.max(...observedTimes)).toISOString()
|
||||
return { observedAt, signals }
|
||||
}
|
||||
|
||||
const ADMIN_RETRY_ATTEMPTS = 3
|
||||
const ADMIN_RETRY_INTERVAL_MS = 5_000
|
||||
const ADMIN_RETRYABLE_STATUSES = new Set([500, 502, 503, 504])
|
||||
const AdminErrorBodySchema = z.object({ error: z.string() })
|
||||
// The director's /v1/admin/cell-status maps any thrown operation error onto 404
|
||||
// with an {error: message} body, so a Cloud SQL pool connect timeout arrives
|
||||
// here as a 404. These are the pool-acquire and transient SQLSTATE messages the
|
||||
// relay app's own isRelayDatabaseTransientError treats as re-runnable.
|
||||
const ADMIN_TRANSIENT_ERROR_MESSAGES = [
|
||||
'timeout exceeded when trying to connect',
|
||||
'Connection terminated due to connection timeout',
|
||||
'Connection terminated unexpectedly',
|
||||
'database_temporarily_unavailable',
|
||||
'deadlock detected',
|
||||
'canceling statement due to',
|
||||
'too many connections',
|
||||
'the database system is shutting down',
|
||||
'the database system is starting up'
|
||||
]
|
||||
|
||||
const defaultWait = async (ms: number): Promise<void> =>
|
||||
await new Promise((resolveWait) => setTimeout(resolveWait, ms))
|
||||
|
||||
// A dropped socket or the 30 s AbortSignal firing; both leave the request with
|
||||
// no verdict, and every admin read here is a plain query.
|
||||
function transientFetchFailure(error: unknown): boolean {
|
||||
if (error instanceof TypeError) return true
|
||||
const name = (error as { name?: unknown } | null)?.name
|
||||
return name === 'AbortError' || name === 'TimeoutError'
|
||||
}
|
||||
|
||||
function adminErrorMessage(body: string): string | null {
|
||||
const parsed = AdminErrorBodySchema.safeParse(
|
||||
((): unknown => {
|
||||
try {
|
||||
return JSON.parse(body)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
})()
|
||||
)
|
||||
return parsed.success ? parsed.data.error : null
|
||||
}
|
||||
|
||||
function retryableAdminStatus(status: number, body: string): boolean {
|
||||
if (ADMIN_RETRYABLE_STATUSES.has(status)) return true
|
||||
// 401/403/400/413 and a 404 for 'director_only' are decisions, not weather.
|
||||
if (status !== 404) return false
|
||||
const message = adminErrorMessage(body)
|
||||
return (
|
||||
message !== null &&
|
||||
ADMIN_TRANSIENT_ERROR_MESSAGES.some((transient) => message.includes(transient))
|
||||
)
|
||||
}
|
||||
|
||||
// Step 5: cell-status refuses a reserve cell with exactly this 404 body; any other 404 stays a failure.
|
||||
class AdminReserveCellRefusal extends Error {
|
||||
constructor(status: number) {
|
||||
super(`Relay admin telemetry returned ${status}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function adminPost(
|
||||
fetchImpl: typeof fetch,
|
||||
origin: string,
|
||||
token: string,
|
||||
path: string,
|
||||
body: unknown,
|
||||
wait: (ms: number) => Promise<void> = defaultWait
|
||||
): Promise<unknown> {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
const lastAttempt = attempt >= ADMIN_RETRY_ATTEMPTS
|
||||
let response: Response
|
||||
try {
|
||||
response = await fetchImpl(`${origin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
} catch (error) {
|
||||
if (lastAttempt || !transientFetchFailure(error)) throw error
|
||||
console.warn(
|
||||
`relay admin telemetry retrying ${path} after network failure` +
|
||||
` (attempt ${attempt}/${ADMIN_RETRY_ATTEMPTS})`
|
||||
)
|
||||
await wait(ADMIN_RETRY_INTERVAL_MS)
|
||||
continue
|
||||
}
|
||||
if (response.ok) return await response.json()
|
||||
// Status only: the body carries a director error message, which must not
|
||||
// reach the log the way an identity would.
|
||||
const errorBody = await response.text().catch(() => '')
|
||||
if (response.status === 404 && adminErrorMessage(errorBody) === 'cell_in_reserve_mode') {
|
||||
throw new AdminReserveCellRefusal(response.status)
|
||||
}
|
||||
if (lastAttempt || !retryableAdminStatus(response.status, errorBody)) {
|
||||
throw new Error(`Relay admin telemetry returned ${response.status}`)
|
||||
}
|
||||
console.warn(
|
||||
`relay admin telemetry retrying ${path} after ${response.status}` +
|
||||
` (attempt ${attempt}/${ADMIN_RETRY_ATTEMPTS})`
|
||||
)
|
||||
await wait(ADMIN_RETRY_INTERVAL_MS)
|
||||
}
|
||||
}
|
||||
|
||||
export async function directorSignals(
|
||||
environmentId: RelayOpsEnvironmentId,
|
||||
expectedSelector: AdmissionSelector,
|
||||
gcloud: GcloudClient,
|
||||
nowMs: number,
|
||||
fetchImpl: typeof fetch,
|
||||
wait: (ms: number) => Promise<void> = defaultWait
|
||||
): Promise<{
|
||||
source: IncidentSource
|
||||
selector: AdmissionSelector
|
||||
cells: IncidentSample['cells']
|
||||
}> {
|
||||
const environment = relayOpsEnvironment(environmentId)
|
||||
if (!gcloud.identityToken) throw new Error('gcloud identity-token support is unavailable')
|
||||
const token = await gcloud.identityToken(`${environment.directorOrigin}/v1/admin/drain`)
|
||||
const configuredCellIds = new Set(environment.cells.map((cell) => cell.cellId))
|
||||
const rawSelector = SelectorStatusSchema.parse(
|
||||
await adminPost(
|
||||
fetchImpl,
|
||||
environment.directorOrigin,
|
||||
token,
|
||||
'/v1/admin/admission-selector/status',
|
||||
{ v: 1 },
|
||||
wait
|
||||
)
|
||||
).selector
|
||||
const selector = {
|
||||
generation: rawSelector.generation,
|
||||
membership: normalizeSelectorMembership(rawSelector.membership, configuredCellIds)
|
||||
}
|
||||
const statuses: Array<{
|
||||
cell: RelayOpsEnvironment['cells'][number]
|
||||
status: ({ reserve: false } & z.infer<typeof CellStatusSchema>['status']) | { reserve: true }
|
||||
}> = []
|
||||
for (const cell of environment.cells) {
|
||||
let response: unknown
|
||||
try {
|
||||
response = await adminPost(
|
||||
fetchImpl,
|
||||
environment.directorOrigin,
|
||||
token,
|
||||
'/v1/admin/cell-status',
|
||||
{ v: 1, cellId: cell.cellId },
|
||||
wait
|
||||
)
|
||||
} catch (error) {
|
||||
if (!(error instanceof AdminReserveCellRefusal)) throw error
|
||||
statuses.push({ cell, status: { reserve: true } })
|
||||
continue
|
||||
}
|
||||
statuses.push({
|
||||
cell,
|
||||
status: { reserve: false, ...CellStatusSchema.parse(response).status }
|
||||
})
|
||||
}
|
||||
const reserveCellIds = new Set(
|
||||
statuses.filter(({ status }) => status.reserve).map(({ cell }) => cell.cellId)
|
||||
)
|
||||
const migrationEntries: Array<{
|
||||
sourceCellId: string
|
||||
migration: z.infer<typeof MigrationStatusSchema>
|
||||
}> = []
|
||||
const migrationTargets = new Set(expectedSelector.membership.migrationOnly)
|
||||
for (const source of environment.cells) {
|
||||
for (const target of environment.cells) {
|
||||
if (source.cellId === target.cellId || !migrationTargets.has(target.cellId)) continue
|
||||
// The director refuses migration reads, and every migration, on a reserve cell.
|
||||
if (reserveCellIds.has(source.cellId)) continue
|
||||
migrationEntries.push({
|
||||
sourceCellId: source.cellId,
|
||||
migration: MigrationStatusSchema.parse(
|
||||
await adminPost(
|
||||
fetchImpl,
|
||||
environment.directorOrigin,
|
||||
token,
|
||||
'/v1/admin/evacuation-status',
|
||||
{
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
completeReady: false
|
||||
},
|
||||
wait
|
||||
)
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
const migrationBySource = new Map<string, { blocked: number; targetInactive: number }>()
|
||||
for (const { sourceCellId, migration } of migrationEntries) {
|
||||
const aggregate = migrationBySource.get(sourceCellId) ?? { blocked: 0, targetInactive: 0 }
|
||||
aggregate.blocked += migration.blocked + migration.blockedExpiredUnregistered
|
||||
aggregate.targetInactive += migration.registeredTargetInactive
|
||||
migrationBySource.set(sourceCellId, aggregate)
|
||||
}
|
||||
const nowAt = new Date(nowMs).toISOString()
|
||||
const signals: Record<string, IncidentSignal> = {}
|
||||
for (const { cell, status } of statuses) {
|
||||
const prefix = `cell.${cell.cellId}`
|
||||
if (status.reserve) {
|
||||
// cell-status carries the live cap; unset in Terraform keeps the evaluator's default, as there.
|
||||
if (cell.connectionHardCap !== null) {
|
||||
addSignal(signals, `${prefix}.connection_hard_cap`, cell.connectionHardCap, nowAt)
|
||||
}
|
||||
// Legacy admission needs relay_cells.enabled, which only cell-status reads: left missing.
|
||||
if (selector.generation !== 0) {
|
||||
addSignal(
|
||||
signals,
|
||||
`${prefix}.admission_state`,
|
||||
['existing-only', 'migration-only', 'general'].indexOf(
|
||||
selectorCellState(selector, cell.cellId)
|
||||
),
|
||||
nowAt
|
||||
)
|
||||
}
|
||||
continue
|
||||
}
|
||||
const admissionState = effectiveAdmissionState(
|
||||
selector,
|
||||
status.enabled,
|
||||
cell.cellId
|
||||
)
|
||||
addSignal(
|
||||
signals,
|
||||
`${prefix}.admission_state`,
|
||||
['existing-only', 'migration-only', 'general'].indexOf(admissionState),
|
||||
nowAt
|
||||
)
|
||||
addSignal(
|
||||
signals,
|
||||
`${prefix}.connection_hard_cap`,
|
||||
status.connectionCapacity?.hardCap ??
|
||||
INCIDENT_MONITOR_THRESHOLDS.cellConnections,
|
||||
nowAt
|
||||
)
|
||||
if (status.runtime) {
|
||||
addSignal(
|
||||
signals,
|
||||
`${prefix}.heartbeat_fresh`,
|
||||
Number(status.runtime.heartbeatFresh),
|
||||
nowAt
|
||||
)
|
||||
addSignal(
|
||||
signals,
|
||||
`${prefix}.heartbeat_age_ms`,
|
||||
Math.max(0, nowMs - status.runtime.lastHeartbeatAt),
|
||||
nowAt
|
||||
)
|
||||
}
|
||||
const migration = migrationBySource.get(cell.cellId) ?? { blocked: 0, targetInactive: 0 }
|
||||
addSignal(signals, `${prefix}.migration_blocked`, migration.blocked, nowAt)
|
||||
addSignal(
|
||||
signals,
|
||||
`${prefix}.migration_target_inactive`,
|
||||
migration.targetInactive,
|
||||
nowAt
|
||||
)
|
||||
}
|
||||
return {
|
||||
source: { observedAt: nowAt, signals },
|
||||
selector,
|
||||
cells: statuses.map(({ cell, status }) => ({
|
||||
cellId: cell.cellId,
|
||||
region: cell.region,
|
||||
runtimeKnown: true,
|
||||
powered: true,
|
||||
expectedAdmissionState: selectorCellState(expectedSelector, cell.cellId),
|
||||
...(status.reserve ? { reserve: true } : {})
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
export type IncidentSampleCollectorOptions = {
|
||||
environment: RelayOpsEnvironmentId
|
||||
expectedSelector: AdmissionSelector
|
||||
fetchImpl?: typeof fetch
|
||||
now?: () => number
|
||||
wait?: (ms: number) => Promise<void>
|
||||
}
|
||||
|
||||
export function createIncidentSampleCollector(
|
||||
gcloud: GcloudClient,
|
||||
options: IncidentSampleCollectorOptions
|
||||
): () => Promise<IncidentSample> {
|
||||
const fetchImpl = options.fetchImpl ?? fetch
|
||||
const now = options.now ?? Date.now
|
||||
return async () => {
|
||||
const nowMs = now()
|
||||
const nowAt = new Date(nowMs).toISOString()
|
||||
const startAt = new Date(nowMs - 5 * 60_000).toISOString()
|
||||
const environment = relayOpsEnvironment(options.environment)
|
||||
const accessToken = gcloud.accessToken()
|
||||
const cloudMetricEntries = accessToken.then(async (token) => await Promise.all(
|
||||
GOOGLE_METRICS.map(async (definition) => [
|
||||
definition.signal,
|
||||
await readGoogleMetricWithEmptyRetry(
|
||||
environment,
|
||||
definition,
|
||||
token,
|
||||
startAt,
|
||||
nowAt,
|
||||
fetchImpl,
|
||||
now
|
||||
)
|
||||
] as const)
|
||||
))
|
||||
const [snapshot, director, metricEntries] = await Promise.all([
|
||||
buildDashboardSnapshot(options.environment, gcloud, {
|
||||
windowMinutes: 5,
|
||||
now: new Date(nowMs),
|
||||
fetchImpl
|
||||
}),
|
||||
directorSignals(
|
||||
options.environment,
|
||||
options.expectedSelector,
|
||||
gcloud,
|
||||
nowMs,
|
||||
fetchImpl,
|
||||
options.wait ?? defaultWait
|
||||
),
|
||||
cloudMetricEntries
|
||||
])
|
||||
const cloudSignals = Object.fromEntries(
|
||||
metricEntries.filter((entry): entry is [string, IncidentSignal] => entry[1] !== null)
|
||||
)
|
||||
const relay = relaySignals(snapshot)
|
||||
const poweredByCell = new Map(
|
||||
snapshot.resources.cells.map((cell) => [
|
||||
cell.cellId,
|
||||
{
|
||||
runtimeKnown: cell.targetSize !== null,
|
||||
powered: (cell.targetSize ?? 0) > 0
|
||||
}
|
||||
])
|
||||
)
|
||||
return {
|
||||
collectedAt: nowAt,
|
||||
selector: director.selector,
|
||||
expectedSelector: options.expectedSelector,
|
||||
sources: {
|
||||
'active-probe': endpointSignals(snapshot, nowAt),
|
||||
'cloud-monitoring': { observedAt: nowAt, signals: cloudSignals },
|
||||
'relay-logs': relay,
|
||||
'director-admin': director.source
|
||||
},
|
||||
cells: director.cells.map((cell) => ({
|
||||
...cell,
|
||||
runtimeKnown: poweredByCell.get(cell.cellId)?.runtimeKnown ?? false,
|
||||
powered: poweredByCell.get(cell.cellId)?.powered ?? false
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,77 +0,0 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
export const AdmissionStateSchema = z.enum([
|
||||
'existing-only',
|
||||
'migration-only',
|
||||
'general'
|
||||
])
|
||||
|
||||
export type AdmissionState = z.infer<typeof AdmissionStateSchema>
|
||||
|
||||
export const SelectorMembershipSchema = z.object({
|
||||
existingOnly: z.array(z.string()),
|
||||
migrationOnly: z.array(z.string()),
|
||||
general: z.array(z.string())
|
||||
})
|
||||
|
||||
export type SelectorMembership = z.infer<typeof SelectorMembershipSchema>
|
||||
|
||||
export const AdmissionSelectorSchema = z.object({
|
||||
generation: z.number().int().nonnegative(),
|
||||
membership: SelectorMembershipSchema
|
||||
})
|
||||
|
||||
export type AdmissionSelector = z.infer<typeof AdmissionSelectorSchema>
|
||||
|
||||
export function normalizeSelectorMembership(
|
||||
membership: SelectorMembership,
|
||||
configuredCellIds: ReadonlySet<string>
|
||||
): SelectorMembership {
|
||||
const normalized = {
|
||||
existingOnly: [...membership.existingOnly].sort(),
|
||||
migrationOnly: [...membership.migrationOnly].sort(),
|
||||
general: [...membership.general].sort()
|
||||
}
|
||||
const all = [
|
||||
...normalized.existingOnly,
|
||||
...normalized.migrationOnly,
|
||||
...normalized.general
|
||||
]
|
||||
if (
|
||||
all.length !== configuredCellIds.size ||
|
||||
new Set(all).size !== all.length ||
|
||||
all.some((cellId) => !configuredCellIds.has(cellId))
|
||||
) {
|
||||
throw new Error('selector membership must contain every configured cell exactly once')
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
export function selectorCellState(
|
||||
selector: AdmissionSelector,
|
||||
cellId: string
|
||||
): AdmissionState {
|
||||
if (selector.membership.existingOnly.includes(cellId)) return 'existing-only'
|
||||
if (selector.membership.migrationOnly.includes(cellId)) return 'migration-only'
|
||||
if (selector.membership.general.includes(cellId)) return 'general'
|
||||
throw new Error(`selector does not contain ${cellId}`)
|
||||
}
|
||||
|
||||
export function effectiveAdmissionState(
|
||||
selector: AdmissionSelector,
|
||||
legacyEnabled: boolean,
|
||||
cellId: string
|
||||
): AdmissionState {
|
||||
if (selector.generation === 0) return legacyEnabled ? 'general' : 'existing-only'
|
||||
return selectorCellState(selector, cellId)
|
||||
}
|
||||
|
||||
export function exactAdmissionSelector(
|
||||
actual: AdmissionSelector,
|
||||
expected: AdmissionSelector
|
||||
): boolean {
|
||||
return (
|
||||
actual.generation === expected.generation &&
|
||||
JSON.stringify(actual.membership) === JSON.stringify(expected.membership)
|
||||
)
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
import { randomBytes, timingSafeEqual } from 'node:crypto'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { resolve } from 'node:path'
|
||||
import { serve } from '@hono/node-server'
|
||||
import { Hono } from 'hono'
|
||||
import { z } from 'zod'
|
||||
import { DashboardSnapshotCache } from './dashboard-snapshot.js'
|
||||
import { createGcloudClient } from './gcloud-client.js'
|
||||
import {
|
||||
dispatchStagingPowerWorkflow,
|
||||
parseStagingPowerRequest
|
||||
} from './staging-workflow.js'
|
||||
|
||||
const QuerySchema = z.object({
|
||||
environment: z.enum(['production', 'staging']).default('production'),
|
||||
window: z.coerce.number().int().min(30).max(1440).default(360)
|
||||
})
|
||||
const port = z.coerce.number().int().min(1024).max(65_535).parse(process.env.PORT ?? 2455)
|
||||
const controlsEnabled = process.env.RELAY_OPS_ENABLE_STAGING_CONTROLS === '1'
|
||||
const csrfToken = randomBytes(32).toString('base64url')
|
||||
const publicDirectory = resolve(import.meta.dirname, '../public')
|
||||
const cache = new DashboardSnapshotCache(createGcloudClient())
|
||||
const app = new Hono()
|
||||
|
||||
function safeEqual(left: string, right: string): boolean {
|
||||
const leftBuffer = Buffer.from(left)
|
||||
const rightBuffer = Buffer.from(right)
|
||||
return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer)
|
||||
}
|
||||
|
||||
app.use('*', async (context, next) => {
|
||||
await next()
|
||||
context.header('Cache-Control', 'no-store')
|
||||
context.header('Content-Security-Policy', [
|
||||
"default-src 'self'",
|
||||
"script-src 'self'",
|
||||
"style-src 'self'",
|
||||
"font-src 'self'",
|
||||
"connect-src 'self'",
|
||||
"img-src 'self' data:",
|
||||
"object-src 'none'",
|
||||
"base-uri 'none'",
|
||||
"frame-ancestors 'none'",
|
||||
"form-action 'self'"
|
||||
].join('; '))
|
||||
context.header('Referrer-Policy', 'no-referrer')
|
||||
context.header('X-Content-Type-Options', 'nosniff')
|
||||
context.header('X-Frame-Options', 'DENY')
|
||||
})
|
||||
|
||||
app.get('/health', (context) => context.json({ status: 'ok' }))
|
||||
app.get('/api/config', (context) => context.json({
|
||||
stagingControlsEnabled: controlsEnabled,
|
||||
csrfToken: controlsEnabled ? csrfToken : null
|
||||
}))
|
||||
app.get('/api/snapshot', async (context) => {
|
||||
const query = QuerySchema.safeParse(context.req.query())
|
||||
if (!query.success) return context.json({ error: 'Invalid dashboard query' }, 400)
|
||||
try {
|
||||
return context.json(await cache.read(query.data.environment, query.data.window))
|
||||
} catch {
|
||||
return context.json({
|
||||
error: 'Relay operations data is unavailable. Check local gcloud and gh authentication.'
|
||||
}, 503)
|
||||
}
|
||||
})
|
||||
app.post('/api/staging/power', async (context) => {
|
||||
if (!controlsEnabled) return context.json({ error: 'Staging controls are disabled' }, 403)
|
||||
const origin = context.req.header('origin')
|
||||
const expectedOrigin = `http://127.0.0.1:${port}`
|
||||
if (origin !== expectedOrigin) return context.json({ error: 'Origin rejected' }, 403)
|
||||
if (!safeEqual(context.req.header('x-csrf-token') ?? '', csrfToken)) {
|
||||
return context.json({ error: 'Request token rejected' }, 403)
|
||||
}
|
||||
try {
|
||||
const request = parseStagingPowerRequest(await context.req.json())
|
||||
await dispatchStagingPowerWorkflow(request)
|
||||
return context.json({ accepted: true })
|
||||
} catch {
|
||||
return context.json({ error: 'Invalid or failed staging workflow dispatch' }, 400)
|
||||
}
|
||||
})
|
||||
|
||||
const staticTypes: Record<string, string> = {
|
||||
'/app.js': 'text/javascript; charset=utf-8',
|
||||
'/styles.css': 'text/css; charset=utf-8'
|
||||
}
|
||||
for (const [route, contentType] of Object.entries(staticTypes)) {
|
||||
app.get(route, async (context) => {
|
||||
const file = route.slice(1)
|
||||
try {
|
||||
const content = await readFile(resolve(publicDirectory, file))
|
||||
return context.body(content, 200, { 'Content-Type': contentType })
|
||||
} catch {
|
||||
return context.notFound()
|
||||
}
|
||||
})
|
||||
}
|
||||
app.get('/', async (context) => {
|
||||
const html = await readFile(resolve(publicDirectory, 'index.html'), 'utf8')
|
||||
return context.html(html)
|
||||
})
|
||||
|
||||
serve({ fetch: app.fetch, hostname: '127.0.0.1', port }, () => {
|
||||
// Loopback is intentional; operators may add authenticated Tailscale Serve separately.
|
||||
console.log(`Orca Relay Operations: http://127.0.0.1:${port}`)
|
||||
console.log(`Staging controls: ${controlsEnabled ? 'enabled through GitHub workflow' : 'read-only'}`)
|
||||
})
|
||||
@@ -1,85 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { RELAY_METRICS, readMonitoringSnapshot } from './monitoring-snapshot.js'
|
||||
import type { GcloudClient } from './gcloud-client.js'
|
||||
import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js'
|
||||
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => 'a'.repeat(40)
|
||||
}
|
||||
|
||||
function distribution(at: string, mean: number | undefined, count = 1) {
|
||||
return {
|
||||
interval: { endTime: at },
|
||||
value: { distributionValue: { count, ...(mean === undefined ? {} : { mean }) } }
|
||||
}
|
||||
}
|
||||
|
||||
describe('readMonitoringSnapshot', () => {
|
||||
it('aggregates gauge series per minute and distribution deltas by count', async () => {
|
||||
const fetchImpl: typeof fetch = async (input) => {
|
||||
const url = new URL(String(input))
|
||||
if (url.pathname.endsWith('/alertPolicies')) {
|
||||
return Response.json({ alertPolicies: [] })
|
||||
}
|
||||
const filter = url.searchParams.get('filter') ?? ''
|
||||
if (filter.includes('orca_relay_controls')) {
|
||||
return Response.json({ timeSeries: [
|
||||
{
|
||||
metric: { labels: { cell_id: 'production-gce-c1' } },
|
||||
resource: { type: 'gce_instance', labels: { instance_id: 'one' } },
|
||||
points: [
|
||||
distribution('2026-07-15T12:00:10Z', 1),
|
||||
distribution('2026-07-15T12:00:50Z', 2)
|
||||
]
|
||||
},
|
||||
{
|
||||
metric: { labels: { cell_id: 'production-gce-c1' } },
|
||||
resource: { type: 'gce_instance', labels: { instance_id: 'two' } },
|
||||
points: [distribution('2026-07-15T12:00:20Z', 3)]
|
||||
},
|
||||
{
|
||||
metric: { labels: { cell_id: 'production-gce-c1' } },
|
||||
resource: { type: 'gce_instance', labels: { instance_id: 'stale' } },
|
||||
points: [distribution('2026-07-15T11:59:20Z', 100)]
|
||||
}
|
||||
] })
|
||||
}
|
||||
if (filter.includes('orca_relay_forwarded_bytes')) {
|
||||
return Response.json({ timeSeries: [{
|
||||
metric: { labels: { cell_id: 'production-gce-c1' } },
|
||||
resource: { type: 'gce_instance', labels: { instance_id: 'one' } },
|
||||
points: [distribution('2026-07-15T12:00:20Z', 10, 4)]
|
||||
}] })
|
||||
}
|
||||
return Response.json({ timeSeries: [] })
|
||||
}
|
||||
|
||||
const result = await readMonitoringSnapshot(RELAY_OPS_ENVIRONMENTS.production, gcloud, {
|
||||
now: new Date('2026-07-15T12:01:00Z'),
|
||||
windowMinutes: 30,
|
||||
fetchImpl
|
||||
})
|
||||
|
||||
expect(result.warnings).toEqual([])
|
||||
expect(result.metrics.controls.points).toEqual([
|
||||
{ at: '2026-07-15T11:59:00.000Z', value: 100 },
|
||||
{ at: '2026-07-15T12:00:00.000Z', value: 5 }
|
||||
])
|
||||
expect(result.metrics.controls.latestByCell).toEqual({ 'production-gce-c1': 5 })
|
||||
expect(result.metrics.forwarded_bytes.latest).toBe(40)
|
||||
expect(result.metrics.postgres_retries.available).toBe(true)
|
||||
expect(Object.keys(result.metrics)).toHaveLength(RELAY_METRICS.length)
|
||||
})
|
||||
|
||||
it('degrades safely when credentials are unavailable', async () => {
|
||||
const unavailable: GcloudClient = {
|
||||
accessToken: async () => { throw new Error('sensitive context') }
|
||||
}
|
||||
const result = await readMonitoringSnapshot(RELAY_OPS_ENVIRONMENTS.production, unavailable)
|
||||
expect(result.warnings).toEqual([
|
||||
'Cloud Monitoring credentials are unavailable. Run gcloud auth login.'
|
||||
])
|
||||
expect(result.metrics.postgres_retries.available).toBe(false)
|
||||
expect(JSON.stringify(result)).not.toContain('sensitive context')
|
||||
})
|
||||
})
|
||||
@@ -1,379 +0,0 @@
|
||||
import { z } from 'zod'
|
||||
import type { RelayOpsEnvironment } from './environment-config.js'
|
||||
import type { GcloudClient } from './gcloud-client.js'
|
||||
|
||||
type MetricMode = 'gauge-sum' | 'delta-sum' | 'maximum'
|
||||
|
||||
export type RelayMetricName =
|
||||
| 'total_connections'
|
||||
| 'controls'
|
||||
| 'splices'
|
||||
| 'pending_splices'
|
||||
| 'queued_bytes'
|
||||
| 'http_latency_ms'
|
||||
| 'sql_latency_ms'
|
||||
| 'heap_used_bytes'
|
||||
| 'event_loop_ms_p99'
|
||||
| 'forwarded_bytes'
|
||||
| 'auth_successes'
|
||||
| 'auth_failures'
|
||||
| 'reconnects'
|
||||
| 'sql_queries'
|
||||
| 'sql_failures'
|
||||
| 'assignment_5xx'
|
||||
| 'postgres_retries'
|
||||
| 'postgres_retry_exhausted'
|
||||
| 'db_pool_total'
|
||||
| 'db_pool_idle'
|
||||
| 'db_pool_waiting'
|
||||
| 'db_waiters_max'
|
||||
| 'db_oldest_wait_ms'
|
||||
| 'db_wait_ms_max'
|
||||
|
||||
type MetricDefinition = {
|
||||
name: RelayMetricName
|
||||
label: string
|
||||
unit: 'count' | 'bytes' | 'milliseconds'
|
||||
mode: MetricMode
|
||||
}
|
||||
|
||||
export const RELAY_METRICS: MetricDefinition[] = [
|
||||
{ name: 'total_connections', label: 'Connections', unit: 'count', mode: 'gauge-sum' },
|
||||
{ name: 'controls', label: 'Desktop controls', unit: 'count', mode: 'gauge-sum' },
|
||||
{ name: 'splices', label: 'Phone splices', unit: 'count', mode: 'gauge-sum' },
|
||||
{ name: 'pending_splices', label: 'Pending splices', unit: 'count', mode: 'gauge-sum' },
|
||||
{ name: 'queued_bytes', label: 'Queued bytes', unit: 'bytes', mode: 'maximum' },
|
||||
{ name: 'http_latency_ms', label: 'HTTP latency', unit: 'milliseconds', mode: 'maximum' },
|
||||
{ name: 'sql_latency_ms', label: 'SQL latency', unit: 'milliseconds', mode: 'maximum' },
|
||||
{ name: 'heap_used_bytes', label: 'Heap used', unit: 'bytes', mode: 'maximum' },
|
||||
{
|
||||
name: 'event_loop_ms_p99',
|
||||
label: 'Event-loop p99',
|
||||
unit: 'milliseconds',
|
||||
mode: 'maximum'
|
||||
},
|
||||
{ name: 'forwarded_bytes', label: 'Forwarded bytes', unit: 'bytes', mode: 'delta-sum' },
|
||||
{ name: 'auth_successes', label: 'Auth successes', unit: 'count', mode: 'delta-sum' },
|
||||
{ name: 'auth_failures', label: 'Auth failures', unit: 'count', mode: 'delta-sum' },
|
||||
{ name: 'reconnects', label: 'Reconnects', unit: 'count', mode: 'delta-sum' },
|
||||
{ name: 'sql_queries', label: 'SQL queries', unit: 'count', mode: 'delta-sum' },
|
||||
{ name: 'sql_failures', label: 'SQL failures', unit: 'count', mode: 'delta-sum' },
|
||||
{ name: 'assignment_5xx', label: 'Assignment 5xx', unit: 'count', mode: 'delta-sum' },
|
||||
{
|
||||
name: 'postgres_retries',
|
||||
label: 'PostgreSQL retries',
|
||||
unit: 'count',
|
||||
mode: 'delta-sum'
|
||||
},
|
||||
{
|
||||
name: 'postgres_retry_exhausted',
|
||||
label: 'PostgreSQL retry exhausted',
|
||||
unit: 'count',
|
||||
mode: 'delta-sum'
|
||||
},
|
||||
{ name: 'db_pool_total', label: 'Database pool total', unit: 'count', mode: 'gauge-sum' },
|
||||
{ name: 'db_pool_idle', label: 'Database pool idle', unit: 'count', mode: 'gauge-sum' },
|
||||
{
|
||||
name: 'db_pool_waiting',
|
||||
label: 'Database pool waiting',
|
||||
unit: 'count',
|
||||
mode: 'gauge-sum'
|
||||
},
|
||||
{
|
||||
name: 'db_waiters_max',
|
||||
label: 'Database waiters max',
|
||||
unit: 'count',
|
||||
mode: 'maximum'
|
||||
},
|
||||
{
|
||||
name: 'db_oldest_wait_ms',
|
||||
label: 'Database oldest wait',
|
||||
unit: 'milliseconds',
|
||||
mode: 'maximum'
|
||||
},
|
||||
{
|
||||
name: 'db_wait_ms_max',
|
||||
label: 'Database wait max',
|
||||
unit: 'milliseconds',
|
||||
mode: 'maximum'
|
||||
}
|
||||
]
|
||||
|
||||
const NumericSchema = z.union([z.number(), z.string()]).transform((value) => Number(value))
|
||||
const DistributionSchema = z.object({
|
||||
count: NumericSchema.default(0),
|
||||
mean: NumericSchema.optional()
|
||||
})
|
||||
const PointSchema = z.object({
|
||||
interval: z.object({ endTime: z.string() }),
|
||||
value: z.object({
|
||||
doubleValue: NumericSchema.optional(),
|
||||
int64Value: NumericSchema.optional(),
|
||||
distributionValue: DistributionSchema.optional()
|
||||
})
|
||||
})
|
||||
const TimeSeriesSchema = z.object({
|
||||
metric: z.object({ labels: z.record(z.string(), z.string()).default({}) }),
|
||||
resource: z.object({ type: z.string(), labels: z.record(z.string(), z.string()).default({}) }),
|
||||
points: z.array(PointSchema).default([])
|
||||
})
|
||||
const TimeSeriesResponseSchema = z.object({
|
||||
timeSeries: z.array(TimeSeriesSchema).default([])
|
||||
})
|
||||
|
||||
export type MetricPoint = { at: string; value: number }
|
||||
|
||||
export type RelayMetricSnapshot = MetricDefinition & {
|
||||
available: boolean
|
||||
points: MetricPoint[]
|
||||
latest: number | null
|
||||
latestAt: string | null
|
||||
latestByCell: Record<string, number>
|
||||
}
|
||||
|
||||
export type AlertPolicySnapshot = {
|
||||
id: string
|
||||
displayName: string
|
||||
enabled: boolean
|
||||
documentation: string | null
|
||||
}
|
||||
|
||||
export type MonitoringSnapshot = {
|
||||
startAt: string
|
||||
endAt: string
|
||||
resolutionSeconds: number
|
||||
metrics: Record<RelayMetricName, RelayMetricSnapshot>
|
||||
alertPolicies: AlertPolicySnapshot[]
|
||||
warnings: string[]
|
||||
}
|
||||
|
||||
type ParsedPoint = {
|
||||
atMs: number
|
||||
bucketMs: number
|
||||
value: number
|
||||
sampleTotal: number
|
||||
seriesKey: string
|
||||
cellId: string
|
||||
}
|
||||
|
||||
function pointValue(point: z.infer<typeof PointSchema>): { value: number; sampleTotal: number } {
|
||||
if (point.value.distributionValue) {
|
||||
const count = point.value.distributionValue.count
|
||||
const value = point.value.distributionValue.mean ?? 0
|
||||
return { value, sampleTotal: value * count }
|
||||
}
|
||||
const value = point.value.doubleValue ?? point.value.int64Value ?? 0
|
||||
return { value, sampleTotal: value }
|
||||
}
|
||||
|
||||
function parsePoints(series: z.infer<typeof TimeSeriesSchema>[]): ParsedPoint[] {
|
||||
return series.flatMap((entry) => {
|
||||
const cellId = entry.metric.labels.cell_id ?? 'unknown'
|
||||
const resourceId =
|
||||
entry.resource.labels.instance_id ?? entry.resource.labels.revision_name ?? entry.resource.type
|
||||
const seriesKey = `${cellId}:${resourceId}`
|
||||
return entry.points.flatMap((point) => {
|
||||
const atMs = Date.parse(point.interval.endTime)
|
||||
if (!Number.isFinite(atMs)) return []
|
||||
const values = pointValue(point)
|
||||
return [{
|
||||
atMs,
|
||||
bucketMs: Math.floor(atMs / 60_000) * 60_000,
|
||||
value: values.value,
|
||||
sampleTotal: values.sampleTotal,
|
||||
seriesKey,
|
||||
cellId
|
||||
}]
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function aggregatePoints(points: ParsedPoint[], mode: MetricMode): MetricPoint[] {
|
||||
if (mode === 'gauge-sum') {
|
||||
const buckets = new Map<number, Map<string, ParsedPoint>>()
|
||||
for (const point of points) {
|
||||
const bySeries = buckets.get(point.bucketMs) ?? new Map<string, ParsedPoint>()
|
||||
const previous = bySeries.get(point.seriesKey)
|
||||
if (!previous || previous.atMs < point.atMs) bySeries.set(point.seriesKey, point)
|
||||
buckets.set(point.bucketMs, bySeries)
|
||||
}
|
||||
return [...buckets.entries()]
|
||||
.sort(([left], [right]) => left - right)
|
||||
.map(([at, bySeries]) => ({
|
||||
at: new Date(at).toISOString(),
|
||||
value: [...bySeries.values()].reduce((total, point) => total + point.value, 0)
|
||||
}))
|
||||
}
|
||||
const buckets = new Map<number, number>()
|
||||
for (const point of points) {
|
||||
const value = mode === 'delta-sum' ? point.sampleTotal : point.value
|
||||
const previous = buckets.get(point.bucketMs)
|
||||
buckets.set(
|
||||
point.bucketMs,
|
||||
mode === 'maximum' ? Math.max(previous ?? 0, value) : (previous ?? 0) + value
|
||||
)
|
||||
}
|
||||
return [...buckets.entries()]
|
||||
.sort(([left], [right]) => left - right)
|
||||
.map(([at, value]) => ({ at: new Date(at).toISOString(), value }))
|
||||
}
|
||||
|
||||
function latestByCell(points: ParsedPoint[], mode: MetricMode): Record<string, number> {
|
||||
const newestBucketByCell = new Map<string, number>()
|
||||
for (const point of points) {
|
||||
newestBucketByCell.set(
|
||||
point.cellId,
|
||||
Math.max(newestBucketByCell.get(point.cellId) ?? 0, point.bucketMs)
|
||||
)
|
||||
}
|
||||
const newestBySeries = new Map<string, ParsedPoint>()
|
||||
for (const point of points) {
|
||||
if (point.bucketMs !== newestBucketByCell.get(point.cellId)) continue
|
||||
const previous = newestBySeries.get(point.seriesKey)
|
||||
if (!previous || previous.atMs < point.atMs) newestBySeries.set(point.seriesKey, point)
|
||||
}
|
||||
const totals = new Map<string, number>()
|
||||
for (const point of newestBySeries.values()) {
|
||||
const value = mode === 'delta-sum' ? point.sampleTotal : point.value
|
||||
const previous = totals.get(point.cellId)
|
||||
totals.set(
|
||||
point.cellId,
|
||||
mode === 'maximum' ? Math.max(previous ?? 0, value) : (previous ?? 0) + value
|
||||
)
|
||||
}
|
||||
return Object.fromEntries(totals)
|
||||
}
|
||||
|
||||
async function monitoringRequest(
|
||||
fetchImpl: typeof fetch,
|
||||
token: string,
|
||||
url: URL
|
||||
): Promise<unknown> {
|
||||
const response = await fetchImpl(url, {
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
if (!response.ok) throw new Error(`Cloud Monitoring returned ${response.status}`)
|
||||
return await response.json()
|
||||
}
|
||||
|
||||
async function readMetric(
|
||||
environment: RelayOpsEnvironment,
|
||||
definition: MetricDefinition,
|
||||
token: string,
|
||||
startAt: string,
|
||||
endAt: string,
|
||||
fetchImpl: typeof fetch
|
||||
): Promise<RelayMetricSnapshot> {
|
||||
const url = new URL(
|
||||
`https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries`
|
||||
)
|
||||
url.searchParams.set(
|
||||
'filter',
|
||||
`metric.type="logging.googleapis.com/user/orca_relay_${definition.name}"`
|
||||
)
|
||||
url.searchParams.set('interval.startTime', startAt)
|
||||
url.searchParams.set('interval.endTime', endAt)
|
||||
url.searchParams.set('view', 'FULL')
|
||||
url.searchParams.set('pageSize', '1000')
|
||||
const body = TimeSeriesResponseSchema.parse(
|
||||
await monitoringRequest(fetchImpl, token, url)
|
||||
)
|
||||
const parsed = parsePoints(body.timeSeries)
|
||||
const points = aggregatePoints(parsed, definition.mode)
|
||||
const latest = points.at(-1) ?? null
|
||||
return {
|
||||
...definition,
|
||||
available: true,
|
||||
points,
|
||||
latest: latest?.value ?? null,
|
||||
latestAt: latest?.at ?? null,
|
||||
latestByCell: latestByCell(parsed, definition.mode)
|
||||
}
|
||||
}
|
||||
|
||||
async function readAlertPolicies(
|
||||
environment: RelayOpsEnvironment,
|
||||
token: string,
|
||||
fetchImpl: typeof fetch
|
||||
): Promise<AlertPolicySnapshot[]> {
|
||||
const url = new URL(
|
||||
`https://monitoring.googleapis.com/v3/projects/${environment.project}/alertPolicies`
|
||||
)
|
||||
url.searchParams.set('pageSize', '100')
|
||||
const body = (await monitoringRequest(fetchImpl, token, url)) as {
|
||||
alertPolicies?: Array<{
|
||||
name?: string
|
||||
displayName?: string
|
||||
enabled?: boolean
|
||||
documentation?: { content?: string }
|
||||
}>
|
||||
}
|
||||
return (body.alertPolicies ?? [])
|
||||
.filter((policy) => policy.displayName?.startsWith('Orca Relay:'))
|
||||
.map((policy) => ({
|
||||
id: policy.name ?? '',
|
||||
displayName: policy.displayName ?? 'Orca Relay alert',
|
||||
enabled: policy.enabled === true,
|
||||
documentation: policy.documentation?.content ?? null
|
||||
}))
|
||||
.sort((left, right) => left.displayName.localeCompare(right.displayName))
|
||||
}
|
||||
|
||||
function emptyMetric(definition: MetricDefinition): RelayMetricSnapshot {
|
||||
return {
|
||||
...definition,
|
||||
available: false,
|
||||
points: [],
|
||||
latest: null,
|
||||
latestAt: null,
|
||||
latestByCell: {}
|
||||
}
|
||||
}
|
||||
|
||||
export async function readMonitoringSnapshot(
|
||||
environment: RelayOpsEnvironment,
|
||||
gcloud: GcloudClient,
|
||||
options: { now?: Date; windowMinutes?: number; fetchImpl?: typeof fetch } = {}
|
||||
): Promise<MonitoringSnapshot> {
|
||||
const now = options.now ?? new Date()
|
||||
const windowMinutes = Math.min(24 * 60, Math.max(30, options.windowMinutes ?? 360))
|
||||
const endAt = now.toISOString()
|
||||
const startAt = new Date(now.getTime() - windowMinutes * 60_000).toISOString()
|
||||
const fetchImpl = options.fetchImpl ?? fetch
|
||||
const warnings: string[] = []
|
||||
let token: string
|
||||
try {
|
||||
token = await gcloud.accessToken()
|
||||
} catch {
|
||||
return {
|
||||
startAt,
|
||||
endAt,
|
||||
resolutionSeconds: 60,
|
||||
metrics: Object.fromEntries(
|
||||
RELAY_METRICS.map((definition) => [definition.name, emptyMetric(definition)])
|
||||
) as Record<RelayMetricName, RelayMetricSnapshot>,
|
||||
alertPolicies: [],
|
||||
warnings: ['Cloud Monitoring credentials are unavailable. Run gcloud auth login.']
|
||||
}
|
||||
}
|
||||
const settled = await Promise.allSettled(
|
||||
RELAY_METRICS.map((definition) =>
|
||||
readMetric(environment, definition, token, startAt, endAt, fetchImpl)
|
||||
)
|
||||
)
|
||||
const metrics = {} as Record<RelayMetricName, RelayMetricSnapshot>
|
||||
settled.forEach((result, index) => {
|
||||
const definition = RELAY_METRICS[index]!
|
||||
if (result.status === 'fulfilled') metrics[definition.name] = result.value
|
||||
else {
|
||||
metrics[definition.name] = emptyMetric(definition)
|
||||
warnings.push(`${definition.label} metric is unavailable.`)
|
||||
}
|
||||
})
|
||||
const alertPolicies = await readAlertPolicies(environment, token, fetchImpl).catch(() => {
|
||||
warnings.push('Cloud Monitoring alert policies are unavailable.')
|
||||
return []
|
||||
})
|
||||
return { startAt, endAt, resolutionSeconds: 60, metrics, alertPolicies, warnings }
|
||||
}
|
||||
@@ -1,210 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { relayOpsEnvironment } from './environment-config.js'
|
||||
import {
|
||||
CELL_PROCESS_EXIT_METRIC,
|
||||
createPreDrainHardRuleReader,
|
||||
INSTANCE_CELL_LOOKBACK_MS,
|
||||
type PreDrainExitScope
|
||||
} from './pre-drain-hard-rule-sources.js'
|
||||
|
||||
const environment = relayOpsEnvironment('production')
|
||||
const now = Date.parse('2026-10-01T12:00:00.000Z')
|
||||
const scope: PreDrainExitScope = {
|
||||
targetCellId: 'production-gce-c25',
|
||||
placementCellIds: new Set(['production-gce-c25', 'production-gce-c26']),
|
||||
configuredCellIds: new Set(environment.cells.map((cell) => cell.cellId))
|
||||
}
|
||||
|
||||
function points(values: number[], kind: 'int64Value' | 'doubleValue' = 'int64Value') {
|
||||
return values.map((value, index) => ({
|
||||
interval: { endTime: new Date(now - index * 60_000).toISOString() },
|
||||
value: { [kind]: kind === 'int64Value' ? String(value) : value }
|
||||
}))
|
||||
}
|
||||
|
||||
type Fake = {
|
||||
// instance id -> per-minute exit counts
|
||||
exits?: Record<string, number[]>
|
||||
// instance id -> the cell its runtime-metrics lines name; absent means it logged none
|
||||
cells?: Record<string, string>
|
||||
director503?: number[][]
|
||||
concurrency?: number[][]
|
||||
descriptorStatus?: number
|
||||
loggingStatus?: number
|
||||
}
|
||||
|
||||
function fakeGoogle(fake: Fake) {
|
||||
const requests: { url: URL; body: string | null }[] = []
|
||||
const fetchImpl: typeof fetch = async (input, init) => {
|
||||
const url = new URL(String(input))
|
||||
const body = typeof init?.body === 'string' ? init.body : null
|
||||
requests.push({ url, body })
|
||||
if (url.pathname.includes('/metricDescriptors/')) {
|
||||
return new Response('{}', { status: fake.descriptorStatus ?? 200 })
|
||||
}
|
||||
if (url.hostname === 'logging.googleapis.com') {
|
||||
if (fake.loggingStatus) return new Response('{}', { status: fake.loggingStatus })
|
||||
const filter = String(JSON.parse(body ?? '{}').filter ?? '')
|
||||
const instanceId = /instance_id="([0-9]+)"/.exec(filter)?.[1] ?? ''
|
||||
const cellId = fake.cells?.[instanceId]
|
||||
return Response.json({ entries: cellId ? [{ jsonPayload: { cellId } }] : [] })
|
||||
}
|
||||
const filter = url.searchParams.get('filter') ?? ''
|
||||
if (filter.includes(CELL_PROCESS_EXIT_METRIC)) {
|
||||
return Response.json({
|
||||
timeSeries: Object.entries(fake.exits ?? {}).map(([instanceId, values]) => ({
|
||||
resource: { labels: { instance_id: instanceId } },
|
||||
points: points(values)
|
||||
}))
|
||||
})
|
||||
}
|
||||
const series = filter.includes('request_count')
|
||||
? (fake.director503 ?? []).map((values) => points(values))
|
||||
: (fake.concurrency ?? [[1]]).map((values) => points(values, 'doubleValue'))
|
||||
return Response.json({ timeSeries: series.map((entry) => ({ points: entry })) })
|
||||
}
|
||||
return { fetchImpl, requests }
|
||||
}
|
||||
|
||||
function reader(fetchImpl: typeof fetch) {
|
||||
return createPreDrainHardRuleReader(environment, async () => 'token', scope, {
|
||||
fetchImpl,
|
||||
now: () => now
|
||||
})
|
||||
}
|
||||
|
||||
describe('pre-drain hard rule reader', () => {
|
||||
it('takes the busiest 503 minute and the highest concurrency p99', async () => {
|
||||
const { fetchImpl } = fakeGoogle({
|
||||
director503: [[40, 620, 55]],
|
||||
concurrency: [[12.5, 66.8]]
|
||||
})
|
||||
await expect(reader(fetchImpl)()).resolves.toEqual({
|
||||
cellProcessExits: 0,
|
||||
unattributedExitInstances: [],
|
||||
director503PeakPerMinute: 620,
|
||||
directorConcurrencyP99: 66.8
|
||||
})
|
||||
})
|
||||
|
||||
it('reads an empty concurrency series as unknown, not calm', async () => {
|
||||
const { fetchImpl } = fakeGoogle({ concurrency: [] })
|
||||
await expect(reader(fetchImpl)()).resolves.toMatchObject({ directorConcurrencyP99: null })
|
||||
})
|
||||
|
||||
it('ignores the target cell rolling the fix for its own crashes', async () => {
|
||||
const { fetchImpl } = fakeGoogle({
|
||||
exits: { '2525': [3, 4] },
|
||||
cells: { '2525': 'production-gce-c25' }
|
||||
})
|
||||
await expect(reader(fetchImpl)()).resolves.toMatchObject({
|
||||
cellProcessExits: 0,
|
||||
unattributedExitInstances: []
|
||||
})
|
||||
})
|
||||
|
||||
it('counts an exit on another cell that takes placements', async () => {
|
||||
const { fetchImpl } = fakeGoogle({
|
||||
exits: { '2626': [1], '2525': [5] },
|
||||
cells: { '2626': 'production-gce-c26', '2525': 'production-gce-c25' }
|
||||
})
|
||||
await expect(reader(fetchImpl)()).resolves.toMatchObject({
|
||||
cellProcessExits: 1,
|
||||
unattributedExitInstances: []
|
||||
})
|
||||
})
|
||||
|
||||
it('ignores an existing-only legacy cell', async () => {
|
||||
const { fetchImpl } = fakeGoogle({
|
||||
exits: { '505': [15] },
|
||||
cells: { '505': 'production-gce-c5' }
|
||||
})
|
||||
await expect(reader(fetchImpl)()).resolves.toMatchObject({
|
||||
cellProcessExits: 0,
|
||||
unattributedExitInstances: []
|
||||
})
|
||||
})
|
||||
|
||||
it('reports an exit it cannot name a cell for as unattributed', async () => {
|
||||
const { fetchImpl } = fakeGoogle({ exits: { '999': [1] } })
|
||||
await expect(reader(fetchImpl)()).resolves.toMatchObject({
|
||||
cellProcessExits: 0,
|
||||
unattributedExitInstances: ['999']
|
||||
})
|
||||
})
|
||||
|
||||
it('fails the read, rather than guessing, when the cell lookup itself fails', async () => {
|
||||
const { fetchImpl } = fakeGoogle({ exits: { '2626': [1] }, loggingStatus: 403 })
|
||||
await expect(reader(fetchImpl)()).rejects.toThrow('Google telemetry returned 403')
|
||||
})
|
||||
|
||||
it('names an instance from its own runtime-metrics lines, once', async () => {
|
||||
const { fetchImpl, requests } = fakeGoogle({
|
||||
exits: { '2626': [1] },
|
||||
cells: { '2626': 'production-gce-c26' }
|
||||
})
|
||||
const read = reader(fetchImpl)
|
||||
await read()
|
||||
await read()
|
||||
const lookups = requests.filter(({ url }) => url.hostname === 'logging.googleapis.com')
|
||||
expect(lookups).toHaveLength(1)
|
||||
const body = JSON.parse(lookups[0]?.body ?? '{}')
|
||||
expect(body.filter).toContain('resource.labels.instance_id="2626"')
|
||||
expect(body.filter).toContain('jsonPayload.event="orca_relay_runtime_metrics"')
|
||||
expect(body.filter).toContain(
|
||||
`timestamp>="${new Date(now - INSTANCE_CELL_LOOKBACK_MS).toISOString()}"`
|
||||
)
|
||||
expect(body.orderBy).toBe('timestamp desc')
|
||||
})
|
||||
|
||||
it('asks again next sample for an instance it could not name', async () => {
|
||||
const fake: Fake = { exits: { '2626': [1] } }
|
||||
const { fetchImpl, requests } = fakeGoogle(fake)
|
||||
const read = reader(fetchImpl)
|
||||
await expect(read()).resolves.toMatchObject({ unattributedExitInstances: ['2626'] })
|
||||
fake.cells = { '2626': 'production-gce-c26' }
|
||||
await expect(read()).resolves.toMatchObject({
|
||||
cellProcessExits: 1,
|
||||
unattributedExitInstances: []
|
||||
})
|
||||
expect(requests.filter(({ url }) => url.hostname === 'logging.googleapis.com'))
|
||||
.toHaveLength(2)
|
||||
})
|
||||
|
||||
it('aligns each read per minute over its own lookback on the director service', async () => {
|
||||
const { fetchImpl, requests } = fakeGoogle({})
|
||||
await reader(fetchImpl)()
|
||||
const reads = requests.map(({ url }) => url).filter((url) => url.pathname.endsWith('/timeSeries'))
|
||||
expect(reads).toHaveLength(3)
|
||||
const byMetric = (needle: string): URL => {
|
||||
const url = reads.find((entry) => entry.searchParams.get('filter')?.includes(needle))
|
||||
if (!url) throw new Error(`no read for ${needle}`)
|
||||
return url
|
||||
}
|
||||
const exits = byMetric(CELL_PROCESS_EXIT_METRIC)
|
||||
expect(exits.searchParams.get('interval.startTime')).toBe('2026-10-01T11:50:00.000Z')
|
||||
expect(exits.searchParams.get('aggregation.groupByFields'))
|
||||
.toBe('resource.label."instance_id"')
|
||||
const director503 = byMetric('run.googleapis.com/request_count')
|
||||
expect(director503.searchParams.get('filter')).toContain('metric.label."response_code"="503"')
|
||||
expect(director503.searchParams.get('filter')).toContain(
|
||||
`resource.label."service_name"="${environment.directorService}"`
|
||||
)
|
||||
expect(director503.searchParams.get('aggregation.perSeriesAligner')).toBe('ALIGN_DELTA')
|
||||
expect(director503.searchParams.get('aggregation.crossSeriesReducer')).toBe('REDUCE_SUM')
|
||||
const concurrency = byMetric('max_request_concurrencies')
|
||||
expect(concurrency.searchParams.get('interval.startTime')).toBe('2026-10-01T11:56:00.000Z')
|
||||
expect(concurrency.searchParams.get('aggregation.perSeriesAligner'))
|
||||
.toBe('ALIGN_PERCENTILE_99')
|
||||
for (const url of reads) {
|
||||
expect(url.searchParams.get('aggregation.alignmentPeriod')).toBe('60s')
|
||||
expect(url.searchParams.get('interval.endTime')).toBe('2026-10-01T12:00:00.000Z')
|
||||
}
|
||||
})
|
||||
|
||||
// Why: a deleted or renamed log metric reads as an empty series, which is zero exits forever.
|
||||
it('refuses to count exits from a metric that does not exist', async () => {
|
||||
const { fetchImpl } = fakeGoogle({ descriptorStatus: 404 })
|
||||
await expect(reader(fetchImpl)()).rejects.toThrow('Google telemetry returned 404')
|
||||
})
|
||||
})
|
||||
@@ -1,217 +0,0 @@
|
||||
import { z } from 'zod'
|
||||
import type { RelayOpsEnvironment } from './environment-config.js'
|
||||
import {
|
||||
googleJson,
|
||||
MonitoringPointSchema,
|
||||
MonitoringResponseSchema,
|
||||
pointValue
|
||||
} from './incident-monitor-sources.js'
|
||||
import {
|
||||
attributeCellExits,
|
||||
PRE_DRAIN_HARD_RULES,
|
||||
type PreDrainHardRuleReadings
|
||||
} from './pre-drain-sample.js'
|
||||
|
||||
// The Docker `container die` count per relay cell, from infra/terraform/relay-observability.tf.
|
||||
export const CELL_PROCESS_EXIT_METRIC = 'logging.googleapis.com/user/orca_relay_cell_process_exit'
|
||||
|
||||
// How far back a cell's own runtime-metrics line is searched to name an instance. The instance id
|
||||
// survives a container restart, so a crash-looping container is still named by its earlier lines.
|
||||
export const INSTANCE_CELL_LOOKBACK_MS = 2 * 60 * 60_000
|
||||
|
||||
const ExitsByInstanceSchema = z.object({
|
||||
timeSeries: z.array(z.object({
|
||||
resource: z.object({ labels: z.object({ instance_id: z.string().regex(/^[0-9]+$/) }) }),
|
||||
points: z.array(MonitoringPointSchema)
|
||||
})).default([]),
|
||||
nextPageToken: z.string().optional()
|
||||
})
|
||||
|
||||
const RuntimeMetricsEntriesSchema = z.object({
|
||||
entries: z.array(z.object({
|
||||
jsonPayload: z.object({ cellId: z.string() })
|
||||
})).default([])
|
||||
})
|
||||
|
||||
export type PreDrainExitScope = {
|
||||
targetCellId: string
|
||||
placementCellIds: ReadonlySet<string>
|
||||
configuredCellIds: ReadonlySet<string>
|
||||
}
|
||||
|
||||
type AlignedRead = {
|
||||
filter: string
|
||||
lookbackMs: number
|
||||
aligner: 'ALIGN_DELTA' | 'ALIGN_PERCENTILE_99'
|
||||
reducer: 'REDUCE_SUM' | 'REDUCE_MAX'
|
||||
}
|
||||
|
||||
// Server-side per-minute alignment, so a burst is counted rather than sampled and no read can be
|
||||
// truncated the way a log read is.
|
||||
async function readPerMinute(
|
||||
environment: RelayOpsEnvironment,
|
||||
token: string,
|
||||
fetchImpl: typeof fetch,
|
||||
nowMs: number,
|
||||
read: AlignedRead
|
||||
): Promise<number[]> {
|
||||
const url = new URL(
|
||||
`https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries`
|
||||
)
|
||||
url.searchParams.set('filter', read.filter)
|
||||
url.searchParams.set('interval.startTime', new Date(nowMs - read.lookbackMs).toISOString())
|
||||
url.searchParams.set('interval.endTime', new Date(nowMs).toISOString())
|
||||
url.searchParams.set('aggregation.alignmentPeriod', '60s')
|
||||
url.searchParams.set('aggregation.perSeriesAligner', read.aligner)
|
||||
url.searchParams.set('aggregation.crossSeriesReducer', read.reducer)
|
||||
url.searchParams.set('pageSize', '1000')
|
||||
const parsed = MonitoringResponseSchema.parse(await googleJson(fetchImpl, token, url))
|
||||
if (parsed.nextPageToken) throw new Error('Google metric pagination is incomplete')
|
||||
return parsed.timeSeries.flatMap((series) => series.points.map(pointValue))
|
||||
}
|
||||
|
||||
function directorFilter(environment: RelayOpsEnvironment, metricType: string): string {
|
||||
return [
|
||||
`metric.type="${metricType}"`,
|
||||
'resource.type="cloud_run_revision"',
|
||||
`resource.label."service_name"="${environment.directorService}"`
|
||||
].join(' AND ')
|
||||
}
|
||||
|
||||
async function readExitsByInstance(
|
||||
environment: RelayOpsEnvironment,
|
||||
token: string,
|
||||
fetchImpl: typeof fetch,
|
||||
nowMs: number
|
||||
): Promise<Map<string, number>> {
|
||||
const url = new URL(
|
||||
`https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries`
|
||||
)
|
||||
url.searchParams.set(
|
||||
'filter',
|
||||
`metric.type="${CELL_PROCESS_EXIT_METRIC}" AND resource.type="gce_instance"`
|
||||
)
|
||||
url.searchParams.set(
|
||||
'interval.startTime',
|
||||
new Date(nowMs - PRE_DRAIN_HARD_RULES.cellProcessExitLookbackMs).toISOString()
|
||||
)
|
||||
url.searchParams.set('interval.endTime', new Date(nowMs).toISOString())
|
||||
url.searchParams.set('aggregation.alignmentPeriod', '60s')
|
||||
url.searchParams.set('aggregation.perSeriesAligner', 'ALIGN_SUM')
|
||||
url.searchParams.set('aggregation.crossSeriesReducer', 'REDUCE_SUM')
|
||||
url.searchParams.set('aggregation.groupByFields', 'resource.label."instance_id"')
|
||||
url.searchParams.set('pageSize', '1000')
|
||||
const parsed = ExitsByInstanceSchema.parse(await googleJson(fetchImpl, token, url))
|
||||
if (parsed.nextPageToken) throw new Error('Google metric pagination is incomplete')
|
||||
const exits = new Map<string, number>()
|
||||
for (const series of parsed.timeSeries) {
|
||||
const instanceId = series.resource.labels.instance_id
|
||||
const total = series.points.map(pointValue).reduce((sum, value) => sum + value, 0)
|
||||
exits.set(instanceId, (exits.get(instanceId) ?? 0) + total)
|
||||
}
|
||||
return exits
|
||||
}
|
||||
|
||||
// The cell an instance serves, from that instance's own newest runtime-metrics line; the exit
|
||||
// metric carries only the instance id. Null when the instance logged none in the lookback.
|
||||
async function readInstanceCell(
|
||||
environment: RelayOpsEnvironment,
|
||||
token: string,
|
||||
fetchImpl: typeof fetch,
|
||||
nowMs: number,
|
||||
instanceId: string
|
||||
): Promise<string | null> {
|
||||
const since = new Date(nowMs - INSTANCE_CELL_LOOKBACK_MS).toISOString()
|
||||
const body = RuntimeMetricsEntriesSchema.parse(await googleJson(
|
||||
fetchImpl,
|
||||
token,
|
||||
'https://logging.googleapis.com/v2/entries:list',
|
||||
{
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
resourceNames: [`projects/${environment.project}`],
|
||||
filter: [
|
||||
'resource.type="gce_instance"',
|
||||
`resource.labels.instance_id="${instanceId}"`,
|
||||
'jsonPayload.event="orca_relay_runtime_metrics"',
|
||||
`timestamp>="${since}"`
|
||||
].join(' AND '),
|
||||
orderBy: 'timestamp desc',
|
||||
pageSize: 1
|
||||
})
|
||||
}
|
||||
))
|
||||
return body.entries[0]?.jsonPayload.cellId ?? null
|
||||
}
|
||||
|
||||
export function createPreDrainHardRuleReader(
|
||||
environment: RelayOpsEnvironment,
|
||||
accessToken: () => Promise<string>,
|
||||
scope: PreDrainExitScope,
|
||||
options: { fetchImpl?: typeof fetch; now?: () => number } = {}
|
||||
): () => Promise<PreDrainHardRuleReadings> {
|
||||
const fetchImpl = options.fetchImpl ?? fetch
|
||||
const now = options.now ?? Date.now
|
||||
// An exit count of zero is only evidence if the metric exists; a renamed or deleted log metric
|
||||
// reads as an empty, calm series forever. Checked once, before the first reading is trusted.
|
||||
let descriptorChecked: Promise<void> | null = null
|
||||
const requireExitMetric = (token: string): Promise<void> => {
|
||||
descriptorChecked ??= googleJson(
|
||||
fetchImpl,
|
||||
token,
|
||||
`https://monitoring.googleapis.com/v3/projects/${environment.project}` +
|
||||
`/metricDescriptors/${CELL_PROCESS_EXIT_METRIC}`
|
||||
).then(() => undefined, (error: unknown) => {
|
||||
descriptorChecked = null
|
||||
throw error
|
||||
})
|
||||
return descriptorChecked
|
||||
}
|
||||
// An instance belongs to one cell for its whole life, so a resolved name is kept; an unresolved
|
||||
// one is asked again next sample.
|
||||
const cellByInstance = new Map<string, string>()
|
||||
return async () => {
|
||||
const token = await accessToken()
|
||||
await requireExitMetric(token)
|
||||
const nowMs = now()
|
||||
const [exitsByInstance, director503, concurrency] = await Promise.all([
|
||||
readExitsByInstance(environment, token, fetchImpl, nowMs),
|
||||
readPerMinute(environment, token, fetchImpl, nowMs, {
|
||||
filter: `${directorFilter(environment, 'run.googleapis.com/request_count')}` +
|
||||
' AND metric.label."response_code"="503"',
|
||||
lookbackMs: PRE_DRAIN_HARD_RULES.director503LookbackMs,
|
||||
aligner: 'ALIGN_DELTA',
|
||||
reducer: 'REDUCE_SUM'
|
||||
}),
|
||||
readPerMinute(environment, token, fetchImpl, nowMs, {
|
||||
filter: `${directorFilter(
|
||||
environment,
|
||||
'run.googleapis.com/container/max_request_concurrencies'
|
||||
)} AND metric.label."state"="active"`,
|
||||
lookbackMs: PRE_DRAIN_HARD_RULES.directorConcurrencyLookbackMs,
|
||||
aligner: 'ALIGN_PERCENTILE_99',
|
||||
reducer: 'REDUCE_MAX'
|
||||
})
|
||||
])
|
||||
const resolved = new Map<string, string | null>()
|
||||
for (const [instanceId, exits] of exitsByInstance) {
|
||||
if (exits <= 0) continue
|
||||
const known = cellByInstance.get(instanceId)
|
||||
const cellId = known ?? await readInstanceCell(environment, token, fetchImpl, nowMs, instanceId)
|
||||
if (cellId !== null) cellByInstance.set(instanceId, cellId)
|
||||
resolved.set(instanceId, cellId)
|
||||
}
|
||||
const attribution = attributeCellExits({
|
||||
exitsByInstance,
|
||||
cellByInstance: resolved,
|
||||
...scope
|
||||
})
|
||||
return {
|
||||
cellProcessExits: attribution.counted,
|
||||
unattributedExitInstances: attribution.unattributed,
|
||||
// 503s are a counter, so a minute with no point had none.
|
||||
director503PeakPerMinute: Math.max(0, ...director503),
|
||||
directorConcurrencyP99: concurrency.length === 0 ? null : Math.max(...concurrency)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { relayOpsEnvironment } from './environment-config.js'
|
||||
import { parsePreDrainSampleArgs, runPreDrainSampleCli } from './pre-drain-sample-cli.js'
|
||||
|
||||
const directories: string[] = []
|
||||
|
||||
afterEach(() => {
|
||||
for (const directory of directories.splice(0)) {
|
||||
rmSync(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
function membershipFile(): string {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-pre-drain-selector-'))
|
||||
directories.push(directory)
|
||||
const path = join(directory, 'selector.json')
|
||||
writeFileSync(path, JSON.stringify({
|
||||
existingOnly: relayOpsEnvironment('production').cells.map((cell) => cell.cellId),
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}))
|
||||
return path
|
||||
}
|
||||
|
||||
function args(overrides: Record<string, string> = {}): string[] {
|
||||
const values: Record<string, string> = {
|
||||
'--target-cell-id': 'production-gce-c25',
|
||||
'--target-hosts': '1200',
|
||||
'--expected-selector-generation': '40',
|
||||
'--selector-membership-file': membershipFile(),
|
||||
'--wave-index': '2',
|
||||
'--selector-wave-delta': '2',
|
||||
...overrides
|
||||
}
|
||||
return Object.entries(values).flatMap(([name, value]) => [name, value])
|
||||
}
|
||||
|
||||
describe('pre-drain sample CLI', () => {
|
||||
it('requires every option exactly once and nothing else', () => {
|
||||
expect(() => parsePreDrainSampleArgs(['--', ...args()])).not.toThrow()
|
||||
const full = args()
|
||||
for (let index = 0; index < full.length; index += 2) {
|
||||
const missing = [...full.slice(0, index), ...full.slice(index + 2)]
|
||||
expect(() => parsePreDrainSampleArgs(missing)).toThrow('usage')
|
||||
}
|
||||
expect(() => parsePreDrainSampleArgs([...full, '--wave-index', '2'])).toThrow('usage')
|
||||
expect(() => parsePreDrainSampleArgs([...full, '--skip-window', '1'])).toThrow('usage')
|
||||
expect(() => parsePreDrainSampleArgs(args({ '--target-hosts': '-1' }))).toThrow('usage')
|
||||
expect(() => parsePreDrainSampleArgs(args({ '--target-hosts': '' }))).toThrow('usage')
|
||||
})
|
||||
|
||||
it('rejects a target cell the environment does not configure', async () => {
|
||||
await expect(runPreDrainSampleCli(args({ '--target-cell-id': 'production-gce-c99' }), {
|
||||
collect: async () => { throw new Error('must not sample') },
|
||||
readHardRules: async () => { throw new Error('must not sample') }
|
||||
})).rejects.toThrow('target cell is unknown')
|
||||
})
|
||||
|
||||
it('rejects a wave index or selector delta no wave produces', async () => {
|
||||
for (const overrides of [{ '--wave-index': '10' }, { '--selector-wave-delta': '1' }]) {
|
||||
await expect(runPreDrainSampleCli(args(overrides), {
|
||||
collect: async () => { throw new Error('must not sample') },
|
||||
readHardRules: async () => { throw new Error('must not sample') }
|
||||
})).rejects.toThrow('wave index or selector wave delta is invalid')
|
||||
}
|
||||
})
|
||||
|
||||
it('sizes the window from the target hosts and reports the verdict', async () => {
|
||||
let clock = Date.parse('2026-10-01T12:00:00.000Z')
|
||||
const lines: string[] = []
|
||||
let collected = 0
|
||||
// A sample that cannot be judged green trips the run, which is enough to see the window.
|
||||
await expect(runPreDrainSampleCli(args(), {
|
||||
now: () => clock,
|
||||
wait: async (ms) => { clock += ms },
|
||||
collect: async () => {
|
||||
collected += 1
|
||||
throw new Error('collector down')
|
||||
},
|
||||
readHardRules: async () => ({
|
||||
cellProcessExits: 0,
|
||||
unattributedExitInstances: [],
|
||||
director503PeakPerMinute: 0,
|
||||
directorConcurrencyP99: 1
|
||||
}),
|
||||
print: (line) => lines.push(line)
|
||||
})).rejects.toThrow('relay pre-drain sample tripped')
|
||||
const events = lines.map((line) => JSON.parse(line))
|
||||
expect(events[0]).toEqual({
|
||||
event: 'relay_pre_drain_sample_window',
|
||||
targetCellId: 'production-gce-c25',
|
||||
targetHosts: 1200,
|
||||
windowMinutes: 5
|
||||
})
|
||||
expect(events.at(-1)).toMatchObject({ event: 'relay_pre_drain_sample_tripped' })
|
||||
expect(collected).toBe(3)
|
||||
})
|
||||
})
|
||||
@@ -1,158 +0,0 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { relayOpsEnvironment } from './environment-config.js'
|
||||
import { createGcloudClient } from './gcloud-client.js'
|
||||
import {
|
||||
livePreflightGcloud,
|
||||
readDispatchSelector,
|
||||
waveAdjustedSelector
|
||||
} from './incident-live-preflight-cli.js'
|
||||
import { createIncidentSampleCollector } from './incident-monitor-sources.js'
|
||||
import type { IncidentSample } from './incident-monitor.js'
|
||||
import { createPreDrainHardRuleReader } from './pre-drain-hard-rule-sources.js'
|
||||
import {
|
||||
preDrainSampleWindowMinutes,
|
||||
PreDrainSampleTripped,
|
||||
runPreDrainSample,
|
||||
type PreDrainHardRuleReadings
|
||||
} from './pre-drain-sample.js'
|
||||
|
||||
const USAGE =
|
||||
'usage: --target-cell-id <cell> --target-hosts <n> --expected-selector-generation <n>' +
|
||||
' --selector-membership-file <json> --wave-index <0-9> --selector-wave-delta <0|2>'
|
||||
|
||||
const OPTIONS = [
|
||||
'--target-cell-id',
|
||||
'--target-hosts',
|
||||
'--expected-selector-generation',
|
||||
'--selector-membership-file',
|
||||
'--wave-index',
|
||||
'--selector-wave-delta'
|
||||
] as const
|
||||
|
||||
type Option = (typeof OPTIONS)[number]
|
||||
|
||||
const COUNT = /^(0|[1-9][0-9]*)$/
|
||||
|
||||
function isOption(value: string): value is Option {
|
||||
return (OPTIONS as readonly string[]).includes(value)
|
||||
}
|
||||
|
||||
// Every option is required exactly once, so a typo can never silently shorten the window.
|
||||
export function parsePreDrainSampleArgs(argv: string[]): Record<Option, string> {
|
||||
const args = argv[0] === '--' ? argv.slice(1) : argv
|
||||
const values = new Map<Option, string>()
|
||||
for (let index = 0; index < args.length; index += 2) {
|
||||
const name = args[index] ?? ''
|
||||
const value = args[index + 1]
|
||||
if (!isOption(name) || values.has(name) || !value) throw new Error(USAGE)
|
||||
values.set(name, value)
|
||||
}
|
||||
const get = (name: Option): string => {
|
||||
const value = values.get(name)
|
||||
if (value === undefined) throw new Error(USAGE)
|
||||
return value
|
||||
}
|
||||
const parsed = {
|
||||
'--target-cell-id': get('--target-cell-id'),
|
||||
'--target-hosts': get('--target-hosts'),
|
||||
'--expected-selector-generation': get('--expected-selector-generation'),
|
||||
'--selector-membership-file': get('--selector-membership-file'),
|
||||
'--wave-index': get('--wave-index'),
|
||||
'--selector-wave-delta': get('--selector-wave-delta')
|
||||
}
|
||||
if (
|
||||
!COUNT.test(parsed['--target-hosts']) ||
|
||||
!COUNT.test(parsed['--expected-selector-generation'])
|
||||
) throw new Error(USAGE)
|
||||
return parsed
|
||||
}
|
||||
|
||||
export async function runPreDrainSampleCli(
|
||||
argv: string[],
|
||||
dependencies: {
|
||||
now?: () => number
|
||||
wait?: (ms: number) => Promise<void>
|
||||
collect?: () => Promise<IncidentSample>
|
||||
readHardRules?: () => Promise<PreDrainHardRuleReadings>
|
||||
gcloud?: ReturnType<typeof createGcloudClient>
|
||||
environment?: NodeJS.ProcessEnv
|
||||
print?: (line: string) => void
|
||||
} = {}
|
||||
): Promise<void> {
|
||||
const options = parsePreDrainSampleArgs(argv)
|
||||
const environment = relayOpsEnvironment('production')
|
||||
const configuredCellIds = new Set(environment.cells.map((cell) => cell.cellId))
|
||||
const targetCellId = options['--target-cell-id']
|
||||
// An unknown target would exempt nothing and make every one of its exits unattributed.
|
||||
if (!configuredCellIds.has(targetCellId)) throw new Error('pre-drain sample target cell is unknown')
|
||||
const windowMinutes = preDrainSampleWindowMinutes(Number(options['--target-hosts']))
|
||||
const expectedSelector = waveAdjustedSelector(
|
||||
await readDispatchSelector(
|
||||
options['--expected-selector-generation'],
|
||||
options['--selector-membership-file']
|
||||
),
|
||||
options['--wave-index'],
|
||||
options['--selector-wave-delta']
|
||||
)
|
||||
const print = dependencies.print ?? ((line: string) => console.log(line))
|
||||
const gcloud = livePreflightGcloud(
|
||||
dependencies.gcloud ?? createGcloudClient(),
|
||||
dependencies.environment
|
||||
)
|
||||
const collect = dependencies.collect ?? createIncidentSampleCollector(gcloud, {
|
||||
environment: 'production',
|
||||
expectedSelector,
|
||||
...(dependencies.now ? { now: dependencies.now } : {})
|
||||
})
|
||||
const { membership } = expectedSelector
|
||||
const readHardRules = dependencies.readHardRules ?? createPreDrainHardRuleReader(
|
||||
environment,
|
||||
() => gcloud.accessToken(),
|
||||
{
|
||||
targetCellId,
|
||||
placementCellIds: new Set([...membership.general, ...membership.migrationOnly]),
|
||||
configuredCellIds
|
||||
},
|
||||
dependencies.now ? { now: dependencies.now } : {}
|
||||
)
|
||||
print(JSON.stringify({
|
||||
event: 'relay_pre_drain_sample_window',
|
||||
targetCellId,
|
||||
targetHosts: Number(options['--target-hosts']),
|
||||
windowMinutes
|
||||
}))
|
||||
try {
|
||||
const result = await runPreDrainSample({
|
||||
windowMinutes,
|
||||
collect,
|
||||
readHardRules,
|
||||
...(dependencies.now ? { now: dependencies.now } : {}),
|
||||
...(dependencies.wait ? { wait: dependencies.wait } : {}),
|
||||
log: (record) => print(JSON.stringify({ event: 'relay_pre_drain_sample', ...record }))
|
||||
})
|
||||
print(JSON.stringify({
|
||||
event: 'relay_pre_drain_sample_passed',
|
||||
windowMinutes: result.windowMinutes,
|
||||
startedAt: result.startedAt,
|
||||
completedAt: result.completedAt,
|
||||
samples: result.samples.length
|
||||
}))
|
||||
} catch (error) {
|
||||
if (error instanceof PreDrainSampleTripped) {
|
||||
print(JSON.stringify({
|
||||
event: 'relay_pre_drain_sample_tripped',
|
||||
windowMinutes,
|
||||
samples: error.samples.length,
|
||||
failures: error.failures
|
||||
}))
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
runPreDrainSampleCli(process.argv.slice(2)).catch((error: unknown) => {
|
||||
console.error(error instanceof Error ? error.message : 'relay pre-drain sample failed')
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,358 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
INCIDENT_FRESHNESS_TOLERANCE_SAMPLES,
|
||||
INCIDENT_MONITOR_THRESHOLDS,
|
||||
type IncidentSample
|
||||
} from './incident-monitor.js'
|
||||
import {
|
||||
attributeCellExits,
|
||||
evaluatePreDrainHardRules,
|
||||
PRE_DRAIN_HARD_RULES,
|
||||
PRE_DRAIN_MAX_OVERRUN_SAMPLES,
|
||||
preDrainSampleWindowMinutes,
|
||||
PreDrainSampleTripped,
|
||||
runPreDrainSample,
|
||||
type PreDrainHardRuleReadings
|
||||
} from './pre-drain-sample.js'
|
||||
|
||||
const selector = {
|
||||
generation: 1,
|
||||
membership: { existingOnly: ['production-gce-c1'], migrationOnly: [], general: [] }
|
||||
}
|
||||
|
||||
function greenSample(at: number): IncidentSample {
|
||||
const observedAt = new Date(at).toISOString()
|
||||
const signal = (value: number) => ({ value, observedAt })
|
||||
const source = (signals: Record<string, number>) => ({
|
||||
observedAt,
|
||||
signals: Object.fromEntries(
|
||||
Object.entries(signals).map(([name, value]) => [name, signal(value)])
|
||||
)
|
||||
})
|
||||
return {
|
||||
collectedAt: observedAt,
|
||||
selector,
|
||||
expectedSelector: selector,
|
||||
cells: [{
|
||||
cellId: 'production-gce-c1',
|
||||
region: 'us-central1',
|
||||
runtimeKnown: true,
|
||||
powered: true,
|
||||
expectedAdmissionState: 'existing-only'
|
||||
}],
|
||||
sources: {
|
||||
'active-probe': source({
|
||||
'director.health': 1,
|
||||
'director.ready': 1,
|
||||
'director.latency_ms': 1,
|
||||
'auth.health': 1,
|
||||
'auth.ready': 1,
|
||||
'auth.latency_ms': 1,
|
||||
'cell.production-gce-c1.health': 1,
|
||||
'cell.production-gce-c1.ready': 1,
|
||||
'cell.production-gce-c1.latency_ms': 1
|
||||
}),
|
||||
'cloud-monitoring': source({
|
||||
'cloud_sql.cpu': 0.1,
|
||||
'cloud_sql.memory': 0.1,
|
||||
'cloud_sql.backends': 1,
|
||||
'cloud_sql.lock_waits': 0,
|
||||
'cloud_sql.deadlocks': 0,
|
||||
'director.instances': 5,
|
||||
'director.cpu': 0.1,
|
||||
'director.memory': 0.1,
|
||||
'director.concurrency': 1,
|
||||
'director.errors': 0,
|
||||
'auth.errors': 0
|
||||
}),
|
||||
'relay-logs': source({
|
||||
'relay.pool_waiting': 0,
|
||||
'relay.pool_wait_ms': 0,
|
||||
'relay.postgres_retries': 0,
|
||||
'relay.postgres_retry_exhausted': 0,
|
||||
'cell.production-gce-c1.connections': 1,
|
||||
'cell.production-gce-c1.queued_bytes': 0
|
||||
}),
|
||||
'director-admin': source({
|
||||
'cell.production-gce-c1.admission_state': 0,
|
||||
'cell.production-gce-c1.heartbeat_fresh': 1,
|
||||
'cell.production-gce-c1.heartbeat_age_ms': 1,
|
||||
'cell.production-gce-c1.migration_blocked': 0,
|
||||
'cell.production-gce-c1.migration_target_inactive': 0
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const calm: PreDrainHardRuleReadings = {
|
||||
cellProcessExits: 0,
|
||||
unattributedExitInstances: [],
|
||||
director503PeakPerMinute: 40,
|
||||
directorConcurrencyP99: 20
|
||||
}
|
||||
|
||||
// A fake clock the sampler's own waits advance, so a window runs in no real time.
|
||||
function harness(input: {
|
||||
windowMinutes: number
|
||||
sampleAt?: (index: number, at: number) => IncidentSample | Error
|
||||
hardRulesAt?: (index: number) => PreDrainHardRuleReadings | Error
|
||||
}) {
|
||||
let clock = Date.parse('2026-10-01T12:00:00.000Z')
|
||||
let samples = 0
|
||||
let reads = 0
|
||||
const run = runPreDrainSample({
|
||||
windowMinutes: input.windowMinutes,
|
||||
now: () => clock,
|
||||
wait: async (ms) => {
|
||||
clock += ms
|
||||
},
|
||||
collect: async () => {
|
||||
const result = input.sampleAt?.(samples, clock) ?? greenSample(clock)
|
||||
samples += 1
|
||||
if (result instanceof Error) throw result
|
||||
return result
|
||||
},
|
||||
readHardRules: async () => {
|
||||
const result = input.hardRulesAt?.(reads) ?? calm
|
||||
reads += 1
|
||||
if (result instanceof Error) throw result
|
||||
return result
|
||||
}
|
||||
})
|
||||
return { run, startedAt: clock, samples: () => samples }
|
||||
}
|
||||
|
||||
describe('pre-drain sample window', () => {
|
||||
it('sizes the window to the hosts the drain will re-place', () => {
|
||||
expect(preDrainSampleWindowMinutes(0)).toBe(3)
|
||||
expect(preDrainSampleWindowMinutes(150)).toBe(3)
|
||||
expect(preDrainSampleWindowMinutes(500)).toBe(3)
|
||||
expect(preDrainSampleWindowMinutes(501)).toBe(5)
|
||||
expect(preDrainSampleWindowMinutes(1_500)).toBe(5)
|
||||
expect(preDrainSampleWindowMinutes(1_501)).toBe(8)
|
||||
expect(preDrainSampleWindowMinutes(2_750)).toBe(8)
|
||||
})
|
||||
|
||||
it('refuses a host count it cannot size', () => {
|
||||
for (const hosts of [-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY]) {
|
||||
expect(() => preDrainSampleWindowMinutes(hosts)).toThrow('host count is invalid')
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('cell exit attribution', () => {
|
||||
const scope = {
|
||||
targetCellId: 'production-gce-c25',
|
||||
placementCellIds: new Set(['production-gce-c25', 'production-gce-c26', 'production-gce-c17']),
|
||||
configuredCellIds: new Set([
|
||||
'production-gce-c5', 'production-gce-c17', 'production-gce-c25', 'production-gce-c26'
|
||||
])
|
||||
}
|
||||
const attribute = (exits: Record<string, number>, cells: Record<string, string | null>) =>
|
||||
attributeCellExits({
|
||||
exitsByInstance: new Map(Object.entries(exits)),
|
||||
cellByInstance: new Map(Object.entries(cells)),
|
||||
...scope
|
||||
})
|
||||
|
||||
it('ignores the target cell rolling the fix for its own crashes', () => {
|
||||
expect(attribute({ '25': 12 }, { '25': 'production-gce-c25' }))
|
||||
.toEqual({ counted: 0, unattributed: [] })
|
||||
})
|
||||
|
||||
it('counts an exit on another general or migration-only cell', () => {
|
||||
expect(attribute({ '26': 1 }, { '26': 'production-gce-c26' }))
|
||||
.toEqual({ counted: 1, unattributed: [] })
|
||||
expect(attribute({ '17': 2 }, { '17': 'production-gce-c17' }))
|
||||
.toEqual({ counted: 2, unattributed: [] })
|
||||
})
|
||||
|
||||
it('ignores an existing-only legacy cell, which takes no placements', () => {
|
||||
expect(attribute({ '5': 15 }, { '5': 'production-gce-c5' }))
|
||||
.toEqual({ counted: 0, unattributed: [] })
|
||||
})
|
||||
|
||||
it('reports an instance with no cell, or an unknown cell, as unattributed', () => {
|
||||
expect(attribute({ '9': 1, '8': 1 }, { '9': null, '8': 'production-gce-c99' }))
|
||||
.toEqual({ counted: 0, unattributed: ['8', '9'] })
|
||||
expect(attribute({ '7': 1 }, {})).toEqual({ counted: 0, unattributed: ['7'] })
|
||||
})
|
||||
|
||||
it('trips the rule through the evaluator for a counted exit', () => {
|
||||
const { counted } = attribute({ '26': 1, '25': 3, '5': 4 }, {
|
||||
'26': 'production-gce-c26', '25': 'production-gce-c25', '5': 'production-gce-c5'
|
||||
})
|
||||
expect(evaluatePreDrainHardRules({ ...calm, cellProcessExits: counted })).toEqual([
|
||||
expect.objectContaining({ signal: 'cells.process_exits_10m', observed: 1 })
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('pre-drain hard rules', () => {
|
||||
it('passes a calm fleet', () => {
|
||||
expect(evaluatePreDrainHardRules(calm)).toEqual([])
|
||||
})
|
||||
|
||||
it('trips on any cell container exit in the lookback', () => {
|
||||
expect(evaluatePreDrainHardRules({ ...calm, cellProcessExits: 1 })).toEqual([
|
||||
expect.objectContaining({ signal: 'cells.process_exits_10m', observed: 1, threshold: 0 })
|
||||
])
|
||||
})
|
||||
|
||||
it('trips on an exit no cell could be named for', () => {
|
||||
expect(evaluatePreDrainHardRules({ ...calm, unattributedExitInstances: ['111', '222'] }))
|
||||
.toEqual([expect.objectContaining({
|
||||
code: 'exit_unattributed',
|
||||
signal: 'cells.process_exits_10m.instances=111+222',
|
||||
observed: 2
|
||||
})])
|
||||
})
|
||||
|
||||
it('trips on a disconnect pulse, not on a busy healthy minute', () => {
|
||||
expect(evaluatePreDrainHardRules({ ...calm, director503PeakPerMinute: 500 })).toEqual([])
|
||||
expect(evaluatePreDrainHardRules({ ...calm, director503PeakPerMinute: 501 })).toEqual([
|
||||
expect.objectContaining({ signal: 'director.503_peak_minute_10m', observed: 501 })
|
||||
])
|
||||
})
|
||||
|
||||
it('holds director concurrency p99 to the monitor threshold', () => {
|
||||
expect(PRE_DRAIN_HARD_RULES.directorConcurrencyP99Max)
|
||||
.toBe(INCIDENT_MONITOR_THRESHOLDS.directorConcurrency)
|
||||
const max = PRE_DRAIN_HARD_RULES.directorConcurrencyP99Max
|
||||
expect(evaluatePreDrainHardRules({ ...calm, directorConcurrencyP99: max })).toEqual([])
|
||||
// The 2026-10-01 13:42Z refusal.
|
||||
expect(evaluatePreDrainHardRules({ ...calm, directorConcurrencyP99: 66.8 })).toEqual([
|
||||
expect.objectContaining({ code: 'threshold_max', signal: 'director.concurrency_p99_4m' })
|
||||
])
|
||||
})
|
||||
|
||||
it('reads an unpublished concurrency series as missing, never as calm', () => {
|
||||
expect(evaluatePreDrainHardRules({ ...calm, directorConcurrencyP99: null })).toEqual([
|
||||
expect.objectContaining({ code: 'signal_missing', signal: 'director.concurrency_p99_4m' })
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('runPreDrainSample', () => {
|
||||
it('passes after one sample per minute across the whole window', async () => {
|
||||
for (const windowMinutes of [3, 5, 8]) {
|
||||
const { run, startedAt, samples } = harness({ windowMinutes })
|
||||
const result = await run
|
||||
expect(samples()).toBe(windowMinutes + 1)
|
||||
expect(Date.parse(result.completedAt) - startedAt).toBeGreaterThanOrEqual(
|
||||
windowMinutes * 60_000
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('refuses a window that is not one of the sized bands', async () => {
|
||||
await expect(harness({ windowMinutes: 15 }).run).rejects.toThrow('window is invalid')
|
||||
})
|
||||
|
||||
it('trips at once on a hard rule, before the window ends', async () => {
|
||||
const { run, samples } = harness({
|
||||
windowMinutes: 8,
|
||||
hardRulesAt: (index) => (index === 2 ? { ...calm, cellProcessExits: 1 } : calm)
|
||||
})
|
||||
await expect(run).rejects.toBeInstanceOf(PreDrainSampleTripped)
|
||||
await expect(run).rejects.toThrow('cells.process_exits_10m')
|
||||
expect(samples()).toBe(3)
|
||||
})
|
||||
|
||||
it('trips at once on a monitor threshold the monitor never tolerates', async () => {
|
||||
const { run } = harness({
|
||||
windowMinutes: 3,
|
||||
sampleAt: (index, at) => {
|
||||
const next = greenSample(at)
|
||||
if (index === 1) {
|
||||
next.sources['cloud-monitoring']!.signals['director.concurrency'] = {
|
||||
value: 80,
|
||||
observedAt: new Date(at).toISOString()
|
||||
}
|
||||
}
|
||||
return next
|
||||
}
|
||||
})
|
||||
await expect(run).rejects.toThrow('director.concurrency observed=80')
|
||||
})
|
||||
|
||||
it('rides out a cell probe within the monitor tolerance and trips past it', async () => {
|
||||
const failingProbe = (failFor: number) => (index: number, at: number) => {
|
||||
const next = greenSample(at)
|
||||
if (index >= 1 && index < 1 + failFor) {
|
||||
next.sources['active-probe']!.signals['cell.production-gce-c1.health'] = {
|
||||
value: 0,
|
||||
observedAt: new Date(at).toISOString()
|
||||
}
|
||||
}
|
||||
return next
|
||||
}
|
||||
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
await expect(
|
||||
harness({ windowMinutes: 3, sampleAt: failingProbe(tolerance) }).run
|
||||
).resolves.toBeDefined()
|
||||
await expect(
|
||||
harness({ windowMinutes: 3, sampleAt: failingProbe(tolerance + 1) }).run
|
||||
).rejects.toThrow('cell.production-gce-c1.health')
|
||||
})
|
||||
|
||||
it('rides out failed reads within the freshness tolerance and trips past it', async () => {
|
||||
const failingReads = (failFor: number) => (index: number) =>
|
||||
index >= 1 && index < 1 + failFor ? new Error('Cloud Monitoring returned 503') : calm
|
||||
await expect(
|
||||
harness({
|
||||
windowMinutes: 3,
|
||||
hardRulesAt: failingReads(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES)
|
||||
}).run
|
||||
).resolves.toBeDefined()
|
||||
await expect(
|
||||
harness({
|
||||
windowMinutes: 3,
|
||||
hardRulesAt: failingReads(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1)
|
||||
}).run
|
||||
).rejects.toThrow('collector_failed pre-drain.hard-rules')
|
||||
})
|
||||
|
||||
// Why: the drain starts right after this returns, so it must not start on a reading the
|
||||
// tolerance was still carrying.
|
||||
it('keeps sampling past the window until a sample is clean', async () => {
|
||||
const { run, samples } = harness({
|
||||
windowMinutes: 3,
|
||||
hardRulesAt: (index) => (index === 3 ? { ...calm, directorConcurrencyP99: null } : calm)
|
||||
})
|
||||
const result = await run
|
||||
expect(samples()).toBe(5)
|
||||
expect(result.samples.at(-1)?.failures).toEqual([])
|
||||
})
|
||||
|
||||
// Why: two tolerated readings that alternate never build a streak, so only the overrun cap
|
||||
// stops them from holding the step open until its timeout.
|
||||
it('trips when alternating tolerated readings never leave a clean sample', async () => {
|
||||
const { run, samples } = harness({
|
||||
windowMinutes: 3,
|
||||
hardRulesAt: (index) =>
|
||||
index >= 3 && index % 2 === 1 ? { ...calm, directorConcurrencyP99: null } : calm,
|
||||
sampleAt: (index, at) =>
|
||||
index >= 3 && index % 2 === 0 ? new Error('collector down') : greenSample(at)
|
||||
})
|
||||
await expect(run).rejects.toBeInstanceOf(PreDrainSampleTripped)
|
||||
expect(samples()).toBe(4 + PRE_DRAIN_MAX_OVERRUN_SAMPLES)
|
||||
})
|
||||
|
||||
it('records why a read failed', async () => {
|
||||
const records: string[][] = []
|
||||
let clock = 0
|
||||
await runPreDrainSample({
|
||||
windowMinutes: 3,
|
||||
now: () => clock,
|
||||
wait: async (ms) => { clock += ms },
|
||||
collect: async () => greenSample(clock),
|
||||
readHardRules: async () => {
|
||||
if (records.length === 1) throw new Error('Google telemetry returned 403')
|
||||
return calm
|
||||
},
|
||||
log: (record) => records.push(record.errors)
|
||||
})
|
||||
expect(records[1]).toEqual(['hard rules: Google telemetry returned 403'])
|
||||
})
|
||||
})
|
||||
@@ -1,259 +0,0 @@
|
||||
import {
|
||||
describeIncidentFailure,
|
||||
evaluateIncidentSample,
|
||||
FRESHNESS_FAILURE_CODES,
|
||||
INCIDENT_FRESHNESS_TOLERANCE_SAMPLES,
|
||||
INCIDENT_MONITOR_THRESHOLDS,
|
||||
toleratedStreakKey,
|
||||
type IncidentFailure,
|
||||
type IncidentSample
|
||||
} from './incident-monitor.js'
|
||||
|
||||
// Inline fleet-health sample a same-cap wave takes right before it isolates its cell. It judges
|
||||
// every sample with the monitor's own evaluator and tolerances; what it adds is a window sized to
|
||||
// the drain and three lookback rules no single sample can see.
|
||||
|
||||
export const PRE_DRAIN_SAMPLE_INTERVAL_MS = 60_000
|
||||
|
||||
// A bigger drain re-places more hosts through the director's single sticky lane, so it gets a
|
||||
// longer quiet window before it starts. Ordered; the first band whose ceiling covers the cell wins.
|
||||
export const PRE_DRAIN_SAMPLE_WINDOWS = [
|
||||
{ maxHosts: 500, minutes: 3 },
|
||||
{ maxHosts: 1_500, minutes: 5 },
|
||||
{ maxHosts: Number.POSITIVE_INFINITY, minutes: 8 }
|
||||
] as const
|
||||
|
||||
export const PRE_DRAIN_HARD_RULES = {
|
||||
// Any container exit on a cell that takes placements, other than the cell being rolled; a drain
|
||||
// into a crash loop re-places its hosts twice.
|
||||
cellProcessExitLookbackMs: 10 * 60_000,
|
||||
cellProcessExitsMax: 0,
|
||||
// A disconnect pulse lands as thousands of director 503s in one minute (7,750 and 13,000 on
|
||||
// 2026-09-20 and 2026-10-01) against a healthy 20-70.
|
||||
director503LookbackMs: 10 * 60_000,
|
||||
director503PerMinuteMax: 500,
|
||||
// The live preflight reads only the newest concurrency point; this holds the last four minutes.
|
||||
directorConcurrencyLookbackMs: 4 * 60_000,
|
||||
directorConcurrencyP99Max: INCIDENT_MONITOR_THRESHOLDS.directorConcurrency
|
||||
} as const
|
||||
|
||||
export type PreDrainHardRuleReadings = {
|
||||
// Exits on general or migration-only cells other than the target.
|
||||
cellProcessExits: number
|
||||
// Exits on an instance no cell could be resolved for; never assumed harmless.
|
||||
unattributedExitInstances: string[]
|
||||
director503PeakPerMinute: number
|
||||
// Null when Cloud Monitoring published no point in the lookback.
|
||||
directorConcurrencyP99: number | null
|
||||
}
|
||||
|
||||
// Which exits count. The target's own exits are why it is being rolled (c25 crashed 12 times in
|
||||
// a week and is the first US roll), and existing-only legacy cells take no placements (c5 alone
|
||||
// exited 15 times in that week), so neither can stop a drain. An instance with no known cell
|
||||
// counts as unattributed and trips the rule.
|
||||
export function attributeCellExits(input: {
|
||||
exitsByInstance: ReadonlyMap<string, number>
|
||||
cellByInstance: ReadonlyMap<string, string | null>
|
||||
targetCellId: string
|
||||
placementCellIds: ReadonlySet<string>
|
||||
configuredCellIds: ReadonlySet<string>
|
||||
}): { counted: number; unattributed: string[] } {
|
||||
let counted = 0
|
||||
const unattributed: string[] = []
|
||||
for (const [instanceId, exits] of input.exitsByInstance) {
|
||||
if (exits <= 0) continue
|
||||
const cellId = input.cellByInstance.get(instanceId) ?? null
|
||||
if (cellId === null || !input.configuredCellIds.has(cellId)) {
|
||||
unattributed.push(instanceId)
|
||||
continue
|
||||
}
|
||||
if (cellId === input.targetCellId || !input.placementCellIds.has(cellId)) continue
|
||||
counted += exits
|
||||
}
|
||||
return { counted, unattributed: unattributed.sort() }
|
||||
}
|
||||
|
||||
export function preDrainSampleWindowMinutes(hostCount: number): number {
|
||||
if (!Number.isSafeInteger(hostCount) || hostCount < 0) {
|
||||
throw new Error('pre-drain sample host count is invalid')
|
||||
}
|
||||
const band = PRE_DRAIN_SAMPLE_WINDOWS.find((entry) => hostCount <= entry.maxHosts)
|
||||
if (!band) throw new Error('pre-drain sample host count is invalid')
|
||||
return band.minutes
|
||||
}
|
||||
|
||||
export function evaluatePreDrainHardRules(readings: PreDrainHardRuleReadings): IncidentFailure[] {
|
||||
const failures: IncidentFailure[] = []
|
||||
if (readings.cellProcessExits > PRE_DRAIN_HARD_RULES.cellProcessExitsMax) {
|
||||
failures.push({
|
||||
code: 'threshold_max',
|
||||
source: 'cloud-monitoring',
|
||||
signal: 'cells.process_exits_10m',
|
||||
observed: readings.cellProcessExits,
|
||||
threshold: PRE_DRAIN_HARD_RULES.cellProcessExitsMax
|
||||
})
|
||||
}
|
||||
if (readings.unattributedExitInstances.length > 0) {
|
||||
failures.push({
|
||||
code: 'exit_unattributed',
|
||||
source: 'cloud-monitoring',
|
||||
signal: `cells.process_exits_10m.instances=${readings.unattributedExitInstances.join('+')}`,
|
||||
observed: readings.unattributedExitInstances.length,
|
||||
threshold: 0
|
||||
})
|
||||
}
|
||||
if (readings.director503PeakPerMinute > PRE_DRAIN_HARD_RULES.director503PerMinuteMax) {
|
||||
failures.push({
|
||||
code: 'threshold_max',
|
||||
source: 'cloud-monitoring',
|
||||
signal: 'director.503_peak_minute_10m',
|
||||
observed: readings.director503PeakPerMinute,
|
||||
threshold: PRE_DRAIN_HARD_RULES.director503PerMinuteMax
|
||||
})
|
||||
}
|
||||
if (readings.directorConcurrencyP99 === null) {
|
||||
failures.push({
|
||||
code: 'signal_missing',
|
||||
source: 'cloud-monitoring',
|
||||
signal: 'director.concurrency_p99_4m'
|
||||
})
|
||||
} else if (readings.directorConcurrencyP99 > PRE_DRAIN_HARD_RULES.directorConcurrencyP99Max) {
|
||||
failures.push({
|
||||
code: 'threshold_max',
|
||||
source: 'cloud-monitoring',
|
||||
signal: 'director.concurrency_p99_4m',
|
||||
observed: readings.directorConcurrencyP99,
|
||||
threshold: PRE_DRAIN_HARD_RULES.directorConcurrencyP99Max
|
||||
})
|
||||
}
|
||||
return failures
|
||||
}
|
||||
|
||||
export type PreDrainSampleRecord = {
|
||||
at: string
|
||||
failures: IncidentFailure[]
|
||||
// Why a read failed; a failure code alone hides a permission error or a bad filter.
|
||||
errors: string[]
|
||||
}
|
||||
|
||||
// Samples past the window that may still carry a tolerated failure; alternating tolerated readings
|
||||
// (cell X, then cell Y) never trip a streak, so this is what bounds the overrun.
|
||||
export const PRE_DRAIN_MAX_OVERRUN_SAMPLES = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
|
||||
|
||||
export type PreDrainSampleResult = {
|
||||
windowMinutes: number
|
||||
startedAt: string
|
||||
completedAt: string
|
||||
samples: PreDrainSampleRecord[]
|
||||
}
|
||||
|
||||
export class PreDrainSampleTripped extends Error {
|
||||
constructor(readonly failures: IncidentFailure[], readonly samples: PreDrainSampleRecord[]) {
|
||||
super(`relay pre-drain sample tripped: ${failures.map(describeIncidentFailure).join(',')}`)
|
||||
}
|
||||
}
|
||||
|
||||
const CONTINUITY_CODES = new Set(['collector_failed', ...FRESHNESS_FAILURE_CODES])
|
||||
|
||||
function continuityKey(failure: IncidentFailure): string {
|
||||
return `${failure.source}/${failure.signal ?? '*'}`
|
||||
}
|
||||
|
||||
function bumpStreaks(streaks: Map<string, number>, keys: Set<string>): Map<string, number> {
|
||||
const next = new Map<string, number>()
|
||||
for (const key of keys) next.set(key, (streaks.get(key) ?? 0) + 1)
|
||||
return next
|
||||
}
|
||||
|
||||
/**
|
||||
* Samples once a minute until the window has elapsed, then keeps going until a sample comes back
|
||||
* with no failure at all, so the drain never starts on an open reading. A breach of any
|
||||
* non-tolerated threshold or hard rule trips at once; readings the monitor tolerates (a cell probe
|
||||
* or the director instance count, an unread signal, a failed collector round trip) trip only past
|
||||
* the monitor's own consecutive-sample budget. The overrun is capped separately.
|
||||
*/
|
||||
export async function runPreDrainSample(input: {
|
||||
windowMinutes: number
|
||||
collect: () => Promise<IncidentSample>
|
||||
readHardRules: () => Promise<PreDrainHardRuleReadings>
|
||||
now?: () => number
|
||||
wait?: (ms: number) => Promise<void>
|
||||
log?: (record: PreDrainSampleRecord) => void
|
||||
}): Promise<PreDrainSampleResult> {
|
||||
if (!PRE_DRAIN_SAMPLE_WINDOWS.some((band) => band.minutes === input.windowMinutes)) {
|
||||
throw new Error('pre-drain sample window is invalid')
|
||||
}
|
||||
const now = input.now ?? Date.now
|
||||
const wait = input.wait ?? ((ms: number) => new Promise<void>((resolveWait) => {
|
||||
setTimeout(resolveWait, ms)
|
||||
}))
|
||||
const startedMs = now()
|
||||
const windowMs = input.windowMinutes * 60_000
|
||||
const samples: PreDrainSampleRecord[] = []
|
||||
let continuityStreaks = new Map<string, number>()
|
||||
let probeStreaks = new Map<string, number>()
|
||||
let overrunSamples = 0
|
||||
for (;;) {
|
||||
const sampleStartedMs = now()
|
||||
const failures: IncidentFailure[] = []
|
||||
const errors: string[] = []
|
||||
const reason = (error: unknown) => error instanceof Error ? error.message : String(error)
|
||||
try {
|
||||
failures.push(
|
||||
...evaluateIncidentSample(await input.collect(), now(), 'strict', null, null).failures
|
||||
)
|
||||
} catch (error) {
|
||||
failures.push({ code: 'collector_failed', source: 'cloud-monitoring' })
|
||||
errors.push(`collector: ${reason(error)}`)
|
||||
}
|
||||
try {
|
||||
failures.push(...evaluatePreDrainHardRules(await input.readHardRules()))
|
||||
} catch (error) {
|
||||
failures.push({
|
||||
code: 'collector_failed',
|
||||
source: 'cloud-monitoring',
|
||||
signal: 'pre-drain.hard-rules'
|
||||
})
|
||||
errors.push(`hard rules: ${reason(error)}`)
|
||||
}
|
||||
const record = { at: new Date(now()).toISOString(), failures, errors }
|
||||
samples.push(record)
|
||||
input.log?.(record)
|
||||
const continuity = failures.filter((failure) => CONTINUITY_CODES.has(failure.code))
|
||||
const breaches = failures.filter((failure) => !CONTINUITY_CODES.has(failure.code))
|
||||
continuityStreaks = bumpStreaks(continuityStreaks, new Set(continuity.map(continuityKey)))
|
||||
const probeKeys = new Set<string>()
|
||||
for (const failure of breaches) {
|
||||
const key = toleratedStreakKey(failure)
|
||||
if (key !== null) probeKeys.add(key)
|
||||
}
|
||||
probeStreaks = bumpStreaks(probeStreaks, probeKeys)
|
||||
const tripped = [
|
||||
...breaches.filter((failure) => {
|
||||
const key = toleratedStreakKey(failure)
|
||||
return key === null ||
|
||||
(probeStreaks.get(key) ?? 0) > INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
}),
|
||||
...continuity.filter((failure) =>
|
||||
(continuityStreaks.get(continuityKey(failure)) ?? 0) >
|
||||
INCIDENT_FRESHNESS_TOLERANCE_SAMPLES)
|
||||
]
|
||||
if (tripped.length > 0) throw new PreDrainSampleTripped(tripped, samples)
|
||||
const windowDone = now() - startedMs >= windowMs
|
||||
if (windowDone && failures.length > 0) {
|
||||
overrunSamples += 1
|
||||
if (overrunSamples > PRE_DRAIN_MAX_OVERRUN_SAMPLES) {
|
||||
throw new PreDrainSampleTripped(failures, samples)
|
||||
}
|
||||
}
|
||||
if (windowDone && failures.length === 0) {
|
||||
return {
|
||||
windowMinutes: input.windowMinutes,
|
||||
startedAt: new Date(startedMs).toISOString(),
|
||||
completedAt: record.at,
|
||||
samples
|
||||
}
|
||||
}
|
||||
await wait(Math.max(0, sampleStartedMs + PRE_DRAIN_SAMPLE_INTERVAL_MS - now()))
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
import { readdirSync, readFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
RELAY_GITHUB_REPOSITORY,
|
||||
RELAY_WORKFLOW_FILE_PREFIX,
|
||||
relayRepositoryApiPath,
|
||||
relayWorkflowFile
|
||||
} from './relay-repository.js'
|
||||
|
||||
const sourceDir = fileURLToPath(new URL('.', import.meta.url))
|
||||
const sources = readdirSync(sourceDir)
|
||||
.filter((name) => name.endsWith('.ts') && name !== 'relay-repository.ts')
|
||||
.map((name) => ({ name, text: readFileSync(`${sourceDir}${name}`, 'utf8') }))
|
||||
|
||||
describe('relay repository identity', () => {
|
||||
it('builds API paths and workflow filenames from the one repository name', () => {
|
||||
expect(relayRepositoryApiPath('actions/runs')).toBe(`repos/${RELAY_GITHUB_REPOSITORY}/actions/runs`)
|
||||
expect(relayWorkflowFile('power-relay-staging.yml')).toBe(
|
||||
`${RELAY_WORKFLOW_FILE_PREFIX}power-relay-staging.yml`
|
||||
)
|
||||
})
|
||||
|
||||
// Why: the public-repo copy renames the repository and prefixes every workflow file. Both have to
|
||||
// be one edit, so no other module may restate either.
|
||||
it('is the only module naming a GitHub repository', () => {
|
||||
for (const { name, text } of sources) {
|
||||
expect(text, `${name} restates a GitHub repository`).not.toMatch(/stablyai\//)
|
||||
}
|
||||
})
|
||||
|
||||
it('is the only module naming a workflow file', () => {
|
||||
for (const { name, text } of sources) {
|
||||
for (const match of text.matchAll(/'([^']*\.yml)'/g)) {
|
||||
const file = match[1] ?? ''
|
||||
expect(text, `${name} names ${file} outside relayWorkflowFile`).toMatch(
|
||||
new RegExp(`relayWorkflowFile\\('${file.replaceAll('.', '\\.')}'\\)`)
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,14 +0,0 @@
|
||||
// Single place naming the GitHub repository that holds the Relay workflows. When the Relay tree is
|
||||
// copied to its public repository, only this file changes: the repository moves and every workflow
|
||||
// file gains a prefix, while the workflow display names stay as they are.
|
||||
export const RELAY_GITHUB_REPOSITORY = 'stablyai/orca-cloud'
|
||||
|
||||
export const RELAY_WORKFLOW_FILE_PREFIX = ''
|
||||
|
||||
export function relayWorkflowFile(name: string): string {
|
||||
return `${RELAY_WORKFLOW_FILE_PREFIX}${name}`
|
||||
}
|
||||
|
||||
export function relayRepositoryApiPath(resource: string): string {
|
||||
return `repos/${RELAY_GITHUB_REPOSITORY}/${resource}`
|
||||
}
|
||||
@@ -1,370 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js'
|
||||
import type { GcloudClient } from './gcloud-client.js'
|
||||
import { probeEndpointHealth, readResourceInventory } from './resource-inventory.js'
|
||||
|
||||
const digest = `sha256:${'a'.repeat(64)}`
|
||||
const runService = {
|
||||
template: {
|
||||
scaling: { minInstanceCount: 0, maxInstanceCount: 2 },
|
||||
containers: [{ image: 'registry/image:tag' }]
|
||||
},
|
||||
conditions: [{ state: 'CONDITION_SUCCEEDED' }],
|
||||
latestReadyRevision: 'projects/project/revisions/revision-one'
|
||||
}
|
||||
|
||||
const sleepingStagingGcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
|
||||
|
||||
// Staging's Cloud SQL is stopped, so this inventory reads REST only and probes no endpoint.
|
||||
type MigOutcome = 'ok' | 'throw' | 'missing'
|
||||
const sleepingStagingFetch = (migOutcome: (migName: string) => MigOutcome): typeof fetch =>
|
||||
async (input) => {
|
||||
const url = new URL(String(input))
|
||||
if (url.hostname === 'run.googleapis.com') return Response.json(runService)
|
||||
if (url.hostname === 'sqladmin.googleapis.com') return Response.json({
|
||||
state: 'STOPPED',
|
||||
databaseVersion: 'POSTGRES_17',
|
||||
settings: { activationPolicy: 'NEVER', availabilityType: 'ZONAL', tier: 'db-custom-1-3840' }
|
||||
})
|
||||
if (url.hostname === 'certificatemanager.googleapis.com') return Response.json({
|
||||
managed: { domains: ['*.relay-staging.onorca.dev'], state: 'ACTIVE' }
|
||||
})
|
||||
if (url.pathname.includes('/instanceGroupManagers/')) {
|
||||
const name = url.pathname.split('/').at(-1)!
|
||||
const outcome = migOutcome(name)
|
||||
if (outcome === 'throw') throw new TypeError('fetch failed')
|
||||
if (outcome === 'missing') return new Response(null, { status: 404 })
|
||||
return Response.json({
|
||||
name,
|
||||
targetSize: 0,
|
||||
size: '0',
|
||||
instanceGroup: `projects/project/zones/zone/instanceGroups/${name}`,
|
||||
instanceTemplate: `projects/project/global/instanceTemplates/template-${name}`,
|
||||
status: { isStable: true }
|
||||
})
|
||||
}
|
||||
if (url.pathname.includes('/instanceTemplates/')) return Response.json({ properties: {} })
|
||||
if (url.pathname.endsWith('/getHealth')) return Response.json([])
|
||||
throw new Error(`Unexpected request to ${url.hostname}${url.pathname}`)
|
||||
}
|
||||
|
||||
describe('readResourceInventory', () => {
|
||||
it('does not delay a healthy endpoint sample', async () => {
|
||||
let calls = 0
|
||||
let waits = 0
|
||||
const result = await probeEndpointHealth(
|
||||
'https://c9.relay.onorca.dev',
|
||||
async () => {
|
||||
calls += 1
|
||||
return new Response(null, { status: 200 })
|
||||
},
|
||||
{
|
||||
wait: async () => {
|
||||
waits += 1
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(true)
|
||||
expect(result.ready).toBe(true)
|
||||
expect(calls).toBe(2)
|
||||
expect(waits).toBe(0)
|
||||
})
|
||||
|
||||
it('retries one transient endpoint failure within the same sample', async () => {
|
||||
const calls = new Map<string, number>()
|
||||
const waits: number[] = []
|
||||
const result = await probeEndpointHealth(
|
||||
'https://c9.relay.onorca.dev',
|
||||
async (input) => {
|
||||
const path = new URL(String(input)).pathname
|
||||
const call = (calls.get(path) ?? 0) + 1
|
||||
calls.set(path, call)
|
||||
return new Response(null, { status: path === '/ready' && call === 1 ? 503 : 200 })
|
||||
},
|
||||
{
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(true)
|
||||
expect(result.ready).toBe(true)
|
||||
expect(calls).toEqual(new Map([['/health', 2], ['/ready', 2]]))
|
||||
expect(waits).toEqual([11_000])
|
||||
})
|
||||
|
||||
it('fails closed when the endpoint retry is also unhealthy', async () => {
|
||||
let calls = 0
|
||||
const waits: number[] = []
|
||||
const result = await probeEndpointHealth(
|
||||
'https://c9.relay.onorca.dev',
|
||||
async () => {
|
||||
calls += 1
|
||||
return new Response(null, { status: 503 })
|
||||
},
|
||||
{
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(false)
|
||||
expect(result.ready).toBe(false)
|
||||
expect(calls).toBe(4)
|
||||
// A refusing endpoint is a reading, so only the independent retry runs.
|
||||
expect(waits).toEqual([11_000])
|
||||
})
|
||||
|
||||
it('treats a thrown fetch as no reading and re-asks that path once', async () => {
|
||||
const calls: string[] = []
|
||||
const waits: number[] = []
|
||||
const result = await probeEndpointHealth(
|
||||
'https://c9.relay.onorca.dev',
|
||||
async (input) => {
|
||||
const path = new URL(String(input)).pathname
|
||||
calls.push(path)
|
||||
if (path === '/health' && calls.filter((call) => call === '/health').length === 1) {
|
||||
throw new TypeError('fetch failed')
|
||||
}
|
||||
return new Response(null, { status: 200 })
|
||||
},
|
||||
{
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(true)
|
||||
expect(result.ready).toBe(true)
|
||||
expect(calls.filter((call) => call === '/health')).toEqual(['/health', '/health'])
|
||||
expect(waits).toEqual([1_000])
|
||||
})
|
||||
|
||||
it('fails closed when both attempts of a path throw', async () => {
|
||||
const calls: string[] = []
|
||||
const waits: number[] = []
|
||||
const result = await probeEndpointHealth(
|
||||
'https://c9.relay.onorca.dev',
|
||||
async (input) => {
|
||||
const path = new URL(String(input)).pathname
|
||||
calls.push(path)
|
||||
if (path === '/health') throw new TypeError('fetch failed')
|
||||
return new Response(null, { status: 200 })
|
||||
},
|
||||
{
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(false)
|
||||
expect(calls.filter((call) => call === '/health')).toHaveLength(4)
|
||||
expect(waits).toEqual([1_000, 11_000, 1_000])
|
||||
})
|
||||
|
||||
it('accepts an auth-shaped endpoint that serves no readiness path', async () => {
|
||||
const calls: string[] = []
|
||||
const waits: number[] = []
|
||||
const result = await probeEndpointHealth(
|
||||
'https://login.onorca.dev',
|
||||
async (input) => {
|
||||
const path = new URL(String(input)).pathname
|
||||
calls.push(path)
|
||||
return new Response(null, { status: path === '/ready' ? 404 : 200 })
|
||||
},
|
||||
{
|
||||
requiresReady: false,
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(true)
|
||||
expect(result.ready).toBeNull()
|
||||
expect(calls).toEqual(['/health'])
|
||||
expect(waits).toEqual([])
|
||||
})
|
||||
|
||||
it('still requires readiness for the director and cells', async () => {
|
||||
const waits: number[] = []
|
||||
const result = await probeEndpointHealth(
|
||||
'https://relay.onorca.dev',
|
||||
async (input) => new Response(null, {
|
||||
status: new URL(String(input)).pathname === '/ready' ? 503 : 200
|
||||
}),
|
||||
{
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(true)
|
||||
expect(result.ready).toBe(false)
|
||||
expect(waits).toEqual([11_000])
|
||||
})
|
||||
|
||||
it('measures latency as the answering round trip, not the retry delay', async () => {
|
||||
let healthCalls = 0
|
||||
const result = await probeEndpointHealth(
|
||||
'https://c9.relay.onorca.dev',
|
||||
async (input) => {
|
||||
if (new URL(String(input)).pathname !== '/health') return new Response(null, { status: 200 })
|
||||
healthCalls += 1
|
||||
if (healthCalls === 1) throw new TypeError('fetch failed')
|
||||
return new Response(null, { status: 200 })
|
||||
},
|
||||
{ wait: async (ms) => await new Promise((resolve) => setTimeout(resolve, Math.min(ms, 60))) }
|
||||
)
|
||||
|
||||
expect(result.health).toBe(true)
|
||||
expect(result.latencyMs).not.toBeNull()
|
||||
expect(result.latencyMs!).toBeLessThan(60)
|
||||
})
|
||||
|
||||
it('uses aggregate REST inventory without probing sleeping staging endpoints', async () => {
|
||||
const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
|
||||
let publicProbeCalls = 0
|
||||
const fetchImpl: typeof fetch = async (input) => {
|
||||
const url = new URL(String(input))
|
||||
if (url.hostname.endsWith('onorca.dev')) {
|
||||
publicProbeCalls += 1
|
||||
return Response.json({ status: 'ok' })
|
||||
}
|
||||
if (url.hostname === 'run.googleapis.com') return Response.json(runService)
|
||||
if (url.hostname === 'sqladmin.googleapis.com') return Response.json({
|
||||
state: 'STOPPED',
|
||||
databaseVersion: 'POSTGRES_17',
|
||||
settings: {
|
||||
activationPolicy: 'NEVER',
|
||||
availabilityType: 'ZONAL',
|
||||
tier: 'db-custom-1-3840'
|
||||
}
|
||||
})
|
||||
if (url.hostname === 'certificatemanager.googleapis.com') return Response.json({
|
||||
managed: { domains: ['*.relay-staging.onorca.dev'], state: 'ACTIVE' }
|
||||
})
|
||||
if (url.pathname.includes('/instanceGroupManagers/')) {
|
||||
const name = url.pathname.split('/').at(-1)!
|
||||
return Response.json({
|
||||
name,
|
||||
targetSize: 0,
|
||||
size: '0',
|
||||
instanceGroup: `projects/project/zones/zone/instanceGroups/${name}`,
|
||||
instanceTemplate: `projects/project/global/instanceTemplates/template-${name}`,
|
||||
status: { isStable: true }
|
||||
})
|
||||
}
|
||||
if (url.pathname.includes('/instanceTemplates/')) return Response.json({
|
||||
properties: { metadata: { items: [{
|
||||
key: 'startup-script',
|
||||
value: `SECRET_TEXT\nORCA_RELAY_IMAGE_DIGEST=%s\\n' '${digest}'`
|
||||
}] } }
|
||||
})
|
||||
if (url.pathname.endsWith('/getHealth')) return Response.json([])
|
||||
throw new Error(`Unexpected request to ${url.hostname}${url.pathname}`)
|
||||
}
|
||||
|
||||
const result = await readResourceInventory(
|
||||
RELAY_OPS_ENVIRONMENTS.staging,
|
||||
gcloud,
|
||||
fetchImpl
|
||||
)
|
||||
|
||||
expect(publicProbeCalls).toBe(0)
|
||||
expect(result.cells.every((cell) => cell.targetSize === 0)).toBe(true)
|
||||
expect(result.cells.every((cell) => cell.endpoint.health === null)).toBe(true)
|
||||
expect(result.cells.every((cell) => cell.imageDigest === digest)).toBe(true)
|
||||
expect(JSON.stringify(result)).not.toContain('SECRET_TEXT')
|
||||
})
|
||||
|
||||
it('re-asks a MIG read that failed once before calling a cell powered-unknown', async () => {
|
||||
const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
|
||||
const waits: number[] = []
|
||||
let parkedMigCalls = 0
|
||||
const result = await readResourceInventory(
|
||||
RELAY_OPS_ENVIRONMENTS.staging,
|
||||
sleepingStagingGcloud,
|
||||
sleepingStagingFetch((migName) => {
|
||||
if (!migName.endsWith(parkedCell.hostname)) return 'ok'
|
||||
parkedMigCalls += 1
|
||||
return parkedMigCalls === 1 ? 'throw' : 'ok'
|
||||
}),
|
||||
{ wait: async (ms) => { waits.push(ms) } }
|
||||
)
|
||||
|
||||
const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)!
|
||||
// The MIG was fine and parked at zero; one transient read must not erase that reading.
|
||||
expect(parked.targetSize).toBe(0)
|
||||
expect(parkedMigCalls).toBe(2)
|
||||
expect(waits).toEqual([1_000])
|
||||
expect(result.warnings).toEqual([])
|
||||
})
|
||||
|
||||
it('reports a MIG unavailable only when the retry fails too', async () => {
|
||||
const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
|
||||
const waits: number[] = []
|
||||
let parkedMigCalls = 0
|
||||
const result = await readResourceInventory(
|
||||
RELAY_OPS_ENVIRONMENTS.staging,
|
||||
sleepingStagingGcloud,
|
||||
sleepingStagingFetch((migName) => {
|
||||
if (!migName.endsWith(parkedCell.hostname)) return 'ok'
|
||||
parkedMigCalls += 1
|
||||
return 'throw'
|
||||
}),
|
||||
{ wait: async (ms) => { waits.push(ms) } }
|
||||
)
|
||||
|
||||
const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)!
|
||||
expect(parked.targetSize).toBeNull()
|
||||
expect(parked.backendHealth).toBe('unknown')
|
||||
expect(parkedMigCalls).toBe(2)
|
||||
expect(waits).toEqual([1_000])
|
||||
expect(result.warnings).toEqual([
|
||||
`${parkedCell.hostname.toUpperCase()} MIG inventory is unavailable.`
|
||||
])
|
||||
})
|
||||
|
||||
it('does not re-ask a MIG read the API answered with 404', async () => {
|
||||
const missingCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
|
||||
const waits: number[] = []
|
||||
let missingMigCalls = 0
|
||||
const result = await readResourceInventory(
|
||||
RELAY_OPS_ENVIRONMENTS.staging,
|
||||
sleepingStagingGcloud,
|
||||
sleepingStagingFetch((migName) => {
|
||||
if (!migName.endsWith(missingCell.hostname)) return 'ok'
|
||||
missingMigCalls += 1
|
||||
return 'missing'
|
||||
}),
|
||||
{ wait: async (ms) => { waits.push(ms) } }
|
||||
)
|
||||
|
||||
expect(result.cells.find((cell) => cell.cellId === missingCell.cellId)!.targetSize).toBeNull()
|
||||
expect(missingMigCalls).toBe(1)
|
||||
expect(waits).toEqual([])
|
||||
})
|
||||
|
||||
it('represents missing credentials as unknown inventory, never sleeping', async () => {
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => { throw new Error('sensitive context') }
|
||||
}
|
||||
let fetchCalls = 0
|
||||
const result = await readResourceInventory(
|
||||
RELAY_OPS_ENVIRONMENTS.production,
|
||||
gcloud,
|
||||
async () => { fetchCalls += 1; return Response.json({}) }
|
||||
)
|
||||
expect(fetchCalls).toBe(0)
|
||||
expect(result.cells.every((cell) => cell.targetSize === null)).toBe(true)
|
||||
expect(result.cells.every((cell) => cell.backendHealth === 'unknown')).toBe(true)
|
||||
expect(JSON.stringify(result)).not.toContain('sensitive context')
|
||||
})
|
||||
})
|
||||
@@ -1,437 +0,0 @@
|
||||
import { z } from 'zod'
|
||||
import type { RelayOpsEnvironment, RelayOpsCellConfig } from './environment-config.js'
|
||||
import type { GcloudClient } from './gcloud-client.js'
|
||||
import { INCIDENT_MONITOR_THRESHOLDS } from './incident-monitor.js'
|
||||
|
||||
const RunServiceSchema = z.object({
|
||||
template: z.object({
|
||||
scaling: z.object({
|
||||
minInstanceCount: z.number().optional(),
|
||||
maxInstanceCount: z.number().optional()
|
||||
}).optional(),
|
||||
containers: z.array(z.object({ image: z.string() })).min(1)
|
||||
}),
|
||||
conditions: z.array(z.object({ state: z.string() })).default([]),
|
||||
latestReadyRevision: z.string().optional()
|
||||
})
|
||||
|
||||
const SqlInstanceSchema = z.object({
|
||||
state: z.string(),
|
||||
databaseVersion: z.string(),
|
||||
settings: z.object({
|
||||
activationPolicy: z.string(),
|
||||
availabilityType: z.string().optional(),
|
||||
tier: z.string()
|
||||
})
|
||||
})
|
||||
|
||||
const MigSchema = z.object({
|
||||
name: z.string(),
|
||||
targetSize: z.number(),
|
||||
size: z.union([z.string(), z.number()]).transform(Number).optional(),
|
||||
instanceGroup: z.string(),
|
||||
instanceTemplate: z.string(),
|
||||
status: z.object({ isStable: z.boolean().default(false) }).default({ isStable: false })
|
||||
})
|
||||
|
||||
const TemplateSchema = z.object({
|
||||
properties: z.object({
|
||||
metadata: z.object({
|
||||
items: z.array(z.object({ key: z.string(), value: z.string().optional() })).default([])
|
||||
}).optional()
|
||||
})
|
||||
})
|
||||
|
||||
const BackendHealthGroupSchema = z.object({
|
||||
healthStatus: z.array(z.object({ healthState: z.string() })).default([])
|
||||
})
|
||||
const BackendHealthSchema = z.union([
|
||||
BackendHealthGroupSchema,
|
||||
z.array(z.object({ status: BackendHealthGroupSchema }))
|
||||
])
|
||||
|
||||
const CertificateSchema = z.object({
|
||||
expireTime: z.string().optional(),
|
||||
managed: z.object({
|
||||
domains: z.array(z.string()).default([]),
|
||||
state: z.string()
|
||||
})
|
||||
})
|
||||
|
||||
export type EndpointHealth = {
|
||||
health: boolean | null
|
||||
ready: boolean | null
|
||||
latencyMs: number | null
|
||||
}
|
||||
|
||||
export type ServiceInventory = {
|
||||
ready: boolean
|
||||
revision: string | null
|
||||
image: string
|
||||
minInstances: number
|
||||
maxInstances: number
|
||||
}
|
||||
|
||||
export type CellInventory = RelayOpsCellConfig & {
|
||||
migName: string
|
||||
targetSize: number | null
|
||||
runningInstances: number | null
|
||||
stable: boolean | null
|
||||
template: string | null
|
||||
imageDigest: string | null
|
||||
backendHealth: 'healthy' | 'unhealthy' | 'empty' | 'unknown'
|
||||
endpoint: EndpointHealth
|
||||
}
|
||||
|
||||
export type ResourceInventory = {
|
||||
director: ServiceInventory | null
|
||||
auth: ServiceInventory | null
|
||||
sql: {
|
||||
state: string
|
||||
activationPolicy: string
|
||||
tier: string
|
||||
availabilityType: string
|
||||
databaseVersion: string
|
||||
} | null
|
||||
certificate: { state: string; domains: string[]; expireTime: string | null } | null
|
||||
directorEndpoint: EndpointHealth
|
||||
authEndpoint: EndpointHealth
|
||||
cells: CellInventory[]
|
||||
warnings: string[]
|
||||
}
|
||||
|
||||
const unavailableEndpoint = (): EndpointHealth => ({ health: null, ready: null, latencyMs: null })
|
||||
const independentEndpointRetryDelayMs = 11_000
|
||||
const transientProbeRetryDelayMs = 1_000
|
||||
const sleep = async (ms: number): Promise<void> =>
|
||||
await new Promise((resolvePromise) => setTimeout(resolvePromise, ms))
|
||||
|
||||
function finalSegment(value: string): string {
|
||||
return value.split('/').at(-1) ?? value
|
||||
}
|
||||
|
||||
function parseService(value: unknown): ServiceInventory {
|
||||
const service = RunServiceSchema.parse(value)
|
||||
return {
|
||||
ready: service.conditions.length > 0 && service.conditions.every(
|
||||
(condition) => condition.state === 'CONDITION_SUCCEEDED'
|
||||
),
|
||||
revision: service.latestReadyRevision ? finalSegment(service.latestReadyRevision) : null,
|
||||
image: service.template.containers[0]!.image,
|
||||
minInstances: service.template.scaling?.minInstanceCount ?? 0,
|
||||
maxInstances: service.template.scaling?.maxInstanceCount ?? 0
|
||||
}
|
||||
}
|
||||
|
||||
class GoogleApiError extends Error {
|
||||
constructor(readonly status: number) {
|
||||
super(`Google API returned ${status}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function googleRequest(
|
||||
fetchImpl: typeof fetch,
|
||||
token: string,
|
||||
url: string,
|
||||
init: RequestInit = {}
|
||||
): Promise<unknown> {
|
||||
const response = await fetchImpl(url, {
|
||||
...init,
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
...(init.body ? { 'content-type': 'application/json' } : {})
|
||||
},
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
if (!response.ok) throw new GoogleApiError(response.status)
|
||||
return await response.json()
|
||||
}
|
||||
|
||||
// A 404 is the API's answer about the resource; anything else is the absence of a reading, so re-ask.
|
||||
async function readOnceMore(
|
||||
read: () => Promise<unknown>,
|
||||
wait: (ms: number) => Promise<void>
|
||||
): Promise<unknown> {
|
||||
try {
|
||||
return await read()
|
||||
} catch (error) {
|
||||
if (error instanceof GoogleApiError && error.status === 404) throw error
|
||||
await wait(transientProbeRetryDelayMs)
|
||||
return await read()
|
||||
}
|
||||
}
|
||||
|
||||
// A reading the endpoint actually produced: ok is its answer, latencyMs is that answer's round trip.
|
||||
type PathReading = { ok: boolean; latencyMs: number | null }
|
||||
|
||||
async function probePath(
|
||||
origin: string,
|
||||
path: '/health' | '/ready',
|
||||
fetchImpl: typeof fetch,
|
||||
wait: (ms: number) => Promise<void>
|
||||
): Promise<PathReading> {
|
||||
// null means the request never produced an answer (DNS/TCP/TLS failure or the 8s abort).
|
||||
const attempt = async (): Promise<PathReading | null> => {
|
||||
const startedAt = performance.now()
|
||||
try {
|
||||
const response = await fetchImpl(`${origin}${path}`, {
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(8_000)
|
||||
})
|
||||
return { ok: response.ok, latencyMs: Math.round(performance.now() - startedAt) }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
const first = await attempt()
|
||||
if (first) return first
|
||||
// A thrown fetch is the absence of a reading, not an unhealthy answer, so re-ask before concluding.
|
||||
await wait(transientProbeRetryDelayMs)
|
||||
return (await attempt()) ?? { ok: false, latencyMs: null }
|
||||
}
|
||||
|
||||
async function endpointProbe(
|
||||
origin: string,
|
||||
fetchImpl: typeof fetch,
|
||||
requiresReady: boolean,
|
||||
wait: (ms: number) => Promise<void>
|
||||
): Promise<EndpointHealth> {
|
||||
const [health, ready] = await Promise.all([
|
||||
probePath(origin, '/health', fetchImpl, wait),
|
||||
requiresReady ? probePath(origin, '/ready', fetchImpl, wait) : null
|
||||
])
|
||||
// Latency is the slowest answering round trip in this probe; retry delays are not serving latency.
|
||||
const latencies = [health.latencyMs, ready?.latencyMs ?? null].filter(
|
||||
(value): value is number => value !== null
|
||||
)
|
||||
return {
|
||||
health: health.ok,
|
||||
ready: ready ? ready.ok : null,
|
||||
latencyMs: latencies.length > 0 ? Math.max(...latencies) : null
|
||||
}
|
||||
}
|
||||
|
||||
export type EndpointProbeOptions = {
|
||||
// Auth serves no /ready by design, so it is judged on /health and latency alone.
|
||||
requiresReady?: boolean
|
||||
wait?: (ms: number) => Promise<void>
|
||||
}
|
||||
|
||||
export async function probeEndpointHealth(
|
||||
origin: string,
|
||||
fetchImpl: typeof fetch,
|
||||
options: EndpointProbeOptions = {}
|
||||
): Promise<EndpointHealth> {
|
||||
const requiresReady = options.requiresReady ?? true
|
||||
const wait = options.wait ?? sleep
|
||||
const accepted = (probe: EndpointHealth): boolean =>
|
||||
probe.health === true &&
|
||||
(!requiresReady || probe.ready === true) &&
|
||||
probe.latencyMs !== null &&
|
||||
probe.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
|
||||
const first = await endpointProbe(origin, fetchImpl, requiresReady, wait)
|
||||
if (accepted(first)) return first
|
||||
// Outwait Relay's ten-second readiness cache before treating the retry as independent.
|
||||
await wait(independentEndpointRetryDelayMs)
|
||||
return await endpointProbe(origin, fetchImpl, requiresReady, wait)
|
||||
}
|
||||
|
||||
function imageDigest(template: z.infer<typeof TemplateSchema>): string | null {
|
||||
const startupScript = template.properties.metadata?.items.find(
|
||||
(item) => item.key === 'startup-script'
|
||||
)?.value
|
||||
// Return only the immutable digest; startup metadata contains secret names and operational detail.
|
||||
return startupScript?.match(/ORCA_RELAY_IMAGE_DIGEST=%s\\n' '(sha256:[a-f0-9]{64})'/)?.[1] ?? null
|
||||
}
|
||||
|
||||
function backendState(value: unknown): CellInventory['backendHealth'] {
|
||||
const parsedHealth = BackendHealthSchema.parse(value)
|
||||
const groups = Array.isArray(parsedHealth)
|
||||
? parsedHealth.map((group) => group.status)
|
||||
: [parsedHealth]
|
||||
const states = groups.flatMap((group) => group.healthStatus.map((status) => status.healthState))
|
||||
if (states.length === 0) return 'empty'
|
||||
if (states.every((state) => state === 'HEALTHY')) return 'healthy'
|
||||
return 'unhealthy'
|
||||
}
|
||||
|
||||
function unavailableCell(cell: RelayOpsCellConfig, migName: string): CellInventory {
|
||||
return {
|
||||
...cell,
|
||||
migName,
|
||||
targetSize: null,
|
||||
runningInstances: null,
|
||||
stable: null,
|
||||
template: null,
|
||||
imageDigest: null,
|
||||
backendHealth: 'unknown',
|
||||
endpoint: unavailableEndpoint()
|
||||
}
|
||||
}
|
||||
|
||||
async function readCell(
|
||||
environment: RelayOpsEnvironment,
|
||||
cell: RelayOpsCellConfig,
|
||||
mig: z.infer<typeof MigSchema> | null,
|
||||
token: string,
|
||||
fetchImpl: typeof fetch
|
||||
): Promise<CellInventory> {
|
||||
const migName = `${environment.migPrefix}${cell.hostname}`
|
||||
if (!mig) return unavailableCell(cell, migName)
|
||||
// An empty fixed-one MIG cannot serve and must never be woken by observation.
|
||||
const endpoint = mig.targetSize > 0
|
||||
? await probeEndpointHealth(cell.origin, fetchImpl)
|
||||
: unavailableEndpoint()
|
||||
const templateName = finalSegment(mig.instanceTemplate)
|
||||
const [templateResult, healthResult] = await Promise.allSettled([
|
||||
googleRequest(
|
||||
fetchImpl,
|
||||
token,
|
||||
`https://compute.googleapis.com/compute/v1/projects/${environment.project}/global/instanceTemplates/${templateName}`
|
||||
),
|
||||
googleRequest(
|
||||
fetchImpl,
|
||||
token,
|
||||
`https://compute.googleapis.com/compute/v1/projects/${environment.project}/global/backendServices/${migName}/getHealth`,
|
||||
{ method: 'POST', body: JSON.stringify({ group: mig.instanceGroup }) }
|
||||
)
|
||||
])
|
||||
return {
|
||||
...cell,
|
||||
migName,
|
||||
targetSize: mig.targetSize,
|
||||
runningInstances: mig.size ?? (mig.status.isStable ? mig.targetSize : null),
|
||||
stable: mig.status.isStable,
|
||||
template: templateName,
|
||||
imageDigest: templateResult.status === 'fulfilled'
|
||||
? imageDigest(TemplateSchema.parse(templateResult.value))
|
||||
: null,
|
||||
backendHealth: healthResult.status === 'fulfilled'
|
||||
? backendState(healthResult.value)
|
||||
: 'unknown',
|
||||
endpoint
|
||||
}
|
||||
}
|
||||
|
||||
function parsed<S extends z.ZodTypeAny>(
|
||||
result: PromiseSettledResult<unknown>,
|
||||
schema: S,
|
||||
warning: string,
|
||||
warnings: string[]
|
||||
): z.infer<S> | null {
|
||||
if (result.status === 'rejected') {
|
||||
warnings.push(warning)
|
||||
return null
|
||||
}
|
||||
const parsedValue = schema.safeParse(result.value)
|
||||
if (!parsedValue.success) {
|
||||
warnings.push(warning)
|
||||
return null
|
||||
}
|
||||
return parsedValue.data
|
||||
}
|
||||
|
||||
function unavailableInventory(environment: RelayOpsEnvironment, warning: string): ResourceInventory {
|
||||
return {
|
||||
director: null,
|
||||
auth: null,
|
||||
sql: null,
|
||||
certificate: null,
|
||||
directorEndpoint: unavailableEndpoint(),
|
||||
authEndpoint: unavailableEndpoint(),
|
||||
cells: environment.cells.map((cell) =>
|
||||
unavailableCell(cell, `${environment.migPrefix}${cell.hostname}`)
|
||||
),
|
||||
warnings: [warning]
|
||||
}
|
||||
}
|
||||
|
||||
export type ResourceInventoryOptions = {
|
||||
wait?: (ms: number) => Promise<void>
|
||||
}
|
||||
|
||||
export async function readResourceInventory(
|
||||
environment: RelayOpsEnvironment,
|
||||
gcloud: GcloudClient,
|
||||
fetchImpl: typeof fetch = fetch,
|
||||
options: ResourceInventoryOptions = {}
|
||||
): Promise<ResourceInventory> {
|
||||
const wait = options.wait ?? sleep
|
||||
let token: string
|
||||
try {
|
||||
token = await gcloud.accessToken()
|
||||
} catch {
|
||||
return unavailableInventory(
|
||||
environment,
|
||||
'Google Cloud credentials are unavailable. Run gcloud auth login.'
|
||||
)
|
||||
}
|
||||
const runUrl = (service: string) =>
|
||||
`https://run.googleapis.com/v2/projects/${environment.project}/locations/${environment.region}/services/${service}`
|
||||
const migUrl = (cell: RelayOpsCellConfig) =>
|
||||
`https://compute.googleapis.com/compute/v1/projects/${environment.project}/zones/${cell.zone}/instanceGroupManagers/${environment.migPrefix}${cell.hostname}`
|
||||
const settled = await Promise.allSettled([
|
||||
googleRequest(fetchImpl, token, runUrl(environment.directorService)),
|
||||
googleRequest(fetchImpl, token, runUrl(environment.authService)),
|
||||
googleRequest(
|
||||
fetchImpl,
|
||||
token,
|
||||
`https://sqladmin.googleapis.com/sql/v1beta4/projects/${environment.project}/instances/${environment.sqlInstance}`
|
||||
),
|
||||
googleRequest(
|
||||
fetchImpl,
|
||||
token,
|
||||
`https://certificatemanager.googleapis.com/v1/projects/${environment.project}/locations/global/certificates/${environment.certificateName}`
|
||||
),
|
||||
// One transient Compute read must never become a verdict on a cell's power state.
|
||||
...environment.cells.map((cell) =>
|
||||
readOnceMore(async () => await googleRequest(fetchImpl, token, migUrl(cell)), wait)
|
||||
)
|
||||
])
|
||||
const warnings: string[] = []
|
||||
const directorValue = parsed(settled[0]!, RunServiceSchema, 'Director service inventory is unavailable.', warnings)
|
||||
const authValue = parsed(settled[1]!, RunServiceSchema, 'Auth service inventory is unavailable.', warnings)
|
||||
const sqlValue = parsed(settled[2]!, SqlInstanceSchema, 'Cloud SQL inventory is unavailable.', warnings)
|
||||
const certificateValue = parsed(
|
||||
settled[3]!, CertificateSchema, 'TLS certificate inventory is unavailable.', warnings
|
||||
)
|
||||
const migValues = environment.cells.map((cell, index) => parsed(
|
||||
settled[index + 4]!,
|
||||
MigSchema,
|
||||
`${cell.hostname.toUpperCase()} MIG inventory is unavailable.`,
|
||||
warnings
|
||||
))
|
||||
const controlPlaneSleeping =
|
||||
environment.id === 'staging' && sqlValue?.settings.activationPolicy === 'NEVER'
|
||||
// Health probes would cold-start scale-to-zero Cloud Run services, so sleeping staging is inventory-only.
|
||||
const [directorEndpoint, authEndpoint] = controlPlaneSleeping
|
||||
? [unavailableEndpoint(), unavailableEndpoint()]
|
||||
: await Promise.all([
|
||||
probeEndpointHealth(environment.directorOrigin, fetchImpl),
|
||||
// The auth service exposes no /ready, so requiring it would fail every first probe.
|
||||
probeEndpointHealth(environment.authOrigin, fetchImpl, { requiresReady: false })
|
||||
])
|
||||
const cells = await Promise.all(environment.cells.map((cell, index) =>
|
||||
readCell(environment, cell, migValues[index] ?? null, token, fetchImpl)
|
||||
))
|
||||
return {
|
||||
director: directorValue ? parseService(directorValue) : null,
|
||||
auth: authValue ? parseService(authValue) : null,
|
||||
sql: sqlValue ? {
|
||||
state: sqlValue.state,
|
||||
activationPolicy: sqlValue.settings.activationPolicy,
|
||||
tier: sqlValue.settings.tier,
|
||||
availabilityType: sqlValue.settings.availabilityType ?? 'unknown',
|
||||
databaseVersion: sqlValue.databaseVersion
|
||||
} : null,
|
||||
certificate: certificateValue ? {
|
||||
state: certificateValue.managed.state,
|
||||
domains: certificateValue.managed.domains,
|
||||
expireTime: certificateValue.expireTime ?? null
|
||||
} : null,
|
||||
directorEndpoint,
|
||||
authEndpoint,
|
||||
cells,
|
||||
warnings
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { parseStagingPowerRequest } from './staging-workflow.js'
|
||||
|
||||
describe('parseStagingPowerRequest', () => {
|
||||
it('accepts the exact reviewed confirmations', () => {
|
||||
expect(parseStagingPowerRequest({ mode: 'status', confirmation: '' })).toEqual({
|
||||
mode: 'status', confirmation: ''
|
||||
})
|
||||
expect(parseStagingPowerRequest({ mode: 'wake', confirmation: 'WAKE_STAGING' }).mode).toBe('wake')
|
||||
expect(parseStagingPowerRequest({ mode: 'sleep', confirmation: 'SLEEP_STAGING' }).mode).toBe('sleep')
|
||||
})
|
||||
|
||||
it('rejects missing, swapped, or additional fields', () => {
|
||||
expect(() => parseStagingPowerRequest({ mode: 'wake', confirmation: '' })).toThrow()
|
||||
expect(() => parseStagingPowerRequest({ mode: 'sleep', confirmation: 'WAKE_STAGING' })).toThrow()
|
||||
expect(() => parseStagingPowerRequest({ mode: 'production', confirmation: '' })).toThrow()
|
||||
expect(() => parseStagingPowerRequest({ mode: 'status', confirmation: '', project: 'other' })).toThrow()
|
||||
})
|
||||
})
|
||||
@@ -1,36 +0,0 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import { promisify } from 'node:util'
|
||||
import { z } from 'zod'
|
||||
import { RELAY_GITHUB_REPOSITORY, relayWorkflowFile } from './relay-repository.js'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const DispatchSchema = z.discriminatedUnion('mode', [
|
||||
z.object({ mode: z.literal('status'), confirmation: z.literal('') }).strict(),
|
||||
z.object({ mode: z.literal('wake'), confirmation: z.literal('WAKE_STAGING') }).strict(),
|
||||
z.object({ mode: z.literal('sleep'), confirmation: z.literal('SLEEP_STAGING') }).strict()
|
||||
])
|
||||
|
||||
export type StagingPowerRequest = z.infer<typeof DispatchSchema>
|
||||
|
||||
export function parseStagingPowerRequest(value: unknown): StagingPowerRequest {
|
||||
return DispatchSchema.parse(value)
|
||||
}
|
||||
|
||||
export async function dispatchStagingPowerWorkflow(request: StagingPowerRequest): Promise<void> {
|
||||
const args = [
|
||||
'workflow', 'run', relayWorkflowFile('power-relay-staging.yml'),
|
||||
'--repo', RELAY_GITHUB_REPOSITORY,
|
||||
'-f', `mode=${request.mode}`,
|
||||
'-f', 'wake-cells=configured'
|
||||
]
|
||||
if (request.confirmation) args.push('-f', `confirmation=${request.confirmation}`)
|
||||
try {
|
||||
await execFileAsync('gh', args, {
|
||||
encoding: 'utf8',
|
||||
timeout: 30_000,
|
||||
maxBuffer: 1024 * 1024
|
||||
})
|
||||
} catch {
|
||||
throw new Error('Staging power workflow dispatch failed')
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"exclude": ["src/**/*.test.ts"]
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "src",
|
||||
"outDir": "dist",
|
||||
"types": ["node", "vitest"],
|
||||
"exactOptionalPropertyTypes": true,
|
||||
"verbatimModuleSyntax": true
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
isRelayCellConnectionHardCap,
|
||||
RELAY_ADMISSION_BUDGETS,
|
||||
RELAY_CELL_ADMISSION_BOUNDS,
|
||||
RELAY_CELL_CONNECTION_HARD_CAP,
|
||||
RELAY_CELL_CONNECTION_HARD_CAPS,
|
||||
relayCellAdmissionBounds
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
// Why: dev scripts run standalone in CI and Terraform cannot read TypeScript, so neither can
|
||||
// import the constant. Both restate it instead. This asserts every restatement still agrees, so
|
||||
// raising the cap fails loudly here rather than silently leaving a surface behind.
|
||||
// Repository root, and every path read below stays inside the Relay tree, so this survives the
|
||||
// move under cloud/ in the public repository.
|
||||
const repositoryRoot = fileURLToPath(new URL('../../../', import.meta.url))
|
||||
const read = (path: string): string => readFileSync(`${repositoryRoot}${path}`, 'utf8')
|
||||
|
||||
describe('cell connection hard cap stays consistent across surfaces that cannot import it', () => {
|
||||
it('derives its own bounds from the cap', () => {
|
||||
expect(RELAY_CELL_ADMISSION_BOUNDS.hardCap).toBe(RELAY_CELL_CONNECTION_HARD_CAP)
|
||||
expect(RELAY_CELL_ADMISSION_BOUNDS.socketAdmissionCeiling).toBe(
|
||||
RELAY_CELL_CONNECTION_HARD_CAP - RELAY_ADMISSION_BUDGETS.reservedHostControls
|
||||
)
|
||||
expect(RELAY_CELL_ADMISSION_BOUNDS.maxUnobservedBound).toBe(
|
||||
RELAY_CELL_ADMISSION_BOUNDS.socketAdmissionCeiling - 1
|
||||
)
|
||||
for (const hardCap of RELAY_CELL_CONNECTION_HARD_CAPS) {
|
||||
const bounds = relayCellAdmissionBounds(hardCap)
|
||||
expect(bounds.socketAdmissionCeiling).toBe(
|
||||
hardCap - RELAY_ADMISSION_BUDGETS.reservedHostControls
|
||||
)
|
||||
expect(bounds.maxUnobservedBound).toBe(bounds.socketAdmissionCeiling - 1)
|
||||
}
|
||||
})
|
||||
|
||||
it.each([
|
||||
['dev/scripts/relay-recovery-wave-gate.mjs', /targetConnectionCap:\s*(\d+)/],
|
||||
['dev/scripts/deploy-relay-gce-multi-target.mjs', /DEFAULT_CONNECTION_CEILING\s*=\s*(\d+)/],
|
||||
['dev/scripts/deploy-relay-gce-multi-target.mjs', /CUTOVER_CONNECTION_HARD_CAP\s*=\s*(\d+)/]
|
||||
])('%s matches the contract cap', (path, pattern) => {
|
||||
const match = read(path).match(pattern)
|
||||
expect(match, `${path} no longer declares ${pattern}`).not.toBeNull()
|
||||
expect(Number(match![1])).toBe(RELAY_CELL_CONNECTION_HARD_CAP)
|
||||
})
|
||||
|
||||
it('every production cell declaring a cap uses a supported contract cap', () => {
|
||||
const declared = [
|
||||
...read('infra/terraform/environments/production.tfvars').matchAll(
|
||||
/connection_hard_cap\s*=\s*(\d+)/g
|
||||
)
|
||||
].map((match) => Number(match[1]))
|
||||
expect(declared.length).toBeGreaterThan(0)
|
||||
expect(new Set(declared).has(RELAY_CELL_CONNECTION_HARD_CAP)).toBe(true)
|
||||
expect(declared.every((hardCap) => isRelayCellConnectionHardCap(hardCap))).toBe(true)
|
||||
})
|
||||
|
||||
it('the Terraform cell validation accepts exactly the supported contract caps', () => {
|
||||
const match = read('infra/terraform/variables.tf').match(
|
||||
/contains\(\[([^\]]+)\], cell\.connection_hard_cap\)/
|
||||
)
|
||||
expect(match, 'variables.tf no longer validates connection_hard_cap').not.toBeNull()
|
||||
expect(match![1]!.split(',').map((value) => Number(value.trim()))).toEqual([
|
||||
...RELAY_CELL_CONNECTION_HARD_CAPS
|
||||
])
|
||||
})
|
||||
|
||||
it('the Terraform unobserved bound leaves the contract rebind reserve', () => {
|
||||
expect(read('infra/terraform/variables.tf')).toContain(
|
||||
'cell.connection_unobserved_bound < cell.connection_hard_cap - 100'
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -1,55 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { loadRelayConfig } from './config.js'
|
||||
|
||||
// A same-cap roll swaps only the image: the cell keeps the env its startup script already
|
||||
// wrote. So every image must boot on exactly the variables that script renders, and a
|
||||
// config change that needs a new variable is a template change, not an image-only roll.
|
||||
const template = readFileSync(
|
||||
new URL('../../../infra/terraform/relay-gce-startup.sh.tftpl', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
// One plausible production value per variable the template can write.
|
||||
const RENDERED: Record<string, string> = {
|
||||
DATABASE_URL: 'postgres://relay@127.0.0.1:5432/orca_relay',
|
||||
ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: 'assignment-key-with-at-least-thirty-two-bytes',
|
||||
ORCA_RELAY_PUBLIC_URL: 'https://c25.relay.onorca.dev',
|
||||
ORCA_RELAY_CELL_URL: 'https://c25.relay.onorca.dev',
|
||||
ORCA_RELAY_AUTH_ISSUER: 'https://auth.onorca.dev',
|
||||
ORCA_RELAY_AUTH_AUDIENCE: 'orca-relay',
|
||||
ORCA_RELAY_JWKS_URL: 'https://auth.onorca.dev/.well-known/jwks.json',
|
||||
ORCA_RELAY_ROLE: 'cell',
|
||||
ORCA_RELAY_CELL_ID: 'production-gce-c25',
|
||||
ORCA_RELAY_REGION: 'asia-east2',
|
||||
ORCA_RELAY_CELL_CAPACITY: '3000',
|
||||
ORCA_RELAY_DATABASE_POOL_MAX: '16',
|
||||
ORCA_RELAY_CELL_CONNECTION_HARD_CAP: '3000',
|
||||
ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND: '60',
|
||||
ORCA_RELAY_CELLS_JSON: '[]',
|
||||
ORCA_RELAY_ADMIN_AUDIENCE: 'https://relay.onorca.dev/v1/admin/drain',
|
||||
ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.iam.gserviceaccount.com',
|
||||
ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT: 'capacity@example.iam.gserviceaccount.com',
|
||||
ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT: 'asia-proof@example.iam.gserviceaccount.com',
|
||||
ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT: 'relay-cell@example.iam.gserviceaccount.com',
|
||||
ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT: 'relay-director@example.iam.gserviceaccount.com',
|
||||
ORCA_RELAY_REHOME_AUDIENCE: 'https://relay.onorca.dev/v1/admin/host-drain',
|
||||
ORCA_RELAY_DIRECTOR_URL: 'https://relay.onorca.dev',
|
||||
ORCA_RELAY_HEARTBEAT_AUDIENCE: 'https://relay.onorca.dev/v1/admin/cell-heartbeat',
|
||||
ORCA_RELAY_IMAGE_DIGEST: `sha256:${'a'.repeat(64)}`
|
||||
}
|
||||
|
||||
const TEMPLATE_VARIABLES = [...template.matchAll(/printf '([A-Z0-9_]+)=/g)].map(([, name]) => name!)
|
||||
|
||||
describe('cell startup environment', () => {
|
||||
it('boots a cell on exactly the variables the startup template writes', () => {
|
||||
expect(TEMPLATE_VARIABLES.length).toBeGreaterThan(20)
|
||||
expect(TEMPLATE_VARIABLES.filter((name) => RENDERED[name] === undefined)).toEqual([])
|
||||
const env = Object.fromEntries(TEMPLATE_VARIABLES.map((name) => [name, RENDERED[name]]))
|
||||
expect(loadRelayConfig(env)).toMatchObject({
|
||||
role: 'cell',
|
||||
cellId: 'production-gce-c25',
|
||||
databaseUrl: RENDERED.DATABASE_URL
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,163 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
formatAssignmentInventorySnapshot,
|
||||
type AssignmentInventorySnapshot
|
||||
} from './assignment-inventory-snapshot.js'
|
||||
|
||||
// The enable workflow reads the director's rehome inventory line out of Cloud
|
||||
// Logging with a parser that lives in another language, in another package, and
|
||||
// is never exercised against the formatter that writes the line. When
|
||||
// `hostNotArrivedLast24Hours` shipped, the parser read a healthy line as no
|
||||
// evidence at all and the run failed closed, disabling the durable switch. This
|
||||
// test is the missing edge: the real formatter's output, through the real
|
||||
// parser, so a future field fails here instead of in an operator's run.
|
||||
|
||||
type InventoryEvidence = {
|
||||
active: number
|
||||
awaitingReceipt: number
|
||||
targetRegistered: number
|
||||
completedLast24Hours: number
|
||||
abortedLast24Hours: number
|
||||
hostNotArrivedLast24Hours: number | null
|
||||
oldestActiveAgeMs: number | null
|
||||
}
|
||||
|
||||
// Imported through a computed URL on purpose: the script is plain ESM outside
|
||||
// this package's compile scope, so a static import would not resolve.
|
||||
async function loadParser(): Promise<(entries: unknown[], options: unknown) => InventoryEvidence> {
|
||||
const source = new URL(
|
||||
'../../../dev/scripts/relay-rehome-aggregate-evidence.mjs',
|
||||
import.meta.url
|
||||
).href
|
||||
const loaded: unknown = await import(/* @vite-ignore */ source)
|
||||
if (!(loaded !== null && typeof loaded === 'object' && 'parseRegionalRehomeInventory' in loaded)) {
|
||||
throw new Error('relay-rehome-aggregate-evidence.mjs no longer exports its parser')
|
||||
}
|
||||
const parse = loaded.parseRegionalRehomeInventory
|
||||
if (typeof parse !== 'function') throw new Error('parseRegionalRehomeInventory is not callable')
|
||||
return (entries, options) => readEvidence(parse(entries, options))
|
||||
}
|
||||
|
||||
function readEvidence(value: unknown): InventoryEvidence {
|
||||
if (value === null || typeof value !== 'object') throw new Error('parser returned no evidence')
|
||||
const counts = ['active', 'awaitingReceipt', 'targetRegistered', 'completedLast24Hours', 'abortedLast24Hours'] as const
|
||||
const evidence: Record<string, number | null> = {}
|
||||
for (const key of [...counts, 'hostNotArrivedLast24Hours', 'oldestActiveAgeMs'] as const) {
|
||||
if (!(key in value)) throw new Error(`parser dropped ${key}`)
|
||||
const read: unknown = Reflect.get(value, key)
|
||||
if (read !== null && typeof read !== 'number') throw new Error(`${key} is not a count`)
|
||||
evidence[key] = read
|
||||
}
|
||||
for (const key of counts) {
|
||||
if (evidence[key] === null) throw new Error(`${key} must be a number`)
|
||||
}
|
||||
return {
|
||||
active: Number(evidence['active']),
|
||||
awaitingReceipt: Number(evidence['awaitingReceipt']),
|
||||
targetRegistered: Number(evidence['targetRegistered']),
|
||||
completedLast24Hours: Number(evidence['completedLast24Hours']),
|
||||
abortedLast24Hours: Number(evidence['abortedLast24Hours']),
|
||||
hostNotArrivedLast24Hours: evidence['hostNotArrivedLast24Hours'] ?? null,
|
||||
oldestActiveAgeMs: evidence['oldestActiveAgeMs'] ?? null
|
||||
}
|
||||
}
|
||||
|
||||
function snapshot(
|
||||
regionalRehomes: AssignmentInventorySnapshot['regionalRehomes']
|
||||
): AssignmentInventorySnapshot {
|
||||
return {
|
||||
cells: [],
|
||||
activityLeases: { total: 0, expired: 0, requestUnits: 0 },
|
||||
connectionReservations: { outstanding: 0, lateArrivalDebt: 0 },
|
||||
regionalRehomes
|
||||
}
|
||||
}
|
||||
|
||||
function inventoryLine(snapshotValue: AssignmentInventorySnapshot): string {
|
||||
const line = formatAssignmentInventorySnapshot(snapshotValue).find((candidate) =>
|
||||
candidate.startsWith('[orca-relay] regional rehome inventory ')
|
||||
)
|
||||
if (!line) throw new Error('the formatter no longer emits a rehome inventory line')
|
||||
return line
|
||||
}
|
||||
|
||||
describe('regional rehome inventory line census', () => {
|
||||
it('parses what the director actually prints, field for field', async () => {
|
||||
const parse = await loadParser()
|
||||
const regionalRehomes = {
|
||||
active: 3,
|
||||
awaitingReceipt: 1,
|
||||
targetRegistered: 2,
|
||||
completedLast24Hours: 41,
|
||||
abortedLast24Hours: 12,
|
||||
hostNotArrivedLast24Hours: 5,
|
||||
oldestActiveAgeMs: 77_731_209
|
||||
}
|
||||
const now = Date.parse('2026-09-20T12:00:00Z')
|
||||
|
||||
const evidence = parse(
|
||||
[{ timestamp: '2026-09-20T11:59:00Z', textPayload: inventoryLine(snapshot(regionalRehomes)) }],
|
||||
{ now, maxAgeMs: 5 * 60_000 }
|
||||
)
|
||||
|
||||
expect(evidence).toEqual({ ...regionalRehomes })
|
||||
})
|
||||
|
||||
it('parses the line an idle fleet prints, with no oldest active age', async () => {
|
||||
const parse = await loadParser()
|
||||
const now = Date.parse('2026-09-20T12:00:00Z')
|
||||
|
||||
const evidence = parse(
|
||||
[
|
||||
{
|
||||
timestamp: '2026-09-20T11:59:00Z',
|
||||
textPayload: inventoryLine(
|
||||
snapshot({
|
||||
active: 0,
|
||||
awaitingReceipt: 0,
|
||||
targetRegistered: 0,
|
||||
completedLast24Hours: 0,
|
||||
abortedLast24Hours: 0,
|
||||
hostNotArrivedLast24Hours: 0,
|
||||
oldestActiveAgeMs: null
|
||||
})
|
||||
)
|
||||
}
|
||||
],
|
||||
{ now, maxAgeMs: 5 * 60_000 }
|
||||
)
|
||||
|
||||
expect(evidence.oldestActiveAgeMs).toBeNull()
|
||||
expect(evidence.hostNotArrivedLast24Hours).toBe(0)
|
||||
})
|
||||
|
||||
it('covers every counter the formatter puts on the line', async () => {
|
||||
const parse = await loadParser()
|
||||
// A field the parser ignores is a field the operator never sees, so the
|
||||
// census fails when the formatter gains one and this test is not updated.
|
||||
const line = inventoryLine(
|
||||
snapshot({
|
||||
active: 1,
|
||||
awaitingReceipt: 1,
|
||||
targetRegistered: 1,
|
||||
completedLast24Hours: 1,
|
||||
abortedLast24Hours: 1,
|
||||
hostNotArrivedLast24Hours: 1,
|
||||
oldestActiveAgeMs: 1
|
||||
})
|
||||
)
|
||||
const printed = line
|
||||
.slice('[orca-relay] regional rehome inventory '.length)
|
||||
.split(' ')
|
||||
.map((field) => field.split('=')[0])
|
||||
|
||||
const surfaced = Object.keys(
|
||||
parse([{ timestamp: '2026-09-20T11:59:00Z', textPayload: line }], {
|
||||
now: Date.parse('2026-09-20T12:00:00Z'),
|
||||
maxAgeMs: 5 * 60_000
|
||||
})
|
||||
)
|
||||
|
||||
expect([...printed].sort()).toEqual([...surfaced].sort())
|
||||
})
|
||||
})
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"comment": "Production Cloud SQL consumers owned by the private orca-cloud application tree (auth and API services). The relay ships without them, so the values the connection budget needs are published here; the private repository binds every field back to its source in its own CI.",
|
||||
"authInstances": 2,
|
||||
"authPoolMax": 10,
|
||||
"apiInstances": 10,
|
||||
"apiPoolMax": 5,
|
||||
"maxConnections": 500,
|
||||
"sources": {
|
||||
"authInstances": "private apps tfvars: auth service max instances",
|
||||
"authPoolMax": "private auth service: pg.Pool max",
|
||||
"apiInstances": "private apps tfvars: API service max instances",
|
||||
"apiPoolMax": "private API service: pg.Pool max",
|
||||
"maxConnections": "measured SHOW max_connections = 500 on the live instance 2026-09-16; no max_connections flag is set, so this is the tier default and the previous 400 was an unverified assumption about it"
|
||||
}
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
{
|
||||
"comment": "Production 2026-10-21 07:44-07:58Z, read-only: Cloud Run director 503s per minute (request_count, code 503) and every director runtime-metrics line, instance ids replaced. d5 stalled on its database 07:55-07:57.",
|
||||
"fleet503PerMinute": {"2026-10-21T07:44":6,"2026-10-21T07:45":7,"2026-10-21T07:46":5,"2026-10-21T07:47":55,"2026-10-21T07:48":11,"2026-10-21T07:49":10,"2026-10-21T07:50":12,"2026-10-21T07:51":4,"2026-10-21T07:52":6,"2026-10-21T07:53":8,"2026-10-21T07:54":6,"2026-10-21T07:55":57,"2026-10-21T07:56":173,"2026-10-21T07:57":45},
|
||||
"directorMetricsLines": [
|
||||
{"timestamp":"2026-10-21T07:44:05.179655Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:44:08.062180Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-in-flight":1,"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":3}},
|
||||
{"timestamp":"2026-10-21T07:44:13.822472Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"relay_assignment_row_busy":1}},
|
||||
{"timestamp":"2026-10-21T07:44:14.159472Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:44:25.765825Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:44:35.181579Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:44:38.060162Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:44:43.823368Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:44:44.158999Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:44:55.766172Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:45:05.180654Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:45:08.060467Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:45:13.824498Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:45:14.159451Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:45:25.764958Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"placementRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:45:35.181560Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:45:38.062560Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:45:43.831975Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:45:44.160118Z","instance":"d4","placementRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:45:55.767508Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:46:05.181410Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:46:08.061944Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":3},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":3}},
|
||||
{"timestamp":"2026-10-21T07:46:13.824904Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:46:14.160287Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:46:25.766494Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:46:35.180987Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:46:38.061343Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:46:43.824785Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:46:44.161722Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:46:55.768452Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:47:05.182855Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:47:08.066053Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:47:13.825495Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":3},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":3}},
|
||||
{"timestamp":"2026-10-21T07:47:14.161928Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"relay_assignment_row_busy":1,"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:47:25.767488Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":4},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":4}},
|
||||
{"timestamp":"2026-10-21T07:47:35.183676Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:47:38.064515Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:47:43.827753Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-in-flight":1,"host-rate-limited":1,"wait-timeout":9},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":11}},
|
||||
{"timestamp":"2026-10-21T07:47:44.161502Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-in-flight":1,"wait-timeout":7},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":8}},
|
||||
{"timestamp":"2026-10-21T07:47:55.769396Z","instance":"d5","stickyRejectionsByReasonDelta":{"wait-timeout":9},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":9}},
|
||||
{"timestamp":"2026-10-21T07:48:05.184152Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1,"wait-timeout":10},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":11}},
|
||||
{"timestamp":"2026-10-21T07:48:08.069122Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:48:13.826317Z","instance":"d3","stickyRejectionsByReasonDelta":{"wait-timeout":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:48:14.162388Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1,"wait-timeout":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:48:25.770092Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:48:35.183997Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:48:38.072360Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:48:43.826028Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:48:44.162106Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:48:55.769497Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:49:05.183794Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:49:08.070369Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:49:13.826477Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:49:14.162921Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"relay_assignment_row_busy":1,"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:49:25.771495Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:49:35.183472Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:49:38.072836Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:49:43.826252Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:49:44.163115Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:49:55.769846Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:50:05.184917Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:50:08.072488Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:50:13.827130Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":3},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":3}},
|
||||
{"timestamp":"2026-10-21T07:50:14.163400Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:50:25.769945Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:50:35.184832Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:50:38.075017Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:50:43.827931Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:50:44.164604Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:50:55.772344Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":3},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":3}},
|
||||
{"timestamp":"2026-10-21T07:51:05.184087Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"relay_assignment_row_busy":1}},
|
||||
{"timestamp":"2026-10-21T07:51:08.074651Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:51:13.829283Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:51:14.165434Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:51:25.769048Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:51:35.183654Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:51:38.073759Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:51:43.829359Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:51:44.166698Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:51:55.770350Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:52:05.184492Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:52:08.076738Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:52:13.830929Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:52:14.167578Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:52:25.770188Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:52:35.185557Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:52:38.073056Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:52:43.830612Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:52:44.170744Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:52:55.770072Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:53:05.185184Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:53:08.073799Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:53:13.831126Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:53:14.168954Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:53:25.771642Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:53:35.185375Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:53:38.073871Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:53:43.832470Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:53:44.169801Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:53:55.772778Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:54:05.184779Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:54:08.075259Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:54:13.831013Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:54:14.170568Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:54:25.772879Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:54:35.185747Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:54:38.074242Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:54:43.832593Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:54:44.170223Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:54:55.772796Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:55:05.186732Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:55:08.074506Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:55:13.832282Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:55:14.170168Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:55:25.773007Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:55:35.186844Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:55:38.073584Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"placementRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:55:43.832666Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:55:44.171703Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:55:55.774228Z","instance":"d5","stickyRejectionsByReasonDelta":{"wait-timeout":21},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":21}},
|
||||
{"timestamp":"2026-10-21T07:56:05.186316Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":4},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":4}},
|
||||
{"timestamp":"2026-10-21T07:56:08.074683Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:56:13.836910Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:56:14.171278Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:56:25.774220Z","instance":"d5","stickyRejectionsByReasonDelta":{"wait-timeout":132},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":132}},
|
||||
{"timestamp":"2026-10-21T07:56:35.187168Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:56:38.075749Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:56:43.834101Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":4},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":4}},
|
||||
{"timestamp":"2026-10-21T07:56:44.171918Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:56:55.775731Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1,"wait-timeout":56},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":57}},
|
||||
{"timestamp":"2026-10-21T07:57:05.187236Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:57:08.074119Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:57:13.833738Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:57:14.171816Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:57:25.775798Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:57:35.187493Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}},
|
||||
{"timestamp":"2026-10-21T07:57:38.075303Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:57:43.833226Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}},
|
||||
{"timestamp":"2026-10-21T07:57:44.170780Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0},
|
||||
{"timestamp":"2026-10-21T07:57:55.775393Z","instance":"d5","stickyRejectionsByReasonDelta":{"wait-timeout":5},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":5}},
|
||||
{"timestamp":"2026-10-21T07:58:05.187113Z","instance":"d1","stickyRejectionsByReasonDelta":{"wait-timeout":9},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":9}},
|
||||
{"timestamp":"2026-10-21T07:58:08.073793Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1,"wait-timeout":3},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":4}},
|
||||
{"timestamp":"2026-10-21T07:58:13.849059Z","instance":"d3","stickyRejectionsByReasonDelta":{"wait-timeout":11},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":11}},
|
||||
{"timestamp":"2026-10-21T07:58:14.171116Z","instance":"d4","stickyRejectionsByReasonDelta":{"wait-timeout":13},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":13}},
|
||||
{"timestamp":"2026-10-21T07:58:25.776011Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}
|
||||
]
|
||||
}
|
||||
@@ -1,339 +0,0 @@
|
||||
{
|
||||
"comment": "Terraform root ownership for every resource family declared under infra/terraform (relay), infra/terraform-foundation, and infra/terraform-apps. env_conditional families live in relay for production and in apps for staging (complementary counts). never_in_relay_state lists foundation/apps families that were introduced with their new root and so must NOT appear in infra/terraform/relay-root-carve-removed.tf. Generated by dev/scripts/terraform-root-partition.mjs --write; the test asserts this file matches the declarations.",
|
||||
"foundation": [
|
||||
"google_artifact_registry_repository.api",
|
||||
"google_iam_workload_identity_pool.github",
|
||||
"google_project_service.required",
|
||||
"google_project_service.sqladmin",
|
||||
"google_service_account.runtime",
|
||||
"google_sql_database_instance.auth",
|
||||
"google_storage_bucket_iam_member.cloud_sql_rollout_lease",
|
||||
"google_storage_bucket_iam_member.cloud_sql_rollout_lease_bucket_reader"
|
||||
],
|
||||
"apps": [
|
||||
"cloudflare_record.artifact_onorca",
|
||||
"cloudflare_record.artifact_usercontent",
|
||||
"cloudflare_record.auth",
|
||||
"google_artifact_registry_repository_iam_member.github_production_app_deploy_writer",
|
||||
"google_cloud_run_domain_mapping.artifacts",
|
||||
"google_cloud_run_domain_mapping.auth",
|
||||
"google_cloud_run_v2_job.skill_storage_monitor",
|
||||
"google_cloud_run_v2_job_iam_member.github_production_app_skill_monitor_developer",
|
||||
"google_cloud_run_v2_job_iam_member.skill_storage_scheduler_invoker",
|
||||
"google_cloud_run_v2_service.api",
|
||||
"google_cloud_run_v2_service.auth",
|
||||
"google_cloud_run_v2_service_iam_member.github_production_app_api_developer",
|
||||
"google_cloud_run_v2_service_iam_member.github_production_app_auth_developer",
|
||||
"google_cloud_scheduler_job.skill_storage_monitor",
|
||||
"google_iam_workload_identity_pool_provider.github_production_app_deploy",
|
||||
"google_logging_metric.skill_api",
|
||||
"google_logging_metric.skill_archive_rejection",
|
||||
"google_logging_metric.skill_database",
|
||||
"google_logging_metric.skill_digest_mismatch",
|
||||
"google_logging_metric.skill_finalize_saturation",
|
||||
"google_logging_metric.skill_route_latency",
|
||||
"google_logging_metric.skill_signing_failure",
|
||||
"google_logging_metric.skill_storage_inventory",
|
||||
"google_logging_metric.skill_storage_overdue_quarantine",
|
||||
"google_logging_project_exclusion.skill_share_bearer_request_urls",
|
||||
"google_monitoring_alert_policy.skill_api_5xx",
|
||||
"google_monitoring_alert_policy.skill_cloud_run_resource_pressure",
|
||||
"google_monitoring_alert_policy.skill_database_migration_failure",
|
||||
"google_monitoring_alert_policy.skill_digest_mismatch",
|
||||
"google_monitoring_alert_policy.skill_finalize_failures",
|
||||
"google_monitoring_alert_policy.skill_route_latency",
|
||||
"google_monitoring_alert_policy.skill_signing_failure",
|
||||
"google_monitoring_alert_policy.skill_storage_lifecycle_failure",
|
||||
"google_monitoring_dashboard.skill_sharing",
|
||||
"google_project_iam_custom_role.skill_storage_inventory",
|
||||
"google_project_iam_member.auth_runtime_cloudsql_client",
|
||||
"google_project_iam_member.github_artifact_writer",
|
||||
"google_project_iam_member.github_cloud_run_developer",
|
||||
"google_project_iam_member.runtime_skills_cloudsql_client",
|
||||
"google_secret_manager_secret.auth",
|
||||
"google_secret_manager_secret.auth_database_url",
|
||||
"google_secret_manager_secret.skills_database_url",
|
||||
"google_secret_manager_secret_iam_member.auth_database_url_accessor",
|
||||
"google_secret_manager_secret_iam_member.auth_runtime_accessor",
|
||||
"google_secret_manager_secret_iam_member.runtime_skills_database_url_accessor",
|
||||
"google_secret_manager_secret_version.auth_database_url",
|
||||
"google_secret_manager_secret_version.skills_database_url",
|
||||
"google_service_account.auth_runtime",
|
||||
"google_service_account.github_production_app_deploy",
|
||||
"google_service_account.skill_storage_monitor",
|
||||
"google_service_account.skill_storage_scheduler",
|
||||
"google_service_account_iam_member.github_auth_runtime_service_account_user",
|
||||
"google_service_account_iam_member.github_production_app_auth_runtime_user",
|
||||
"google_service_account_iam_member.github_production_app_deploy_workload_identity_user",
|
||||
"google_service_account_iam_member.github_production_app_runtime_user",
|
||||
"google_service_account_iam_member.github_production_app_skill_monitor_user",
|
||||
"google_service_account_iam_member.github_runtime_service_account_user",
|
||||
"google_service_account_iam_member.github_skill_storage_monitor_user",
|
||||
"google_service_account_iam_member.runtime_skill_package_signer",
|
||||
"google_sql_database.auth",
|
||||
"google_sql_database.skills",
|
||||
"google_sql_user.auth",
|
||||
"google_sql_user.skills",
|
||||
"google_storage_bucket.artifacts",
|
||||
"google_storage_bucket.skill_packages",
|
||||
"google_storage_bucket_iam_member.runtime_artifact_object_admin",
|
||||
"google_storage_bucket_iam_member.runtime_skill_package_object_user",
|
||||
"google_storage_bucket_iam_member.skill_storage_monitor_inventory",
|
||||
"random_password.auth_database",
|
||||
"random_password.skills_database",
|
||||
"time_sleep.skill_metric_descriptor_propagation"
|
||||
],
|
||||
"relay": [
|
||||
"google_artifact_registry_repository_iam_member.github_production_relay_staging_mirror_writer",
|
||||
"google_artifact_registry_repository_iam_member.github_production_relay_writer",
|
||||
"google_artifact_registry_repository_iam_member.github_push_artifact_writer",
|
||||
"google_artifact_registry_repository_iam_member.github_relay_asia_topology_artifact_reader",
|
||||
"google_artifact_registry_repository_iam_member.github_staging_relay_deploy_artifact_reader",
|
||||
"google_certificate_manager_certificate.relay_gce",
|
||||
"google_certificate_manager_certificate_map.relay_gce",
|
||||
"google_certificate_manager_certificate_map_entry.relay_gce",
|
||||
"google_certificate_manager_dns_authorization.relay_gce",
|
||||
"google_cloud_run_domain_mapping.push",
|
||||
"google_cloud_run_domain_mapping.relay",
|
||||
"google_cloud_run_domain_mapping.relay_cell",
|
||||
"google_cloud_run_v2_service.push",
|
||||
"google_cloud_run_v2_service.relay",
|
||||
"google_cloud_run_v2_service.relay_cell",
|
||||
"google_cloud_run_v2_service.relay_fence_broker",
|
||||
"google_cloud_run_v2_service_iam_member.github_production_push_developer",
|
||||
"google_cloud_run_v2_service_iam_member.github_production_relay_director_developer",
|
||||
"google_cloud_run_v2_service_iam_member.github_production_relay_fence_broker_developer",
|
||||
"google_cloud_run_v2_service_iam_member.github_staging_relay_capacity_developer",
|
||||
"google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_auth_developer",
|
||||
"google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer",
|
||||
"google_cloud_run_v2_service_iam_member.relay_fence_broker_deploy_invoker",
|
||||
"google_cloud_run_v2_service_iam_member.relay_fence_broker_invoker",
|
||||
"google_compute_backend_service.relay_gce_cell",
|
||||
"google_compute_firewall.relay_gce_iap_ssh",
|
||||
"google_compute_firewall.relay_gce_load_balancer",
|
||||
"google_compute_global_address.relay_gce",
|
||||
"google_compute_global_forwarding_rule.relay_gce",
|
||||
"google_compute_health_check.relay_gce_liveness",
|
||||
"google_compute_health_check.relay_gce_readiness",
|
||||
"google_compute_instance_group_manager.relay_gce_cell",
|
||||
"google_compute_instance_template.relay_gce_cell",
|
||||
"google_compute_network.relay_gce",
|
||||
"google_compute_router.relay_gce",
|
||||
"google_compute_router.relay_gce_additional",
|
||||
"google_compute_router_nat.relay_gce",
|
||||
"google_compute_router_nat.relay_gce_additional",
|
||||
"google_compute_subnetwork.relay_gce",
|
||||
"google_compute_subnetwork.relay_gce_additional",
|
||||
"google_compute_target_https_proxy.relay_gce",
|
||||
"google_compute_url_map.relay_gce",
|
||||
"google_iam_workload_identity_pool_provider.github_fence",
|
||||
"google_iam_workload_identity_pool_provider.github_monitor",
|
||||
"google_iam_workload_identity_pool_provider.github_production_relay_capacity",
|
||||
"google_iam_workload_identity_pool_provider.github_push",
|
||||
"google_iam_workload_identity_pool_provider.github_relay_asia_proof",
|
||||
"google_iam_workload_identity_pool_provider.github_relay_asia_topology",
|
||||
"google_iam_workload_identity_pool_provider.github_staging_relay_capacity",
|
||||
"google_iam_workload_identity_pool_provider.github_staging_relay_deploy",
|
||||
"google_logging_metric.relay_assignment_lease_shadow",
|
||||
"google_logging_metric.relay_cell_fix_level",
|
||||
"google_logging_metric.relay_cell_pool_herd",
|
||||
"google_logging_metric.relay_cell_reregistration_gave_up",
|
||||
"google_logging_metric.relay_cell_reregistration_stalled",
|
||||
"google_logging_metric.relay_cell_reserve_dead_man_tripped",
|
||||
"google_logging_metric.relay_control_flags_unreadable",
|
||||
"google_logging_metric.relay_dead_cell_verdict_withheld",
|
||||
"google_logging_metric.relay_incident",
|
||||
"google_logging_metric.relay_event_loop_stall",
|
||||
"google_logging_metric.relay_long_cell_inventory_hold",
|
||||
"google_logging_metric.relay_reserve_placement_off_with_reserve_cells",
|
||||
"google_logging_metric.relay_reserve_watch_alarm",
|
||||
"google_logging_metric.relay_snapshot",
|
||||
"google_monitoring_alert_policy.relay_assignment_5xx",
|
||||
"google_monitoring_alert_policy.relay_assignment_edge_429",
|
||||
"google_monitoring_alert_policy.relay_assignment_lease_bad_signature",
|
||||
"google_monitoring_alert_policy.relay_cell_control_close_burst",
|
||||
"google_monitoring_alert_policy.relay_cell_control_rtt",
|
||||
"google_monitoring_alert_policy.relay_cell_outdated_fix_level",
|
||||
"google_monitoring_alert_policy.relay_cell_pool_herd",
|
||||
"google_monitoring_alert_policy.relay_cell_reregistration_gave_up",
|
||||
"google_monitoring_alert_policy.relay_cell_reregistration_stalled",
|
||||
"google_monitoring_alert_policy.relay_cell_reserve_dead_man_tripped",
|
||||
"google_monitoring_alert_policy.relay_cell_process_exit",
|
||||
"google_monitoring_alert_policy.relay_cloud_nat_port_drops",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_backends",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_disk",
|
||||
"google_monitoring_alert_policy.relay_control_flags_unreadable",
|
||||
"google_monitoring_alert_policy.relay_custom",
|
||||
"google_monitoring_alert_policy.relay_database_rejection_fenced",
|
||||
"google_monitoring_alert_policy.relay_dead_cell_verdict_withheld",
|
||||
"google_monitoring_alert_policy.relay_director_cell_inventory_hold",
|
||||
"google_monitoring_alert_policy.relay_far_cell_accept_latency",
|
||||
"google_monitoring_alert_policy.relay_gce_connection_headroom",
|
||||
"google_monitoring_alert_policy.relay_lock_timeout_burst",
|
||||
"google_monitoring_alert_policy.relay_event_loop_stall",
|
||||
"google_monitoring_alert_policy.relay_long_cell_inventory_hold",
|
||||
"google_monitoring_alert_policy.relay_postgres_retry_exhausted",
|
||||
"google_monitoring_alert_policy.relay_process_fatal",
|
||||
"google_monitoring_alert_policy.relay_region_hint_skew",
|
||||
"google_monitoring_alert_policy.relay_reserve_placement_off_with_reserve_cells",
|
||||
"google_monitoring_alert_policy.relay_reserve_watch_alarm",
|
||||
"google_monitoring_dashboard.relay_incident",
|
||||
"google_project_iam_custom_role.github_production_relay_capacity_mutation",
|
||||
"google_project_iam_custom_role.github_relay_asia_topology_mutation",
|
||||
"google_project_iam_custom_role.github_relay_asia_topology_read",
|
||||
"google_project_iam_custom_role.github_relay_asia_topology_state_list",
|
||||
"google_project_iam_custom_role.github_staging_relay_capacity_mutation",
|
||||
"google_project_iam_custom_role.github_staging_relay_power",
|
||||
"google_project_iam_custom_role.relay_control_writer",
|
||||
"google_project_iam_custom_role.relay_fence_broker_mutation",
|
||||
"google_project_iam_member.github_fence_cloudsql_viewer",
|
||||
"google_project_iam_member.github_fence_compute_viewer",
|
||||
"google_project_iam_member.github_fence_logging_viewer",
|
||||
"google_project_iam_member.github_fence_monitoring_viewer",
|
||||
"google_project_iam_member.github_monitor_compute_viewer",
|
||||
"google_project_iam_member.github_production_relay_capacity_artifact_reader",
|
||||
"google_project_iam_member.github_production_relay_capacity_mutation",
|
||||
"google_project_iam_member.github_production_relay_capacity_viewer",
|
||||
"google_project_iam_member.github_relay_asia_proof_logging_viewer",
|
||||
"google_project_iam_member.github_relay_asia_proof_monitoring_viewer",
|
||||
"google_project_iam_member.github_relay_asia_topology_mutation",
|
||||
"google_project_iam_member.github_relay_asia_topology_read",
|
||||
"google_project_iam_member.github_relay_monitor_cloudsql_viewer",
|
||||
"google_project_iam_member.github_relay_monitor_logging_viewer",
|
||||
"google_project_iam_member.github_relay_monitor_monitoring_viewer",
|
||||
"google_project_iam_member.github_staging_relay_capacity_artifact_reader",
|
||||
"google_project_iam_member.github_staging_relay_capacity_mutation",
|
||||
"google_project_iam_member.github_staging_relay_capacity_viewer",
|
||||
"google_project_iam_member.github_staging_relay_deploy_compute_viewer",
|
||||
"google_project_iam_member.github_staging_relay_deploy_logging_viewer",
|
||||
"google_project_iam_member.github_staging_relay_power",
|
||||
"google_project_iam_member.push_runtime_cloudsql_client",
|
||||
"google_project_iam_member.push_runtime_fcm_admin",
|
||||
"google_project_iam_member.push_runtime_service_usage_consumer",
|
||||
"google_project_iam_member.relay_director_runtime_cloudsql_client",
|
||||
"google_project_iam_member.relay_fence_broker_artifact_reader",
|
||||
"google_project_iam_member.relay_fence_broker_compute_viewer",
|
||||
"google_project_iam_member.relay_fence_broker_logging_viewer",
|
||||
"google_project_iam_member.relay_fence_broker_mutation",
|
||||
"google_project_iam_member.relay_runtime_artifact_reader",
|
||||
"google_project_iam_member.relay_runtime_cloudsql_client",
|
||||
"google_project_iam_member.relay_runtime_log_writer",
|
||||
"google_secret_manager_secret.push_dedicated_database_url",
|
||||
"google_secret_manager_secret.push_provider",
|
||||
"google_secret_manager_secret.relay_assignment_signing_key",
|
||||
"google_secret_manager_secret.relay_database_url",
|
||||
"google_secret_manager_secret.relay_director_database_url",
|
||||
"google_secret_manager_secret.relay_regional_placement_enabled",
|
||||
"google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor",
|
||||
"google_secret_manager_secret_iam_member.push_provider_runtime_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_assignment_signing_key_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_assignment_signing_key_director_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_database_url_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_database_url_director_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_director_database_url_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder",
|
||||
"google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer",
|
||||
"google_secret_manager_secret_iam_member.relay_regional_placement_director_accessor",
|
||||
"google_secret_manager_secret_iam_member.relay_regional_placement_runtime_accessor",
|
||||
"google_secret_manager_secret_version.push_dedicated_database_url",
|
||||
"google_secret_manager_secret_version.relay_assignment_signing_key",
|
||||
"google_secret_manager_secret_version.relay_database_url",
|
||||
"google_secret_manager_secret_version.relay_director_database_url",
|
||||
"google_secret_manager_secret_version.relay_regional_placement_enabled",
|
||||
"google_service_account.github_fence",
|
||||
"google_service_account.github_monitor",
|
||||
"google_service_account.github_production_relay_capacity",
|
||||
"google_service_account.github_push_deploy",
|
||||
"google_service_account.github_relay_asia_proof",
|
||||
"google_service_account.github_relay_asia_topology",
|
||||
"google_service_account.github_staging_relay_capacity",
|
||||
"google_service_account.github_staging_relay_deploy",
|
||||
"google_service_account.push_runtime",
|
||||
"google_service_account.relay_director_runtime",
|
||||
"google_service_account.relay_fence_broker",
|
||||
"google_service_account.relay_runtime",
|
||||
"google_service_account_iam_member.github_accepted_repository_workload_identity_user",
|
||||
"google_service_account_iam_member.github_fence_workload_identity_user",
|
||||
"google_service_account_iam_member.github_monitor_workload_identity_user",
|
||||
"google_service_account_iam_member.github_production_push_runtime_token_creator",
|
||||
"google_service_account_iam_member.github_production_push_runtime_user",
|
||||
"google_service_account_iam_member.github_production_relay_capacity_runtime_user",
|
||||
"google_service_account_iam_member.github_production_relay_capacity_workload_identity_user",
|
||||
"google_service_account_iam_member.github_push_workload_identity_user",
|
||||
"google_service_account_iam_member.github_relay_asia_proof_workload_identity_user",
|
||||
"google_service_account_iam_member.github_relay_asia_topology_runtime_user",
|
||||
"google_service_account_iam_member.github_relay_asia_topology_workload_identity_user",
|
||||
"google_service_account_iam_member.github_relay_director_runtime_service_account_user",
|
||||
"google_service_account_iam_member.github_relay_fence_broker_service_account_user",
|
||||
"google_service_account_iam_member.github_relay_runtime_service_account_user",
|
||||
"google_service_account_iam_member.github_staging_relay_capacity_runtime_user",
|
||||
"google_service_account_iam_member.github_staging_relay_capacity_workload_identity_user",
|
||||
"google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user",
|
||||
"google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user",
|
||||
"google_service_account_iam_member.relay_fence_broker_requester_token_creator",
|
||||
"google_sql_database.push_dedicated",
|
||||
"google_sql_database.relay",
|
||||
"google_sql_database_instance.push_dedicated",
|
||||
"google_sql_user.push_dedicated",
|
||||
"google_sql_user.relay",
|
||||
"google_storage_bucket.relay_control",
|
||||
"google_storage_bucket_iam_member.github_production_relay_capacity_state",
|
||||
"google_storage_bucket_iam_member.github_push_rollout_lease",
|
||||
"google_storage_bucket_iam_member.github_relay_asia_topology_state",
|
||||
"google_storage_bucket_iam_member.github_relay_asia_topology_state_list",
|
||||
"google_storage_bucket_iam_member.github_staging_relay_capacity_state",
|
||||
"google_storage_bucket_iam_member.github_staging_relay_deploy_state",
|
||||
"google_storage_bucket_iam_member.github_staging_relay_deploy_state_list",
|
||||
"google_storage_bucket_iam_member.relay_control_cell_reader",
|
||||
"google_storage_bucket_iam_member.relay_control_workflow_writer",
|
||||
"google_storage_bucket_iam_member.relay_fence_broker_bucket_reader",
|
||||
"google_storage_bucket_iam_member.relay_fence_broker_state_objects",
|
||||
"random_password.push_dedicated_database",
|
||||
"random_password.relay_assignment_signing_key",
|
||||
"random_password.relay_database"
|
||||
],
|
||||
"env_conditional": {
|
||||
"google_iam_workload_identity_pool_provider.github": {
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
},
|
||||
"google_project_iam_member.github_cloudsql_viewer": {
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
},
|
||||
"google_project_iam_member.github_compute_viewer": {
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
},
|
||||
"google_project_iam_member.github_log_writer": {
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
},
|
||||
"google_project_iam_member.github_logging_viewer": {
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
},
|
||||
"google_project_iam_member.github_monitoring_viewer": {
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
},
|
||||
"google_service_account.github_deploy": {
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
},
|
||||
"google_service_account_iam_member.github_workload_identity_user": {
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
},
|
||||
"google_storage_bucket_iam_member.github_terraform_state_reader": {
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
}
|
||||
},
|
||||
"state_orphans": {
|
||||
"staging": [],
|
||||
"production": []
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
// Why: the relay root can never plan zero-diff (cell templates roll one at a time by design),
|
||||
// so the split is gated on plan EQUIVALENCE: the normalized change set for a root's addresses
|
||||
// must be identical before and after a state move. This captures that set deterministically.
|
||||
// Read-only: -lock=false, -refresh=false, no apply. Forgets from `removed` blocks are excluded
|
||||
// because the baseline has none.
|
||||
|
||||
const usage =
|
||||
'usage: capture-terraform-plan-baseline.mjs --root <dir> --env <staging|production> --out <dir> [--tag <name>]'
|
||||
|
||||
export function normalizePlan(planJson) {
|
||||
const changes = (planJson.resource_changes ?? [])
|
||||
.filter((entry) => !entry.change.actions.includes('forget'))
|
||||
.map((entry) => ({
|
||||
address: entry.address,
|
||||
actions: entry.change.actions,
|
||||
before: entry.change.before ?? null,
|
||||
after: entry.change.after ?? null,
|
||||
after_unknown: entry.change.after_unknown ?? null
|
||||
}))
|
||||
.sort((left, right) => (left.address < right.address ? -1 : left.address > right.address ? 1 : 0))
|
||||
return changes
|
||||
}
|
||||
|
||||
export function summarize(changes) {
|
||||
const counts = { create: 0, update: 0, delete: 0, replace: 0, 'no-op': 0, read: 0 }
|
||||
for (const change of changes) {
|
||||
const key = change.actions.join('-')
|
||||
if (key === 'create') counts.create += 1
|
||||
else if (key === 'update') counts.update += 1
|
||||
else if (key === 'delete') counts.delete += 1
|
||||
else if (key === 'delete-create' || key === 'create-delete') counts.replace += 1
|
||||
else if (key === 'read') counts.read += 1
|
||||
else counts['no-op'] += 1
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
function argument(flag) {
|
||||
const index = process.argv.indexOf(flag)
|
||||
return index >= 0 ? process.argv[index + 1] : undefined
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url.endsWith(process.argv[1].split('/').pop())) {
|
||||
const root = argument('--root')
|
||||
const environment = argument('--env')
|
||||
const out = argument('--out')
|
||||
const tag = argument('--tag') ?? `${environment}-${root.replaceAll('/', '_')}`
|
||||
if (!root || !['staging', 'production'].includes(environment) || !out) {
|
||||
process.stderr.write(`${usage}\n`)
|
||||
process.exit(2)
|
||||
}
|
||||
// The Cloudflare override only applies to the root that still declares the records; the relay
|
||||
// root dropped them in the carve and errors on a -var for an undeclared variable.
|
||||
const declaresArtifactDns = readFileSync(join(root, 'variables.tf'), 'utf8').includes(
|
||||
'variable "manage_artifact_dns"'
|
||||
)
|
||||
mkdirSync(out, { recursive: true })
|
||||
const planFile = join(out, `${tag}.tfplan`)
|
||||
execFileSync(
|
||||
'terraform',
|
||||
[
|
||||
`-chdir=${root}`, 'plan', '-input=false', '-lock=false', '-refresh=false', '-no-color',
|
||||
`-var-file=environments/${environment}.tfvars`,
|
||||
...(declaresArtifactDns ? ['-var', 'manage_artifact_dns=false'] : []),
|
||||
`-out=${planFile}`
|
||||
],
|
||||
{ stdio: ['ignore', 'inherit', 'inherit'] }
|
||||
)
|
||||
const json = JSON.parse(
|
||||
execFileSync('terraform', [`-chdir=${root}`, 'show', '-json', planFile], {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 256 * 1024 * 1024
|
||||
})
|
||||
)
|
||||
const normalized = normalizePlan(json)
|
||||
writeFileSync(join(out, `${tag}.norm.json`), `${JSON.stringify(normalized, null, 1)}\n`)
|
||||
process.stdout.write(`${tag}: ${JSON.stringify(summarize(normalized))}\n`)
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { normalizePlan, summarize } from './capture-terraform-plan-baseline.mjs'
|
||||
|
||||
test('normalizes, sorts, and drops forgets so a removed block cannot skew equivalence', () => {
|
||||
const changes = normalizePlan({
|
||||
resource_changes: [
|
||||
{ address: 'b.two', change: { actions: ['update'], before: { x: 1 }, after: { x: 2 } } },
|
||||
{ address: 'a.one', change: { actions: ['forget'], before: {}, after: null } },
|
||||
{ address: 'c.three', change: { actions: ['delete', 'create'], before: {}, after: {}, after_unknown: { id: true } } }
|
||||
]
|
||||
})
|
||||
assert.deepEqual(changes.map((change) => change.address), ['b.two', 'c.three'])
|
||||
assert.deepEqual(summarize(changes), { create: 0, update: 1, delete: 0, replace: 1, 'no-op': 0, read: 0 })
|
||||
})
|
||||
@@ -1,167 +0,0 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
|
||||
import { isReserveCellRefusal, reserveCellRollRefusal } from './relay-reserve-cell-refusal.mjs'
|
||||
|
||||
// Drained hosts that find no free slot keep redialling and pin the drained cell (c28,
|
||||
// 2026-10-01). Hosts are counted as controls, but each moved host also brings its splices;
|
||||
// the 20% margin is for those.
|
||||
export const MAX_HEADROOM_FRACTION = 0.8
|
||||
|
||||
function count(value, name) {
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} is invalid`)
|
||||
return value
|
||||
}
|
||||
|
||||
export function parseHeadroomArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['director-origin', 'cell-origin', 'cell-id', 'general-cells']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
for (const key of ['director-origin', 'cell-origin']) {
|
||||
const url = new URL(values[key])
|
||||
if (url.protocol !== 'https:' || url.origin !== values[key]) {
|
||||
throw new Error('origins must be canonical HTTPS origins')
|
||||
}
|
||||
}
|
||||
const generalCells = values['general-cells'] === 'none'
|
||||
? []
|
||||
: values['general-cells'].split(',')
|
||||
if (
|
||||
new Set(generalCells).size !== generalCells.length ||
|
||||
generalCells.some((cellId) => !/^[a-z0-9-]+$/.test(cellId))
|
||||
) {
|
||||
throw new Error('--general-cells is invalid')
|
||||
}
|
||||
return {
|
||||
directorOrigin: values['director-origin'],
|
||||
cellOrigin: values['cell-origin'],
|
||||
cellId: values['cell-id'],
|
||||
generalCells
|
||||
}
|
||||
}
|
||||
|
||||
async function adminJson(fetchImpl, url, token, body, wait) {
|
||||
const response = await fetchAdminOnceMore(
|
||||
fetchImpl,
|
||||
url,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body)
|
||||
},
|
||||
{ wait }
|
||||
)
|
||||
const json = await response.json().catch(() => ({}))
|
||||
// null: the director refused the read because the cell is reserve.
|
||||
if (isReserveCellRefusal(response.status, json)) return null
|
||||
if (!response.ok) throw new Error(`${new URL(url).pathname} returned ${response.status}`)
|
||||
return json
|
||||
}
|
||||
|
||||
// A cell whose capacity view is stale, absent, or not general offers no slot we can count.
|
||||
// Placement admits while enforced units plus outstanding reservations stay under the pause.
|
||||
function freeSlots(status) {
|
||||
const capacity = status.connectionCapacity
|
||||
if (
|
||||
status.enabled !== true ||
|
||||
status.admissionState !== 'general' ||
|
||||
capacity === null ||
|
||||
capacity === undefined ||
|
||||
capacity.heartbeatFresh !== true
|
||||
) {
|
||||
return 0
|
||||
}
|
||||
const pause = count(capacity.normalAdmissionPause, 'normal admission pause')
|
||||
const used = Math.max(
|
||||
count(capacity.observedConnections, 'observed connections'),
|
||||
count(capacity.enforcedConnectionUnits, 'enforced connection units')
|
||||
)
|
||||
const reserved = count(capacity.pendingControlReservations, 'pending control reservations')
|
||||
return Math.max(0, pause - used - reserved)
|
||||
}
|
||||
|
||||
export async function checkSameCapHeadroom(config, overrides = {}) {
|
||||
const fetchImpl = overrides.fetch ?? fetch
|
||||
const wait = overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
|
||||
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
|
||||
const cellStatus = async (cellId) => {
|
||||
const answer = await adminJson(
|
||||
fetchImpl,
|
||||
`${config.directorOrigin}/v1/admin/cell-status`,
|
||||
token,
|
||||
{ v: 1, cellId },
|
||||
wait
|
||||
)
|
||||
if (answer === null) return null
|
||||
if (answer.status?.cellId !== cellId) {
|
||||
throw new Error('director status does not match the cell')
|
||||
}
|
||||
return answer.status
|
||||
}
|
||||
const runtime = await adminJson(
|
||||
fetchImpl,
|
||||
`${config.cellOrigin}/v1/admin/runtime-status`,
|
||||
token,
|
||||
{ v: 1 },
|
||||
wait
|
||||
)
|
||||
if (runtime?.role !== 'cell' || runtime.cellId !== config.cellId) {
|
||||
throw new Error('runtime status does not match the cell')
|
||||
}
|
||||
const targetHosts = count(runtime.runtime?.controls, 'target host controls')
|
||||
const targetStatus = await cellStatus(config.cellId)
|
||||
if (targetStatus === null) throw reserveCellRollRefusal(config.cellId)
|
||||
const targetRegion = targetStatus.region ?? null
|
||||
const cells = []
|
||||
for (const cellId of config.generalCells) {
|
||||
if (cellId === config.cellId) continue
|
||||
const status = await cellStatus(cellId)
|
||||
// A reserve cell's free seats are not in the database: counted as none, never guessed.
|
||||
cells.push(
|
||||
status === null
|
||||
? { cellId, region: null, freeSlots: 0, reserve: true }
|
||||
: { cellId, region: status.region ?? null, freeSlots: freeSlots(status) }
|
||||
)
|
||||
}
|
||||
const total = (filter) =>
|
||||
cells.filter(filter).reduce((sum, cell) => sum + cell.freeSlots, 0)
|
||||
const sameRegion = (cell) => targetRegion !== null && cell.region === targetRegion
|
||||
const free = total(() => true)
|
||||
const allowedHosts = Math.floor(free * MAX_HEADROOM_FRACTION)
|
||||
return {
|
||||
cellId: config.cellId,
|
||||
region: targetRegion,
|
||||
targetHosts,
|
||||
freeSlots: free,
|
||||
sameRegionFreeSlots: total(sameRegion),
|
||||
otherRegionFreeSlots: total((cell) => !sameRegion(cell)),
|
||||
allowedHosts,
|
||||
sufficient: targetHosts <= allowedHosts,
|
||||
cells
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const result = await checkSameCapHeadroom(parseHeadroomArguments(argv))
|
||||
process.stdout.write(`${JSON.stringify({ event: 'relay_same_cap_headroom', ...result })}\n`)
|
||||
if (!result.sufficient) {
|
||||
throw new Error(
|
||||
`${result.targetHosts} hosts on ${result.cellId} exceed ` +
|
||||
`${MAX_HEADROOM_FRACTION * 100}% of ${result.freeSlots} free general-cell slots`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
checkSameCapHeadroom,
|
||||
main,
|
||||
parseHeadroomArguments
|
||||
} from './check-relay-same-cap-headroom.mjs'
|
||||
|
||||
const config = {
|
||||
directorOrigin: 'https://relay.example.com',
|
||||
cellOrigin: 'https://c28.relay.example.com',
|
||||
cellId: 'production-gce-c28',
|
||||
generalCells: ['production-gce-c28', 'production-gce-c29', 'production-gce-asia-c1']
|
||||
}
|
||||
|
||||
function cell({
|
||||
region = 'us-central1',
|
||||
pause = 840,
|
||||
observed = 0,
|
||||
enforced = observed,
|
||||
pending = 0,
|
||||
...rest
|
||||
} = {}) {
|
||||
return {
|
||||
enabled: true,
|
||||
admissionState: 'general',
|
||||
region,
|
||||
connectionCapacity: {
|
||||
normalAdmissionPause: pause,
|
||||
observedConnections: observed,
|
||||
enforcedConnectionUnits: enforced,
|
||||
pendingControlReservations: pending,
|
||||
heartbeatFresh: true
|
||||
},
|
||||
...rest
|
||||
}
|
||||
}
|
||||
|
||||
function harness({ controls = 500, cells = {} } = {}) {
|
||||
const statuses = {
|
||||
'production-gce-c28': cell({ observed: 800 }),
|
||||
'production-gce-c29': cell({ observed: 340 }),
|
||||
'production-gce-asia-c1': cell({ region: 'asia-east2', pause: 2_840, observed: 2_640 }),
|
||||
...cells
|
||||
}
|
||||
return async (url, options) => {
|
||||
if (new URL(url).pathname === '/v1/admin/runtime-status') {
|
||||
return Response.json({ role: 'cell', cellId: config.cellId, runtime: { controls } })
|
||||
}
|
||||
const { cellId } = JSON.parse(options.body)
|
||||
return Response.json({ v: 1, status: { cellId, ...statuses[cellId] } })
|
||||
}
|
||||
}
|
||||
|
||||
const run = async (fetch) =>
|
||||
await checkSameCapHeadroom(config, { fetch, token: 'masked-token', wait: async () => {} })
|
||||
|
||||
test('counts free general-cell slots by region and excludes the target', async () => {
|
||||
const result = await run(harness({ controls: 560 }))
|
||||
assert.deepEqual(
|
||||
{ ...result, cells: undefined },
|
||||
{
|
||||
cellId: config.cellId,
|
||||
region: 'us-central1',
|
||||
targetHosts: 560,
|
||||
freeSlots: 700,
|
||||
sameRegionFreeSlots: 500,
|
||||
otherRegionFreeSlots: 200,
|
||||
allowedHosts: 560,
|
||||
sufficient: true,
|
||||
cells: undefined
|
||||
}
|
||||
)
|
||||
assert.deepEqual(result.cells.map((entry) => entry.cellId), [
|
||||
'production-gce-c29',
|
||||
'production-gce-asia-c1'
|
||||
])
|
||||
})
|
||||
|
||||
test('free slots subtract the larger of observed and enforced use plus reservations', async () => {
|
||||
for (const [state, free] of [
|
||||
[{ observed: 340, enforced: 400 }, 640],
|
||||
[{ observed: 400, enforced: 340 }, 640],
|
||||
[{ observed: 340, pending: 60 }, 640],
|
||||
[{ observed: 800, pending: 60 }, 200]
|
||||
]) {
|
||||
const result = await run(harness({ cells: { 'production-gce-c29': cell(state) } }))
|
||||
assert.equal(result.freeSlots, free)
|
||||
}
|
||||
})
|
||||
|
||||
test('rejects malformed capacity counts', async () => {
|
||||
await assert.rejects(
|
||||
run(harness({
|
||||
cells: { 'production-gce-c29': cell({ pending: null }) }
|
||||
})),
|
||||
/pending control reservations is invalid/
|
||||
)
|
||||
})
|
||||
|
||||
test('refuses a cell whose hosts exceed 80% of the free slots', async () => {
|
||||
const result = await run(harness({ controls: 561 }))
|
||||
assert.equal(result.sufficient, false)
|
||||
})
|
||||
|
||||
test('a stale, full, or non-general cell offers no slots', async () => {
|
||||
for (const blocked of [
|
||||
cell({
|
||||
connectionCapacity: {
|
||||
normalAdmissionPause: 840,
|
||||
observedConnections: 0,
|
||||
enforcedConnectionUnits: 0,
|
||||
pendingControlReservations: 0,
|
||||
heartbeatFresh: false
|
||||
}
|
||||
}),
|
||||
cell({ connectionCapacity: null }),
|
||||
cell({ admissionState: 'migration-only' }),
|
||||
cell({ enabled: false }),
|
||||
cell({ observed: 900 })
|
||||
]) {
|
||||
const result = await run(harness({ cells: { 'production-gce-c29': blocked } }))
|
||||
assert.equal(result.freeSlots, 200)
|
||||
assert.equal(result.sufficient, false)
|
||||
}
|
||||
})
|
||||
|
||||
test('no general cells means no headroom', async () => {
|
||||
const result = await checkSameCapHeadroom(
|
||||
{ ...config, generalCells: [] },
|
||||
{ fetch: harness({ controls: 1 }), token: 'masked-token', wait: async () => {} }
|
||||
)
|
||||
assert.equal(result.freeSlots, 0)
|
||||
assert.equal(result.sufficient, false)
|
||||
})
|
||||
|
||||
const reserveRefusal = () => Response.json({ error: 'cell_in_reserve_mode' }, { status: 404 })
|
||||
|
||||
test('a reserve cell that is not being rolled offers no slots and is named', async () => {
|
||||
const base = harness({ controls: 160 })
|
||||
const result = await run(async (url, options) =>
|
||||
JSON.parse(options.body).cellId === 'production-gce-c29'
|
||||
? reserveRefusal()
|
||||
: await base(url, options))
|
||||
assert.equal(result.freeSlots, 200)
|
||||
assert.equal(result.sufficient, true)
|
||||
assert.deepEqual(result.cells.find((entry) => entry.cellId === 'production-gce-c29'), {
|
||||
cellId: 'production-gce-c29',
|
||||
region: null,
|
||||
freeSlots: 0,
|
||||
reserve: true
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses to roll a reserve cell, naming the flip back to db', async () => {
|
||||
const base = harness()
|
||||
await assert.rejects(
|
||||
run(async (url, options) =>
|
||||
JSON.parse(options.body ?? '{}').cellId === config.cellId
|
||||
? reserveRefusal()
|
||||
: await base(url, options)),
|
||||
/production-gce-c28 is in reserve mode .*flip it to admitMode=db/
|
||||
)
|
||||
})
|
||||
|
||||
test('any other 404 from cell-status still fails', async () => {
|
||||
const base = harness()
|
||||
await assert.rejects(
|
||||
run(async (url, options) =>
|
||||
JSON.parse(options.body).cellId === 'production-gce-c29'
|
||||
? Response.json({ error: 'cell_not_found' }, { status: 404 })
|
||||
: await base(url, options)),
|
||||
/cell-status returned 404/
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects a runtime or director answer for another cell', async () => {
|
||||
const base = harness()
|
||||
await assert.rejects(
|
||||
run(async (url, options) =>
|
||||
new URL(url).pathname === '/v1/admin/runtime-status'
|
||||
? Response.json({ role: 'cell', cellId: 'production-gce-c1', runtime: { controls: 1 } })
|
||||
: await base(url, options)),
|
||||
/runtime status does not match/
|
||||
)
|
||||
await assert.rejects(
|
||||
run(async (url, options) =>
|
||||
new URL(url).pathname === '/v1/admin/cell-status'
|
||||
? Response.json({ v: 1, status: { cellId: 'production-gce-c1' } })
|
||||
: await base(url, options)),
|
||||
/director status does not match/
|
||||
)
|
||||
})
|
||||
|
||||
test('parses the general cell list, including none', () => {
|
||||
const base = [
|
||||
'--director-origin', 'https://relay.example.com',
|
||||
'--cell-origin', 'https://c28.relay.example.com',
|
||||
'--cell-id', 'production-gce-c28'
|
||||
]
|
||||
assert.deepEqual(
|
||||
parseHeadroomArguments([...base, '--general-cells', 'a-c1,b-c2']).generalCells,
|
||||
['a-c1', 'b-c2']
|
||||
)
|
||||
assert.deepEqual(parseHeadroomArguments([...base, '--general-cells', 'none']).generalCells, [])
|
||||
assert.throws(() => parseHeadroomArguments([...base, '--general-cells', 'a,a']), /invalid/)
|
||||
assert.throws(() => parseHeadroomArguments(base), /missing --general-cells/)
|
||||
})
|
||||
|
||||
test('main prints the numbers and fails on insufficient headroom', async (t) => {
|
||||
const writes = []
|
||||
t.mock.method(process.stdout, 'write', (text) => writes.push(text))
|
||||
t.mock.method(globalThis, 'fetch', harness({ controls: 700 }))
|
||||
process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'masked-token'
|
||||
t.after(() => {
|
||||
delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
})
|
||||
await assert.rejects(
|
||||
main([
|
||||
'--director-origin', config.directorOrigin,
|
||||
'--cell-origin', config.cellOrigin,
|
||||
'--cell-id', config.cellId,
|
||||
'--general-cells', config.generalCells.join(',')
|
||||
]),
|
||||
/700 hosts on production-gce-c28 exceed 80% of 700 free general-cell slots/
|
||||
)
|
||||
const line = JSON.parse(writes.join(''))
|
||||
assert.equal(line.event, 'relay_same_cap_headroom')
|
||||
assert.equal(line.allowedHosts, 560)
|
||||
})
|
||||
@@ -1,109 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { isDeepStrictEqual } from 'node:util'
|
||||
|
||||
function parseArguments(argv) {
|
||||
if (argv.length !== 2 || argv[0] !== '--capacity-service-account' || !argv[1]) {
|
||||
throw new Error('missing --capacity-service-account')
|
||||
}
|
||||
return argv[1]
|
||||
}
|
||||
|
||||
export function classifyProductionCapacityDirector(state, capacityServiceAccount) {
|
||||
const currentIdentity = state.currentCapacityServiceAccount
|
||||
if (currentIdentity !== null && currentIdentity !== capacityServiceAccount) {
|
||||
throw new Error('director has an unexpected capacity identity')
|
||||
}
|
||||
const {
|
||||
baseCells,
|
||||
currentCells,
|
||||
capacityCellIds,
|
||||
targetCellId,
|
||||
targetHardCap
|
||||
} = state
|
||||
if (
|
||||
!Array.isArray(baseCells) ||
|
||||
!Array.isArray(currentCells) ||
|
||||
!Array.isArray(capacityCellIds) ||
|
||||
![600, 1000].includes(targetHardCap) ||
|
||||
new Set(capacityCellIds).size !== capacityCellIds.length ||
|
||||
!capacityCellIds.includes(targetCellId) ||
|
||||
baseCells.length !== currentCells.length ||
|
||||
new Set(baseCells.map((cell) => cell?.id)).size !== baseCells.length ||
|
||||
new Set(currentCells.map((cell) => cell?.id)).size !== currentCells.length
|
||||
) {
|
||||
throw new Error('director topology transition input is invalid')
|
||||
}
|
||||
const capacityCells = new Set(capacityCellIds)
|
||||
const normalizedCurrent = currentCells.map((current, index) => {
|
||||
const base = baseCells[index]
|
||||
if (
|
||||
typeof current?.id !== 'string' ||
|
||||
current.id !== base?.id
|
||||
) {
|
||||
throw new Error('director topology cell identity is invalid')
|
||||
}
|
||||
if (!capacityCells.has(current.id)) {
|
||||
if (!isDeepStrictEqual(current, base)) {
|
||||
throw new Error('director topology changed outside the capacity rollout')
|
||||
}
|
||||
return current
|
||||
}
|
||||
if (
|
||||
base.connectionHardCap !== 1000 ||
|
||||
base.connectionUnobservedBound !== 60 ||
|
||||
![600, 1000].includes(current.connectionHardCap) ||
|
||||
current.connectionUnobservedBound !== 60
|
||||
) {
|
||||
throw new Error('director capacity rollout state is invalid')
|
||||
}
|
||||
return {
|
||||
...current,
|
||||
connectionHardCap: base.connectionHardCap,
|
||||
connectionUnobservedBound: base.connectionUnobservedBound
|
||||
}
|
||||
})
|
||||
if (
|
||||
!isDeepStrictEqual(normalizedCurrent, baseCells) ||
|
||||
capacityCellIds.some((cellId) => !baseCells.some((cell) => cell.id === cellId))
|
||||
) {
|
||||
throw new Error('director topology is outside the reviewed capacity envelope')
|
||||
}
|
||||
const withTargetCap = (hardCap) => currentCells.map((cell) =>
|
||||
cell.id === targetCellId
|
||||
? { ...cell, connectionHardCap: hardCap, connectionUnobservedBound: 60 }
|
||||
: cell
|
||||
)
|
||||
const desiredCells = withTargetCap(targetHardCap)
|
||||
const predecessorCells = withTargetCap(targetHardCap === 600 ? 1000 : 600)
|
||||
const topologyPhase = isDeepStrictEqual(currentCells, desiredCells)
|
||||
? 'desired'
|
||||
: isDeepStrictEqual(currentCells, predecessorCells)
|
||||
? 'predecessor'
|
||||
: null
|
||||
if (!topologyPhase) throw new Error('director topology is not a reviewed transition state')
|
||||
return {
|
||||
topologyPhase,
|
||||
directorReady:
|
||||
topologyPhase === 'desired' && currentIdentity === capacityServiceAccount,
|
||||
desiredCells
|
||||
}
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2)) {
|
||||
const capacityServiceAccount = parseArguments(argv)
|
||||
const state = JSON.parse(readFileSync(0, 'utf8'))
|
||||
process.stdout.write(`${JSON.stringify({
|
||||
event: 'relay_production_capacity_director_classified',
|
||||
...classifyProductionCapacityDirector(state, capacityServiceAccount)
|
||||
})}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
try {
|
||||
main()
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
classifyProductionCapacityDirector
|
||||
} from './classify-relay-production-capacity-director.mjs'
|
||||
|
||||
const capacityServiceAccount =
|
||||
'orca-cloud-gha-relay-cap@onorca-cloud.iam.gserviceaccount.com'
|
||||
const capacityCellIds = ['production-gce-c25', 'production-gce-c26']
|
||||
const baseCells = [
|
||||
{ id: 'production-gce-c17', connectionHardCap: 600, connectionUnobservedBound: 60 },
|
||||
{ id: 'production-gce-c25', connectionHardCap: 1000, connectionUnobservedBound: 60 },
|
||||
{ id: 'production-gce-c26', connectionHardCap: 1000, connectionUnobservedBound: 60 }
|
||||
]
|
||||
const mixedCells = [
|
||||
baseCells[0],
|
||||
{ ...baseCells[1], connectionHardCap: 600 },
|
||||
baseCells[2]
|
||||
]
|
||||
|
||||
function state(overrides = {}) {
|
||||
return {
|
||||
baseCells,
|
||||
currentCells: mixedCells,
|
||||
capacityCellIds,
|
||||
targetCellId: 'production-gce-c25',
|
||||
targetHardCap: 1000,
|
||||
currentCapacityServiceAccount: capacityServiceAccount,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
test('classifies one target while preserving completed rollout cells', () => {
|
||||
assert.deepEqual(
|
||||
classifyProductionCapacityDirector(state(), capacityServiceAccount),
|
||||
{
|
||||
topologyPhase: 'predecessor',
|
||||
directorReady: false,
|
||||
desiredCells: baseCells
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('skips deployment only for exact topology and identity', () => {
|
||||
assert.deepEqual(
|
||||
classifyProductionCapacityDirector(state({ currentCells: baseCells }), capacityServiceAccount),
|
||||
{ topologyPhase: 'desired', directorReady: true, desiredCells: baseCells }
|
||||
)
|
||||
assert.deepEqual(
|
||||
classifyProductionCapacityDirector(state({
|
||||
currentCells: baseCells,
|
||||
targetHardCap: 600
|
||||
}), capacityServiceAccount),
|
||||
{
|
||||
topologyPhase: 'predecessor',
|
||||
directorReady: false,
|
||||
desiredCells: mixedCells
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects an unknown topology or capacity identity', () => {
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCells: [baseCells[0], { ...baseCells[1], connectionHardCap: 700 }, baseCells[2]]
|
||||
}), capacityServiceAccount),
|
||||
/rollout state is invalid/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCapacityServiceAccount: 'unexpected@onorca-cloud.iam.gserviceaccount.com'
|
||||
}), capacityServiceAccount),
|
||||
/unexpected capacity identity/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCells: [{ ...baseCells[0], connectionHardCap: 1000 }, mixedCells[1], mixedCells[2]]
|
||||
}), capacityServiceAccount),
|
||||
/outside the capacity rollout/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCells: [baseCells[0], { ...mixedCells[1], url: 'https://wrong.invalid' }, baseCells[2]]
|
||||
}), capacityServiceAccount),
|
||||
/outside the reviewed capacity envelope/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
targetCellId: 'production-gce-c17'
|
||||
}), capacityServiceAccount),
|
||||
/transition input is invalid/
|
||||
)
|
||||
})
|
||||
@@ -1,47 +0,0 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
export function classifyStagingBootstrap({ c2Kind, c2Admission, c3Kind, c3Admission }) {
|
||||
for (const kind of [c2Kind, c3Kind]) {
|
||||
if (!['legacy', 'modern'].includes(kind)) throw new Error('bootstrap runtime kind is invalid')
|
||||
}
|
||||
for (const admission of [c2Admission, c3Admission]) {
|
||||
if (!['general', 'migration-only'].includes(admission)) {
|
||||
throw new Error('bootstrap admission is not recoverable')
|
||||
}
|
||||
}
|
||||
if (c2Kind === 'modern' && c3Kind === 'modern') return 'complete'
|
||||
if (c2Kind === 'modern') return 'roll-c3'
|
||||
if (c3Kind === 'modern') return 'roll-c2'
|
||||
if (c2Admission === 'general') return 'normalize-and-roll-both'
|
||||
if (c3Admission === 'general') return 'resume-c2-then-c3'
|
||||
throw new Error('bootstrap has no general fallback')
|
||||
}
|
||||
|
||||
function parseArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
return {
|
||||
c2Kind: values['c2-kind'],
|
||||
c2Admission: values['c2-admission'],
|
||||
c3Kind: values['c3-kind'],
|
||||
c3Admission: values['c3-admission']
|
||||
}
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2)) {
|
||||
process.stdout.write(`${classifyStagingBootstrap(parseArguments(argv))}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
try {
|
||||
main()
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { classifyStagingBootstrap } from './classify-relay-staging-bootstrap.mjs'
|
||||
|
||||
const state = (c2Kind, c2Admission, c3Kind, c3Admission) => ({
|
||||
c2Kind,
|
||||
c2Admission,
|
||||
c3Kind,
|
||||
c3Admission
|
||||
})
|
||||
|
||||
test('classifies the fresh legacy bootstrap', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'general')),
|
||||
'normalize-and-roll-both'
|
||||
)
|
||||
})
|
||||
|
||||
test('retries normalization after a failed C3 restart', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'migration-only')),
|
||||
'normalize-and-roll-both'
|
||||
)
|
||||
})
|
||||
|
||||
test('resumes after C2 isolation', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'general')),
|
||||
'resume-c2-then-c3'
|
||||
)
|
||||
})
|
||||
|
||||
test('resumes after C2 apply or a partial C3 transition', () => {
|
||||
for (const c2Admission of ['general', 'migration-only']) {
|
||||
for (const c3Admission of ['general', 'migration-only']) {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('modern', c2Admission, 'legacy', c3Admission)),
|
||||
'roll-c3'
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('repairs an unexpected modern C3 before rolling legacy C2', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'migration-only', 'modern', 'general')),
|
||||
'roll-c2'
|
||||
)
|
||||
})
|
||||
|
||||
test('accepts already complete modern cells and rejects no-fallback legacy state', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('modern', 'migration-only', 'modern', 'general')),
|
||||
'complete'
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'migration-only')),
|
||||
/no general fallback/
|
||||
)
|
||||
})
|
||||
@@ -1,314 +0,0 @@
|
||||
// Derives, from workflow and script content, which workflows roll out against the shared Cloud SQL
|
||||
// instance. Hand lists go stale silently; everything here is read back off disk.
|
||||
import { readFileSync, readdirSync } from 'node:fs'
|
||||
import {
|
||||
RELAY_WORKFLOW_DIRECTORY,
|
||||
RELAY_WORKFLOW_FILE_PREFIX,
|
||||
relayWorkflowFile
|
||||
} from './relay-repository.mjs'
|
||||
|
||||
export const WORKFLOW_ROOT = RELAY_WORKFLOW_DIRECTORY
|
||||
export const SCRIPT_ROOT = new URL('./', import.meta.url)
|
||||
|
||||
export const LEASE_ACTION = './.github/actions/cloud-sql-rollout-lease'
|
||||
export const PRODUCTION_LEASE = {
|
||||
bucket: 'onorca-cloud-terraform-state',
|
||||
object: 'terraform/state/cloud-sql-rollout/production.lock'
|
||||
}
|
||||
export const STAGING_LEASE = {
|
||||
bucket: 'onorca-cloud-staging-terraform-state',
|
||||
object: 'terraform/state/cloud-sql-rollout/staging.lock'
|
||||
}
|
||||
|
||||
export const PRODUCTION_GROUP = 'production-cloud-sql-rollout'
|
||||
export const STAGING_GROUP = 'relay-staging-mutation'
|
||||
export const SELECTABLE_GROUP =
|
||||
"${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}"
|
||||
// Prove Relay Asia Staging's load mode drives traffic only and must not hold the staging lock.
|
||||
export const STAGING_OR_LOAD_GROUP =
|
||||
"${{ inputs.mode == 'load' && 'relay-staging-load' || 'relay-staging-mutation' }}"
|
||||
|
||||
const selectable = (production, staging) =>
|
||||
`\${{ inputs.environment == 'production' && '${production}' || '${staging}' }}`
|
||||
|
||||
export const SELECTABLE_LEASE = {
|
||||
bucket: selectable(PRODUCTION_LEASE.bucket, STAGING_LEASE.bucket),
|
||||
object: selectable(PRODUCTION_LEASE.object, STAGING_LEASE.object)
|
||||
}
|
||||
|
||||
export const LOCK_GROUPS = new Set([
|
||||
PRODUCTION_GROUP,
|
||||
STAGING_GROUP,
|
||||
SELECTABLE_GROUP,
|
||||
STAGING_OR_LOAD_GROUP
|
||||
])
|
||||
|
||||
export function readWorkflow(file) {
|
||||
return readFileSync(new URL(file, WORKFLOW_ROOT), 'utf8')
|
||||
}
|
||||
|
||||
export function workflowFiles() {
|
||||
return readdirSync(WORKFLOW_ROOT)
|
||||
.filter((name) => name.endsWith('.yml') && name.startsWith(RELAY_WORKFLOW_FILE_PREFIX))
|
||||
.sort()
|
||||
}
|
||||
|
||||
// --- YAML-shaped readers (line based; the workflows are hand-written and uniformly indented) ---
|
||||
|
||||
function indentOf(line) {
|
||||
return line.length - line.trimStart().length
|
||||
}
|
||||
|
||||
function blockAfter(lines, index) {
|
||||
const base = indentOf(lines[index])
|
||||
const body = []
|
||||
for (let i = index + 1; i < lines.length; i += 1) {
|
||||
if (lines[i].trim() === '') {
|
||||
body.push(lines[i])
|
||||
continue
|
||||
}
|
||||
if (indentOf(lines[i]) <= base) break
|
||||
body.push(lines[i])
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
export function concurrencyBlocks(text) {
|
||||
const lines = text.split('\n')
|
||||
const blocks = []
|
||||
lines.forEach((line, index) => {
|
||||
if (line.trim() !== 'concurrency:') return
|
||||
const body = blockAfter(lines, index)
|
||||
blocks.push({
|
||||
group: body.find((l) => l.trim().startsWith('group:'))?.trim().slice('group:'.length).trim(),
|
||||
cancelInProgress: body
|
||||
.find((l) => l.trim().startsWith('cancel-in-progress:'))
|
||||
?.trim()
|
||||
.slice('cancel-in-progress:'.length)
|
||||
.trim()
|
||||
})
|
||||
})
|
||||
return blocks
|
||||
}
|
||||
|
||||
export function jobs(text) {
|
||||
const lines = text.split('\n')
|
||||
const start = lines.findIndex((line) => line === 'jobs:')
|
||||
if (start === -1) return []
|
||||
const found = []
|
||||
for (let i = start + 1; i < lines.length; i += 1) {
|
||||
const match = /^ {2}([A-Za-z0-9_-]+):\s*$/.exec(lines[i])
|
||||
if (!match) continue
|
||||
found.push({ id: match[1], start: i, body: blockAfter(lines, i) })
|
||||
}
|
||||
return found.map((job) => ({ ...job, text: job.body.join('\n') }))
|
||||
}
|
||||
|
||||
function scalarField(jobText, key) {
|
||||
const lines = jobText.split('\n')
|
||||
const index = lines.findIndex((line) => /^ {4}[A-Za-z-]+:/.test(line) && line.trim().startsWith(`${key}:`))
|
||||
if (index === -1) return undefined
|
||||
const inline = lines[index].trim().slice(`${key}:`.length).trim()
|
||||
if (inline !== '' && inline !== '>-' && inline !== '|') return inline
|
||||
return blockAfter(lines, index).join(' ').replace(/\s+/g, ' ').trim()
|
||||
}
|
||||
|
||||
export function jobNeeds(jobText) {
|
||||
const raw = scalarField(jobText, 'needs')
|
||||
if (!raw) return []
|
||||
return raw
|
||||
.replace(/^\[|\]$/g, '')
|
||||
.split(/[,\n]|\s+-\s+/)
|
||||
.map((entry) => entry.replace(/^-/, '').trim())
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
export function jobIf(jobText) {
|
||||
return scalarField(jobText, 'if') ?? ''
|
||||
}
|
||||
|
||||
export function leaseSteps(text) {
|
||||
const lines = text.split('\n')
|
||||
const steps = []
|
||||
lines.forEach((line, index) => {
|
||||
if (line.trim() !== `- uses: ${LEASE_ACTION}`) return
|
||||
const body = blockAfter(lines, index)
|
||||
const read = (key) =>
|
||||
body.find((l) => l.trim().startsWith(`${key}:`))?.trim().slice(`${key}:`.length).trim()
|
||||
steps.push({
|
||||
line: index + 1,
|
||||
bucket: read('bucket'),
|
||||
object: read('object'),
|
||||
release: read('release')
|
||||
})
|
||||
})
|
||||
return steps
|
||||
}
|
||||
|
||||
export function leaseStepsByJob(file) {
|
||||
const text = readWorkflow(file)
|
||||
const steps = leaseSteps(text)
|
||||
return jobs(text).map((job) => ({
|
||||
id: job.id,
|
||||
steps: steps.filter((step) => step.line > job.start + 1 && step.line <= job.start + 1 + job.body.length)
|
||||
}))
|
||||
}
|
||||
|
||||
// --- trigger and reusable-call graph ---
|
||||
|
||||
export function triggers(text) {
|
||||
const lines = text.split('\n')
|
||||
const index = lines.findIndex((line) => line === 'on:')
|
||||
if (index === -1) return []
|
||||
return blockAfter(lines, index)
|
||||
.map((line) => /^ {2}([a-z_]+):/.exec(line)?.[1])
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
export function isEntrypoint(text) {
|
||||
return triggers(text).some((trigger) => trigger !== 'workflow_call')
|
||||
}
|
||||
|
||||
export function reusableCalls(text) {
|
||||
const counts = new Map()
|
||||
for (const match of text.matchAll(/uses: \.\/\.github\/workflows\/([A-Za-z0-9._-]+\.yml)/g)) {
|
||||
counts.set(match[1], (counts.get(match[1]) ?? 0) + 1)
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
export function entrypointsFor(file, seen = new Set()) {
|
||||
if (seen.has(file)) return new Set()
|
||||
seen.add(file)
|
||||
if (isEntrypoint(readWorkflow(file))) return new Set([file])
|
||||
const reached = new Set()
|
||||
for (const candidate of workflowFiles()) {
|
||||
if (candidate === file) continue
|
||||
if (!reusableCalls(readWorkflow(candidate)).has(file)) continue
|
||||
for (const entry of entrypointsFor(candidate, seen)) reached.add(entry)
|
||||
}
|
||||
return reached
|
||||
}
|
||||
|
||||
// --- what counts as a Cloud SQL connection-budget rollout ---
|
||||
|
||||
const COMMAND_PREFIX = /^(?:-\s+)?(?:run:\s*)?(?:[a-z_]+\s*=\s*"?\$\(\s*)?(?:if\s+|then\s+|else\s+|&&\s+|\|\|\s+|!\s+)*/
|
||||
|
||||
function commandLines(text) {
|
||||
return text.split('\n').map((line) => line.trim().replace(COMMAND_PREFIX, ''))
|
||||
}
|
||||
|
||||
export function appliesTerraform(text) {
|
||||
return commandLines(text).some((line) => /^terraform\b.*\bapply\b/.test(line))
|
||||
}
|
||||
|
||||
export function runsCloudRunMutation(text) {
|
||||
return commandLines(text).some((line) =>
|
||||
/^gcloud run (?:deploy\b|services (?:update|replace)\b|jobs (?:update|deploy)\b)/.test(line)
|
||||
)
|
||||
}
|
||||
|
||||
// Scripts that mint a Cloud Run revision, plus every script that re-exports one of them.
|
||||
export function revisionMintingScripts() {
|
||||
const self = new URL(import.meta.url).pathname.split('/').pop()
|
||||
const names = readdirSync(SCRIPT_ROOT).filter(
|
||||
(name) => name.endsWith('.mjs') && !name.endsWith('.test.mjs') && name !== self
|
||||
)
|
||||
const source = new Map(
|
||||
names.map((name) => [name, readFileSync(new URL(name, SCRIPT_ROOT), 'utf8')])
|
||||
)
|
||||
const minting = new Set(
|
||||
names.filter((name) => {
|
||||
const text = source.get(name)
|
||||
return (
|
||||
text.includes("'--no-traffic'") ||
|
||||
/'run',\s*'services',\s*'update'/.test(text) ||
|
||||
/'run',\s*'deploy'/.test(text)
|
||||
)
|
||||
})
|
||||
)
|
||||
for (let changed = true; changed; ) {
|
||||
changed = false
|
||||
for (const name of names) {
|
||||
if (minting.has(name)) continue
|
||||
const imports = [...source.get(name).matchAll(/from '\.\/([A-Za-z0-9._-]+\.mjs)'/g)].map(
|
||||
(match) => match[1]
|
||||
)
|
||||
if (!imports.some((imported) => minting.has(imported))) continue
|
||||
minting.add(name)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return minting
|
||||
}
|
||||
|
||||
export function mutatesSharedInstance(text, minters = revisionMintingScripts()) {
|
||||
if (appliesTerraform(text)) return 'terraform apply against the reviewed relay cell templates'
|
||||
if (runsCloudRunMutation(text)) return 'gcloud mints or replaces a Cloud Run revision'
|
||||
for (const script of minters) {
|
||||
if (text.includes(`dev/scripts/${script}`)) return `runs ${script}, which mints a Cloud Run revision`
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
// --- the declared contract ---
|
||||
|
||||
const production = (extra = {}) => ({ env: 'production', group: PRODUCTION_GROUP, ...extra })
|
||||
const staging = (extra = {}) => ({ env: 'staging', group: STAGING_GROUP, ...extra })
|
||||
const eitherEnvironment = () => ({ env: 'selectable', group: SELECTABLE_GROUP })
|
||||
|
||||
// Keys are workflow filenames, which the public copy prefixes; the prefix lives in one place.
|
||||
const named = (entries) =>
|
||||
Object.fromEntries(
|
||||
entries.map(([file, entry]) => [
|
||||
relayWorkflowFile(file),
|
||||
entry.leaseFiles
|
||||
? { ...entry, leaseFiles: entry.leaseFiles.map((member) => relayWorkflowFile(member)) }
|
||||
: entry
|
||||
])
|
||||
)
|
||||
|
||||
export const LEASED_WORKFLOWS = named([
|
||||
['deploy-relay-fence-broker.yml', production()],
|
||||
['deploy-relay-production.yml', production()],
|
||||
['deploy-relay-production-director.yml', production()],
|
||||
['deploy-relay-production-multi-target.yml', production()],
|
||||
[
|
||||
'deploy-relay-production-capacity.yml',
|
||||
production({
|
||||
leaseFiles: ['deploy-relay-production-capacity-job.yml'],
|
||||
reentrant: true
|
||||
})
|
||||
],
|
||||
[
|
||||
'deploy-relay-production-same-cap.yml',
|
||||
production({
|
||||
leaseFiles: ['deploy-relay-production-same-cap-job.yml'],
|
||||
reentrant: true
|
||||
})
|
||||
],
|
||||
[
|
||||
'operate-relay-production-rehome.yml',
|
||||
production({ leaseFiles: ['operate-relay-production-rehome-job.yml'] })
|
||||
],
|
||||
['deploy-relay-asia-topology.yml', eitherEnvironment()],
|
||||
['operate-relay-asia-admission.yml', eitherEnvironment()],
|
||||
['deploy-relay-staging.yml', staging()],
|
||||
['deploy-relay-staging-gce-candidate.yml', staging()],
|
||||
['bootstrap-relay-staging-capacity.yml', staging()],
|
||||
['power-relay-staging.yml', staging()],
|
||||
['prove-relay-asia-staging.yml', staging({ group: STAGING_OR_LOAD_GROUP })],
|
||||
[
|
||||
'prove-relay-staging-capacity.yml',
|
||||
staging({ exclusiveBy: "inputs.mode == 'refresh-asia-c4-image'" })
|
||||
],
|
||||
['recover-relay-staging-c4-image.yml', staging()]
|
||||
])
|
||||
|
||||
export const NOT_A_CLOUD_SQL_CANDIDATE = named([
|
||||
['push-deploy.yml', 'Push uses dedicated SQL and its own production-push-rollout group and durable push-rollout lease.'],
|
||||
[
|
||||
'monitor-relay-production.yml',
|
||||
'Read-only. Its identity holds monitoring, logging, Cloud SQL and compute viewer roles only, and it runs `gcloud sql instances describe`, never a mutation. It consumes no connection budget, so the durable lease would only let monitoring block a rollout and a rollout block monitoring.'
|
||||
]
|
||||
])
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,876 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import {
|
||||
inspectAdmissionSelector,
|
||||
selectorCellState,
|
||||
transitionAdmissionSelector
|
||||
} from './relay-admission-selector.mjs'
|
||||
import { isReserveCellRefusal, reserveCellRollRefusal } from './relay-reserve-cell-refusal.mjs'
|
||||
|
||||
const DEFAULT_POLL_INTERVAL_MS = 5_000
|
||||
const DEFAULT_TIMEOUT_MS = 14 * 60 * 1_000
|
||||
const ADMIN_RETRY_ATTEMPTS = 3
|
||||
const ADMIN_RETRY_BASE_MS = 250
|
||||
const CONNECTION_CONTROL_REBIND_RESERVE = 100
|
||||
const SUPPORTED_CONNECTION_HARD_CAPS = new Set([600, 1_000, 3_000])
|
||||
const RETRYABLE_ADMIN_PATHS = new Set([
|
||||
'/v1/admin/runtime-status',
|
||||
'/v1/admin/cell-status',
|
||||
'/v1/admin/evacuation-capacity',
|
||||
'/v1/admin/evacuation-status'
|
||||
])
|
||||
|
||||
function canonicalOrigin(value, name) {
|
||||
const url = new URL(value)
|
||||
if (url.protocol !== 'https:' || url.origin !== value || url.pathname !== '/') {
|
||||
throw new Error(`${name} must be a canonical HTTPS origin`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function adminAudience(value) {
|
||||
const url = new URL(value)
|
||||
if (
|
||||
url.protocol !== 'https:' ||
|
||||
url.pathname !== '/v1/admin/drain' ||
|
||||
url.search ||
|
||||
url.hash ||
|
||||
url.toString() !== value
|
||||
) {
|
||||
throw new Error('--admin-audience must be the canonical HTTPS director drain URL')
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function positiveInteger(value, name, maximum = Number.MAX_SAFE_INTEGER) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isInteger(parsed) || parsed <= 0 || parsed > maximum) {
|
||||
throw new Error(`${name} must be a positive integer`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
function nonnegativeInteger(value, name) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isInteger(parsed) || parsed < 0) {
|
||||
throw new Error(`${name} must be a nonnegative integer`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
export function parseArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`)
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of [
|
||||
'project',
|
||||
'director-origin',
|
||||
'admin-audience',
|
||||
'topology-file',
|
||||
'source-cell-id',
|
||||
'target-cell-id',
|
||||
'runtime-service-account',
|
||||
'mode'
|
||||
]) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
if (
|
||||
![
|
||||
'audit',
|
||||
'preflight',
|
||||
'recover-forward',
|
||||
'continue-evacuation',
|
||||
'disable-cell',
|
||||
'execute',
|
||||
'reset-empty-candidate',
|
||||
'enable-empty-cell'
|
||||
].includes(values.mode)
|
||||
) {
|
||||
throw new Error(
|
||||
'--mode must be audit, preflight, recover-forward, continue-evacuation, disable-cell, execute, reset-empty-candidate, or enable-empty-cell'
|
||||
)
|
||||
}
|
||||
return {
|
||||
project: values.project,
|
||||
directorOrigin: canonicalOrigin(values['director-origin'], '--director-origin'),
|
||||
adminAudience: adminAudience(values['admin-audience']),
|
||||
topologyFile: values['topology-file'],
|
||||
sourceCellId: values['source-cell-id'],
|
||||
targetCellId: values['target-cell-id'],
|
||||
runtimeServiceAccount: values['runtime-service-account'],
|
||||
mode: values.mode,
|
||||
batchSize: positiveInteger(values['batch-size'] ?? 100, '--batch-size', 100),
|
||||
drainGraceMs: positiveInteger(
|
||||
values['drain-grace-ms'] ?? 120_000,
|
||||
'--drain-grace-ms',
|
||||
60 * 60 * 1_000
|
||||
),
|
||||
pollIntervalMs: positiveInteger(
|
||||
values['poll-interval-ms'] ?? DEFAULT_POLL_INTERVAL_MS,
|
||||
'--poll-interval-ms',
|
||||
60_000
|
||||
),
|
||||
timeoutMs: positiveInteger(
|
||||
values['timeout-ms'] ?? DEFAULT_TIMEOUT_MS,
|
||||
'--timeout-ms',
|
||||
60 * 60 * 1_000
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function deployment(value, cellId) {
|
||||
if (!value || typeof value !== 'object') throw new Error(`missing topology for ${cellId}`)
|
||||
const expected = {
|
||||
cellId,
|
||||
origin: canonicalOrigin(value.origin, `${cellId} origin`),
|
||||
region: String(value.region ?? 'us-central1'),
|
||||
zone: String(value.zone ?? ''),
|
||||
migName: String(value.mig_name ?? ''),
|
||||
instanceGroup: String(value.instance_group ?? ''),
|
||||
backendName: String(value.backend_name ?? ''),
|
||||
backendId: String(value.backend_id ?? ''),
|
||||
urlMapName: String(value.url_map_name ?? ''),
|
||||
generationIdentity: String(value.generation_identity ?? ''),
|
||||
image: String(value.image ?? ''),
|
||||
imageDigest: String(value.image ?? '').split('@')[1] ?? '',
|
||||
capacityRequests: positiveInteger(value.capacity_requests, `${cellId} capacity`),
|
||||
databasePoolMax: positiveInteger(
|
||||
value.database_pool_max ?? 10,
|
||||
`${cellId} database pool maximum`,
|
||||
100
|
||||
),
|
||||
connectionHardCap:
|
||||
value.connection_hard_cap === null || value.connection_hard_cap === undefined
|
||||
? undefined
|
||||
: positiveInteger(value.connection_hard_cap, `${cellId} connection hard cap`),
|
||||
connectionUnobservedBound:
|
||||
value.connection_unobserved_bound === null ||
|
||||
value.connection_unobserved_bound === undefined
|
||||
? undefined
|
||||
: nonnegativeInteger(
|
||||
value.connection_unobserved_bound,
|
||||
`${cellId} unobserved connection bound`
|
||||
),
|
||||
initiallyEnabled: value.initially_enabled,
|
||||
fenced: value.fenced,
|
||||
desiredTargetSize: value.desired_target_size
|
||||
}
|
||||
if (!/^[a-z0-9-]+$/.test(expected.zone)) throw new Error(`${cellId} has an invalid zone`)
|
||||
if (!['us-central1', 'asia-east2'].includes(expected.region) || !expected.zone.startsWith(`${expected.region}-`)) {
|
||||
throw new Error(`${cellId} has an invalid region`)
|
||||
}
|
||||
for (const [name, resource] of [
|
||||
['MIG', expected.migName],
|
||||
['instance group', expected.instanceGroup],
|
||||
['backend', expected.backendName],
|
||||
['backend ID', expected.backendId]
|
||||
]) {
|
||||
if (!resource) throw new Error(`${cellId} has no ${name}`)
|
||||
}
|
||||
if (!/^[a-z0-9.-]+\/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$/.test(expected.image)) {
|
||||
throw new Error(`${cellId} image is not digest-pinned`)
|
||||
}
|
||||
if (typeof expected.initiallyEnabled !== 'boolean') {
|
||||
throw new Error(`${cellId} has no initial admission state`)
|
||||
}
|
||||
if (
|
||||
(expected.connectionHardCap === undefined) !==
|
||||
(expected.connectionUnobservedBound === undefined) ||
|
||||
(expected.connectionHardCap !== undefined &&
|
||||
(!SUPPORTED_CONNECTION_HARD_CAPS.has(expected.connectionHardCap) ||
|
||||
expected.connectionUnobservedBound >=
|
||||
expected.connectionHardCap - CONNECTION_CONTROL_REBIND_RESERVE))
|
||||
) {
|
||||
throw new Error(`${cellId} has invalid connection capacity`)
|
||||
}
|
||||
return expected
|
||||
}
|
||||
|
||||
export function assertDeploymentConnectionCapacity(expected, runtime, director) {
|
||||
if (expected.connectionHardCap === undefined) {
|
||||
if (runtime !== null || director !== null) {
|
||||
throw new Error(`${expected.cellId} connection capacity differs from Terraform`)
|
||||
}
|
||||
return
|
||||
}
|
||||
const hardCap = expected.connectionHardCap
|
||||
const unobservedBound = expected.connectionUnobservedBound
|
||||
const ordinaryConnectionLimit = hardCap - CONNECTION_CONTROL_REBIND_RESERVE
|
||||
const normalAdmissionPause = ordinaryConnectionLimit - unobservedBound
|
||||
const matches = (capacity) =>
|
||||
capacity?.hardCap === hardCap &&
|
||||
capacity.controlRebindReserve === CONNECTION_CONTROL_REBIND_RESERVE &&
|
||||
capacity.ordinaryConnectionLimit === ordinaryConnectionLimit &&
|
||||
capacity.unobservedBound === unobservedBound &&
|
||||
capacity.normalAdmissionPause === normalAdmissionPause
|
||||
if (!matches(runtime) || !matches(director) || director.heartbeatFresh !== true) {
|
||||
throw new Error(`${expected.cellId} connection capacity differs from Terraform`)
|
||||
}
|
||||
}
|
||||
|
||||
export function selectDeployments(topology, sourceCellId, targetCellId) {
|
||||
if (sourceCellId === targetCellId) throw new Error('source and target cell IDs must differ')
|
||||
const source = deployment(topology[sourceCellId], sourceCellId)
|
||||
const target = deployment(topology[targetCellId], targetCellId)
|
||||
for (const key of ['origin', 'migName', 'instanceGroup', 'backendName', 'backendId']) {
|
||||
if (source[key] === target[key]) throw new Error(`source and target ${key} overlap`)
|
||||
}
|
||||
if (target.initiallyEnabled) throw new Error('candidate must be declared initially disabled')
|
||||
return { source, target }
|
||||
}
|
||||
|
||||
export function validateMig(mig, instances, expected) {
|
||||
if (Number(mig.targetSize) !== 1) throw new Error(`${expected.cellId} MIG is not fixed-one`)
|
||||
const policy = mig.updatePolicy ?? {}
|
||||
if (
|
||||
policy.replacementMethod !== 'RECREATE' ||
|
||||
Number(policy.maxSurge?.fixed ?? policy.maxSurge) !== 0 ||
|
||||
Number(policy.maxUnavailable?.fixed ?? policy.maxUnavailable) !== 1
|
||||
) {
|
||||
throw new Error(`${expected.cellId} MIG replacement policy is unsafe`)
|
||||
}
|
||||
const serving = instances.filter(
|
||||
(entry) => entry.instanceStatus === 'RUNNING' && entry.currentAction === 'NONE'
|
||||
)
|
||||
if (instances.length !== 1 || serving.length !== 1) {
|
||||
throw new Error(`${expected.cellId} MIG must have one running endpoint`)
|
||||
}
|
||||
return serving[0].instance.split('/').at(-1)
|
||||
}
|
||||
|
||||
export function validateInstance(instance, expected, runtimeServiceAccount) {
|
||||
const publicConfigs = (instance.networkInterfaces ?? []).flatMap(
|
||||
(network) => network.accessConfigs ?? []
|
||||
)
|
||||
if (publicConfigs.length !== 0) throw new Error(`${expected.cellId} instance has a public IP`)
|
||||
const serviceAccounts = (instance.serviceAccounts ?? []).map((entry) => entry.email)
|
||||
if (serviceAccounts.length !== 1 || serviceAccounts[0] !== runtimeServiceAccount) {
|
||||
throw new Error(`${expected.cellId} runtime service account mismatch`)
|
||||
}
|
||||
}
|
||||
|
||||
export function validateBackend(backend, expected) {
|
||||
if (
|
||||
backend.protocol !== 'HTTP' ||
|
||||
Number(backend.timeoutSec) !== 86_400 ||
|
||||
(backend.backends ?? []).length !== 1 ||
|
||||
backend.backends[0].group !== expected.instanceGroup
|
||||
) {
|
||||
throw new Error(`${expected.cellId} backend topology mismatch`)
|
||||
}
|
||||
}
|
||||
|
||||
export function defaultCommandJson(args) {
|
||||
const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`)
|
||||
}
|
||||
return JSON.parse(result.stdout)
|
||||
}
|
||||
|
||||
export function suppliedAdminIdentityToken(environment = process.env) {
|
||||
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (token === undefined) return null
|
||||
return validatedIdentityToken(token, 'admin')
|
||||
}
|
||||
|
||||
export function suppliedFenceMutationIdentityToken(environment = process.env) {
|
||||
const token = environment.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN
|
||||
if (token === undefined) return null
|
||||
return validatedIdentityToken(token, 'fence mutation')
|
||||
}
|
||||
|
||||
function validatedIdentityToken(token, label) {
|
||||
// WIF supplies a masked Google ID token because external-account gcloud cannot mint one directly.
|
||||
if (token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) {
|
||||
throw new Error(`invalid supplied ${label} identity token`)
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
export function defaultIdentityToken(audience) {
|
||||
const supplied = suppliedAdminIdentityToken()
|
||||
if (supplied !== null) return supplied
|
||||
const result = spawnSync(
|
||||
'gcloud',
|
||||
['auth', 'print-identity-token', `--audiences=${audience}`],
|
||||
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }
|
||||
)
|
||||
if (result.status !== 0) throw new Error('gcloud identity-token command failed')
|
||||
return result.stdout.trim()
|
||||
}
|
||||
|
||||
async function responseJson(response, label, cellId) {
|
||||
const body = await response.json().catch(() => ({ error: `http_${response.status}` }))
|
||||
if (isReserveCellRefusal(response.status, body)) throw reserveCellRollRefusal(cellId)
|
||||
if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
export function createAdminPost(config, deps, token) {
|
||||
return async (origin, path, body) => {
|
||||
const requestToken = typeof token === 'function' ? token(path) : token
|
||||
for (let attempt = 1; attempt <= ADMIN_RETRY_ATTEMPTS; attempt++) {
|
||||
let response
|
||||
try {
|
||||
response = await deps.fetch(`${origin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${requestToken}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
} catch (error) {
|
||||
if (!RETRYABLE_ADMIN_PATHS.has(path) || attempt === ADMIN_RETRY_ATTEMPTS) throw error
|
||||
deps.emit({ event: 'candidate_admin_retry', path, attempt, reason: 'transport' })
|
||||
await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1))
|
||||
continue
|
||||
}
|
||||
if (
|
||||
RETRYABLE_ADMIN_PATHS.has(path) &&
|
||||
([502, 503, 504].includes(response.status) ||
|
||||
(path === '/v1/admin/evacuation-status' && response.status === 500)) &&
|
||||
attempt < ADMIN_RETRY_ATTEMPTS
|
||||
) {
|
||||
// These endpoints are read-only or transactionally idempotent, so a
|
||||
// lost response may be retried without widening deployment authority.
|
||||
deps.emit({
|
||||
event: 'candidate_admin_retry',
|
||||
path,
|
||||
attempt,
|
||||
reason: `http_${response.status}`
|
||||
})
|
||||
await response.arrayBuffer().catch(() => undefined)
|
||||
await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1))
|
||||
continue
|
||||
}
|
||||
// The director names no cell, so a two-cell request names both.
|
||||
const cells =
|
||||
body?.cellId ?? [body?.sourceCellId, body?.targetCellId].filter(Boolean).join(' or ')
|
||||
return await responseJson(response, path, cells || null)
|
||||
}
|
||||
throw new Error(`${path} retry attempts exhausted`)
|
||||
}
|
||||
}
|
||||
|
||||
async function checkHttp(deps, origin, path) {
|
||||
const response = await deps.fetch(`${origin}${path}`, { signal: AbortSignal.timeout(15_000) })
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok || body.ok !== true) throw new Error(`${origin}${path} is unavailable`)
|
||||
}
|
||||
|
||||
export async function inspectCell(config, deps, adminPost, expected) {
|
||||
const common = ['--project', config.project, '--zone', expected.zone, '--format=json']
|
||||
const mig = deps.commandJson([
|
||||
'compute',
|
||||
'instance-groups',
|
||||
'managed',
|
||||
'describe',
|
||||
expected.migName,
|
||||
...common
|
||||
])
|
||||
const instances = deps.commandJson([
|
||||
'compute',
|
||||
'instance-groups',
|
||||
'managed',
|
||||
'list-instances',
|
||||
expected.migName,
|
||||
...common
|
||||
])
|
||||
const instanceName = validateMig(mig, instances, expected)
|
||||
const instance = deps.commandJson([
|
||||
'compute',
|
||||
'instances',
|
||||
'describe',
|
||||
instanceName,
|
||||
...common
|
||||
])
|
||||
validateInstance(instance, expected, config.runtimeServiceAccount)
|
||||
const backend = deps.commandJson([
|
||||
'compute',
|
||||
'backend-services',
|
||||
'describe',
|
||||
expected.backendName,
|
||||
'--global',
|
||||
'--project',
|
||||
config.project,
|
||||
'--format=json'
|
||||
])
|
||||
validateBackend(backend, expected)
|
||||
await checkHttp(deps, expected.origin, '/health')
|
||||
await checkHttp(deps, expected.origin, '/ready')
|
||||
const runtime = await adminPost(expected.origin, '/v1/admin/runtime-status', { v: 1 })
|
||||
if (
|
||||
runtime.role !== 'cell' ||
|
||||
runtime.cellId !== expected.cellId ||
|
||||
runtime.cellUrl !== expected.origin ||
|
||||
(runtime.region ?? 'us-central1') !== expected.region ||
|
||||
runtime.imageDigest !== expected.imageDigest
|
||||
) {
|
||||
throw new Error(`${expected.cellId} served runtime does not match Terraform topology`)
|
||||
}
|
||||
const status = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: expected.cellId
|
||||
})
|
||||
if (
|
||||
status.status?.cellUrl !== expected.origin ||
|
||||
(status.status?.region ?? 'us-central1') !== expected.region ||
|
||||
status.status?.runtime?.cellUrl !== expected.origin ||
|
||||
status.status?.runtime?.ready !== true ||
|
||||
status.status?.runtime?.heartbeatFresh !== true
|
||||
) {
|
||||
throw new Error(`${expected.cellId} has no fresh ready authenticated heartbeat`)
|
||||
}
|
||||
assertDeploymentConnectionCapacity(
|
||||
expected,
|
||||
runtime.connectionCapacity ?? null,
|
||||
status.status.connectionCapacity ?? null
|
||||
)
|
||||
return {
|
||||
...status.status,
|
||||
draining: runtime.draining === true,
|
||||
process: runtime.runtime ?? null,
|
||||
runtimeConnectionCapacity: runtime.connectionCapacity ?? null
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForMigration(config, deps, adminPost, completeReady) {
|
||||
const deadline = deps.now() + config.timeoutMs
|
||||
while (deps.now() < deadline) {
|
||||
const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
|
||||
v: 1,
|
||||
sourceCellId: config.sourceCellId,
|
||||
targetCellId: config.targetCellId,
|
||||
completeReady
|
||||
})
|
||||
deps.emit({ event: completeReady ? 'migration_completion' : 'migration_registration', ...status })
|
||||
if (completeReady ? status.inProgress === 0 : status.inProgress === status.targetRegistered) {
|
||||
return status
|
||||
}
|
||||
if (
|
||||
completeReady &&
|
||||
status.targetRegistered === status.inProgress &&
|
||||
status.registeredSourceActive === 0 &&
|
||||
status.registeredCompletable === 0 &&
|
||||
status.registeredTargetInactive === status.inProgress
|
||||
) {
|
||||
// CI waiting cannot revive an offline desktop; keep its proven migration
|
||||
// pending until that target control reconnects.
|
||||
return status
|
||||
}
|
||||
await deps.wait(config.pollIntervalMs)
|
||||
}
|
||||
throw new Error('timed out waiting for candidate migration')
|
||||
}
|
||||
|
||||
export async function setCellState(config, adminPost, cellId, enabled) {
|
||||
const post = async (path, body) => await adminPost(config.directorOrigin, path, body)
|
||||
const inspected = await inspectAdmissionSelector(post)
|
||||
if (inspected.selector.generation > 0) {
|
||||
await transitionAdmissionSelector(post, {
|
||||
[cellId]: enabled ? 'general' : 'existing-only'
|
||||
})
|
||||
return
|
||||
}
|
||||
await post('/v1/admin/cell-state', { v: 1, cellId, enabled })
|
||||
}
|
||||
|
||||
function assertNoDurableActivity(status, operation) {
|
||||
const activity = [
|
||||
status.assignments,
|
||||
status.activityLeases,
|
||||
status.reservedRequests,
|
||||
status.outgoingMigrations,
|
||||
status.incomingMigrations
|
||||
]
|
||||
if (activity.some((value) => Number(value) !== 0)) {
|
||||
throw new Error(`${operation} requires zero durable activity`)
|
||||
}
|
||||
}
|
||||
|
||||
async function recoverCandidateFailure(
|
||||
config,
|
||||
deps,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
allowEmptyAdmissionRollback,
|
||||
selectorActive
|
||||
) {
|
||||
const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
completeReady: false
|
||||
}).catch(() => null)
|
||||
if (!selectorActive && allowEmptyAdmissionRollback && status?.inProgress === 0) {
|
||||
await setCellState(config, adminPost, source.cellId, true).catch(() => undefined)
|
||||
await setCellState(config, adminPost, target.cellId, false).catch(() => undefined)
|
||||
return
|
||||
}
|
||||
if (!selectorActive && status && status.inProgress > 0 && status.targetRegistered === 0) {
|
||||
await setCellState(config, adminPost, source.cellId, true).catch(() => undefined)
|
||||
deps.emit({
|
||||
event: 'candidate_rollback_waiting_for_lease_expiry',
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
inProgress: status.inProgress
|
||||
})
|
||||
return
|
||||
}
|
||||
deps.emit({
|
||||
event: 'candidate_forward_recovery_required',
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
targetRegistered: status?.targetRegistered ?? null
|
||||
})
|
||||
}
|
||||
|
||||
export async function drainSource(
|
||||
config,
|
||||
deps,
|
||||
token,
|
||||
source,
|
||||
graceMs = config.drainGraceMs,
|
||||
traceValue
|
||||
) {
|
||||
const response = await deps.fetch(`${source.origin}/v1/admin/drain`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
'content-type': 'application/json',
|
||||
...(traceValue ? { 'x-orca-drain-trace': traceValue } : {})
|
||||
},
|
||||
body: JSON.stringify({ v: 1, graceMs }),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
await responseJson(response, 'source drain')
|
||||
return {
|
||||
backendStatus: response.status,
|
||||
backendInstance: response.headers.get('x-orca-backend-instance') ?? undefined
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyCandidateCompletion(
|
||||
config,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
event,
|
||||
eventName = 'candidate_complete'
|
||||
) {
|
||||
const finalSource = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: source.cellId
|
||||
})
|
||||
const finalTarget = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: target.cellId
|
||||
})
|
||||
if (
|
||||
finalSource.status.activityLeases !== 0 ||
|
||||
finalSource.status.reservedRequests !== 0 ||
|
||||
finalSource.status.outgoingMigrations !== 0 ||
|
||||
finalSource.status.runtime?.observedRequests !== 0 ||
|
||||
finalTarget.status.incomingMigrations !== 0 ||
|
||||
finalTarget.status.reservedRequests !== finalTarget.status.activityRequestUnits
|
||||
) {
|
||||
throw new Error('aggregate post-migration counts are not reconciled')
|
||||
}
|
||||
event({
|
||||
event: eventName,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
dormantSourceAssignments: finalSource.status.assignments,
|
||||
targetAssignments: finalTarget.status.assignments,
|
||||
targetActivityLeases: finalTarget.status.activityLeases,
|
||||
targetReservedRequests: finalTarget.status.reservedRequests
|
||||
})
|
||||
}
|
||||
|
||||
export async function runCandidateDeployment(config, overrides = {}) {
|
||||
const deps = {
|
||||
commandJson: overrides.commandJson ?? defaultCommandJson,
|
||||
identityToken: overrides.identityToken ?? defaultIdentityToken,
|
||||
fetch: overrides.fetch ?? fetch,
|
||||
emit:
|
||||
overrides.emit ??
|
||||
((event) => process.stdout.write(`${JSON.stringify(event)}\n`)),
|
||||
now: overrides.now ?? Date.now,
|
||||
wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
|
||||
random: overrides.random ?? Math.random
|
||||
}
|
||||
const topology = JSON.parse(readFileSync(config.topologyFile, 'utf8'))
|
||||
const { source, target } = selectDeployments(
|
||||
topology,
|
||||
config.sourceCellId,
|
||||
config.targetCellId
|
||||
)
|
||||
const token = deps.identityToken(config.adminAudience)
|
||||
const adminPost = createAdminPost(config, deps, token)
|
||||
const selectorPost = async (path, body) =>
|
||||
await adminPost(config.directorOrigin, path, body)
|
||||
const selectorInspection = await inspectAdmissionSelector(selectorPost)
|
||||
const selectorActive = selectorInspection.selector.generation > 0
|
||||
const sourceStatus = await inspectCell(config, deps, adminPost, source)
|
||||
const targetStatus = await inspectCell(config, deps, adminPost, target)
|
||||
const sourceAdmission = selectorActive
|
||||
? selectorCellState(selectorInspection.selector, source.cellId)
|
||||
: sourceStatus.enabled
|
||||
? 'general'
|
||||
: 'existing-only'
|
||||
const targetAdmission = selectorActive
|
||||
? selectorCellState(selectorInspection.selector, target.cellId)
|
||||
: targetStatus.enabled
|
||||
? 'general'
|
||||
: 'existing-only'
|
||||
if (config.mode === 'audit') {
|
||||
const migration = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
completeReady: false
|
||||
})
|
||||
// Forward recovery needs durable aggregate evidence without exposing assignment identities.
|
||||
deps.emit({
|
||||
event: 'candidate_audit',
|
||||
source: aggregateCellStatus(sourceStatus),
|
||||
target: aggregateCellStatus(targetStatus),
|
||||
migration
|
||||
})
|
||||
return
|
||||
}
|
||||
if (config.mode === 'recover-forward') {
|
||||
if (
|
||||
selectorActive
|
||||
? sourceAdmission !== 'existing-only' || targetAdmission !== 'migration-only'
|
||||
: sourceStatus.enabled || !targetStatus.enabled
|
||||
) {
|
||||
throw new Error(
|
||||
selectorActive
|
||||
? 'forward recovery requires existing-only source and migration-only target'
|
||||
: 'forward recovery requires disabled source and enabled target'
|
||||
)
|
||||
}
|
||||
await drainSource(config, deps, token, source)
|
||||
await waitForMigration(config, deps, adminPost, false)
|
||||
const completion = await waitForMigration(config, deps, adminPost, true)
|
||||
if (completion.inProgress > 0) {
|
||||
deps.emit({
|
||||
event: 'candidate_forward_pending',
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
inProgress: completion.inProgress,
|
||||
registeredSourceActive: completion.registeredSourceActive,
|
||||
registeredCompletable: completion.registeredCompletable,
|
||||
registeredTargetInactive: completion.registeredTargetInactive
|
||||
})
|
||||
throw new Error('forward recovery remains pending for inactive target controls')
|
||||
}
|
||||
await verifyCandidateCompletion(
|
||||
config,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
deps.emit,
|
||||
'candidate_forward_recovered'
|
||||
)
|
||||
return
|
||||
}
|
||||
if (config.mode === 'disable-cell') {
|
||||
if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false)
|
||||
// Disabling new admission preserves origin-owned sessions and durable recovery work.
|
||||
deps.emit({
|
||||
event: 'cell_admission_disabled',
|
||||
targetCellId: target.cellId,
|
||||
changed: targetStatus.enabled,
|
||||
assignments: targetStatus.assignments,
|
||||
activityLeases: targetStatus.activityLeases,
|
||||
reservedRequests: targetStatus.reservedRequests,
|
||||
outgoingMigrations: targetStatus.outgoingMigrations,
|
||||
incomingMigrations: targetStatus.incomingMigrations
|
||||
})
|
||||
return
|
||||
}
|
||||
// Repair is safe only before a candidate owns assignments or origin-scoped work.
|
||||
if (config.mode === 'reset-empty-candidate') {
|
||||
assertNoDurableActivity(targetStatus, 'candidate admission reset')
|
||||
if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false)
|
||||
deps.emit({
|
||||
event: 'candidate_admission_reset',
|
||||
targetCellId: target.cellId,
|
||||
changed: targetStatus.enabled
|
||||
})
|
||||
return
|
||||
}
|
||||
if (config.mode === 'enable-empty-cell') {
|
||||
assertNoDurableActivity(targetStatus, 'cell admission enable')
|
||||
if (selectorActive ? targetAdmission === 'general' : targetStatus.enabled) {
|
||||
deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: false })
|
||||
return
|
||||
}
|
||||
}
|
||||
const continuingEvacuation = config.mode === 'continue-evacuation'
|
||||
if (continuingEvacuation) {
|
||||
if (
|
||||
selectorActive
|
||||
? targetAdmission !== 'migration-only'
|
||||
: !targetStatus.enabled
|
||||
) {
|
||||
throw new Error(
|
||||
selectorActive
|
||||
? 'continued evacuation requires migration-only target'
|
||||
: 'continued evacuation requires enabled target'
|
||||
)
|
||||
}
|
||||
} else if (!selectorActive && targetStatus.enabled) {
|
||||
throw new Error('candidate cell is already enabled')
|
||||
} else if (
|
||||
selectorActive &&
|
||||
!['migration-only', 'existing-only'].includes(targetAdmission)
|
||||
) {
|
||||
throw new Error('candidate cell must not be generally admitted')
|
||||
}
|
||||
const capacity = await adminPost(config.directorOrigin, '/v1/admin/evacuation-capacity', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId
|
||||
})
|
||||
if (capacity.requiredTargetUnits > capacity.availableTargetUnits) {
|
||||
throw new Error('candidate lacks survivor request-unit headroom')
|
||||
}
|
||||
deps.emit({
|
||||
event: 'candidate_preflight',
|
||||
mode: config.mode,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
sourceOrigin: source.origin,
|
||||
targetOrigin: target.origin,
|
||||
sourceMig: source.migName,
|
||||
targetMig: target.migName,
|
||||
sourceBackend: source.backendName,
|
||||
targetBackend: target.backendName,
|
||||
sourceDigest: source.imageDigest,
|
||||
targetDigest: target.imageDigest,
|
||||
sourceAssignments: capacity.sourceAssignments,
|
||||
requiredTargetUnits: capacity.requiredTargetUnits,
|
||||
availableTargetUnits: capacity.availableTargetUnits
|
||||
})
|
||||
if (config.mode === 'preflight') return
|
||||
if (config.mode === 'enable-empty-cell') {
|
||||
await setCellState(config, adminPost, target.cellId, true)
|
||||
deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: true })
|
||||
return
|
||||
}
|
||||
// Fresh execution starts from source-only admission; continuation preserves its target.
|
||||
if (
|
||||
!continuingEvacuation &&
|
||||
(selectorActive ? sourceAdmission !== 'existing-only' : !sourceStatus.enabled)
|
||||
) {
|
||||
throw new Error(
|
||||
selectorActive ? 'source cell is not existing-only' : 'source cell is not enabled'
|
||||
)
|
||||
}
|
||||
if (selectorActive && targetAdmission !== 'migration-only') {
|
||||
throw new Error('target cell is not migration-only')
|
||||
}
|
||||
|
||||
let migrationsStarted = 0
|
||||
try {
|
||||
if (!selectorActive) {
|
||||
if (sourceStatus.enabled) await setCellState(config, adminPost, source.cellId, false)
|
||||
if (!targetStatus.enabled) await setCellState(config, adminPost, target.cellId, true)
|
||||
}
|
||||
for (;;) {
|
||||
const result = await adminPost(config.directorOrigin, '/v1/admin/evacuate-cell', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
limit: config.batchSize
|
||||
})
|
||||
migrationsStarted += result.started
|
||||
deps.emit({ event: 'migration_batch', started: result.started, totalStarted: migrationsStarted })
|
||||
if (result.started === 0) break
|
||||
}
|
||||
} catch (error) {
|
||||
await recoverCandidateFailure(
|
||||
config,
|
||||
deps,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
!continuingEvacuation,
|
||||
selectorActive
|
||||
)
|
||||
throw error
|
||||
}
|
||||
|
||||
try {
|
||||
if (selectorActive) {
|
||||
const currentSelector = await inspectAdmissionSelector(selectorPost)
|
||||
if (
|
||||
currentSelector.selector.generation !== selectorInspection.selector.generation ||
|
||||
JSON.stringify(currentSelector.selector.membership) !==
|
||||
JSON.stringify(selectorInspection.selector.membership)
|
||||
) {
|
||||
throw new Error('admission selector changed before drain')
|
||||
}
|
||||
}
|
||||
await drainSource(config, deps, token, source)
|
||||
await waitForMigration(config, deps, adminPost, false)
|
||||
const completion = await waitForMigration(config, deps, adminPost, true)
|
||||
if (completion.inProgress > 0) {
|
||||
throw new Error('candidate migration remains pending for inactive target controls')
|
||||
}
|
||||
} catch (error) {
|
||||
// A completion response can be lost after its transaction commits. Never
|
||||
// reverse admission here merely because no in-progress row remains.
|
||||
await recoverCandidateFailure(
|
||||
config,
|
||||
deps,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
false,
|
||||
selectorActive
|
||||
)
|
||||
throw error
|
||||
}
|
||||
|
||||
await verifyCandidateCompletion(config, adminPost, source, target, deps.emit)
|
||||
}
|
||||
|
||||
export function aggregateCellStatus(status) {
|
||||
return {
|
||||
cellId: status.cellId,
|
||||
enabled: status.enabled,
|
||||
assignments: status.assignments,
|
||||
activityLeases: status.activityLeases,
|
||||
activityRequestUnits: status.activityRequestUnits,
|
||||
reservedRequests: status.reservedRequests,
|
||||
outgoingMigrations: status.outgoingMigrations,
|
||||
incomingMigrations: status.incomingMigrations,
|
||||
runtimeReady: status.runtime?.ready ?? false,
|
||||
heartbeatFresh: status.runtime?.heartbeatFresh ?? false,
|
||||
observedRequests: status.runtime?.observedRequests ?? null
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
await runCandidateDeployment(parseArguments(argv))
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,922 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
assertDeploymentConnectionCapacity,
|
||||
createAdminPost,
|
||||
deployment,
|
||||
parseArguments,
|
||||
runCandidateDeployment,
|
||||
selectDeployments,
|
||||
suppliedAdminIdentityToken,
|
||||
validateBackend,
|
||||
validateInstance,
|
||||
validateMig
|
||||
} from './deploy-relay-gce-candidate.mjs'
|
||||
|
||||
const runtimeServiceAccount = 'orca-relay@example.iam.gserviceaccount.com'
|
||||
const digestA = `sha256:${'a'.repeat(64)}`
|
||||
const digestB = `sha256:${'b'.repeat(64)}`
|
||||
|
||||
test('retries evacuation-status HTTP 500 without widening other admin retries', async () => {
|
||||
const events = []
|
||||
let calls = 0
|
||||
const adminPost = createAdminPost({}, {
|
||||
fetch: async () => {
|
||||
calls++
|
||||
return calls === 1
|
||||
? new Response(JSON.stringify({ error: 'transient' }), { status: 500 })
|
||||
: new Response(JSON.stringify({ ok: true }), { status: 200 })
|
||||
},
|
||||
emit: (event) => events.push(event),
|
||||
wait: async () => {},
|
||||
random: () => 0
|
||||
}, 'token')
|
||||
assert.deepEqual(
|
||||
await adminPost('https://relay.example', '/v1/admin/evacuation-status', {}),
|
||||
{ ok: true }
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
assert.deepEqual(events, [{
|
||||
event: 'candidate_admin_retry',
|
||||
path: '/v1/admin/evacuation-status',
|
||||
attempt: 1,
|
||||
reason: 'http_500'
|
||||
}])
|
||||
|
||||
calls = 0
|
||||
await assert.rejects(
|
||||
createAdminPost({}, {
|
||||
fetch: async () => {
|
||||
calls++
|
||||
return new Response(JSON.stringify({ error: 'persistent' }), { status: 500 })
|
||||
},
|
||||
emit: () => {},
|
||||
wait: async () => {},
|
||||
random: () => 0
|
||||
}, 'token')('https://relay.example', '/v1/admin/runtime-status', {}),
|
||||
/runtime-status failed: persistent/
|
||||
)
|
||||
assert.equal(calls, 1)
|
||||
})
|
||||
|
||||
test('a reserve-cell refusal names the flip back to db; other 404s keep their error', async () => {
|
||||
const post = (body) => createAdminPost({}, {
|
||||
fetch: async () => Response.json(body, { status: 404 }),
|
||||
emit: () => {},
|
||||
wait: async () => {},
|
||||
random: () => 0
|
||||
}, 'token')
|
||||
await assert.rejects(
|
||||
post({ error: 'cell_in_reserve_mode' })(
|
||||
'https://relay.example',
|
||||
'/v1/admin/cell-status',
|
||||
{ v: 1, cellId: 'production-gce-c7' }
|
||||
),
|
||||
/production-gce-c7 is in reserve mode .*flip it to admitMode=db/
|
||||
)
|
||||
await assert.rejects(
|
||||
post({ error: 'cell_not_found' })(
|
||||
'https://relay.example',
|
||||
'/v1/admin/cell-status',
|
||||
{ v: 1, cellId: 'production-gce-c7' }
|
||||
),
|
||||
/cell-status failed: cell_not_found/
|
||||
)
|
||||
await assert.rejects(
|
||||
post({ error: 'cell_in_reserve_mode' })(
|
||||
'https://relay.example',
|
||||
'/v1/admin/evacuation-status',
|
||||
{ v: 1, sourceCellId: 'production-gce-c7', targetCellId: 'production-gce-c8' }
|
||||
),
|
||||
/production-gce-c7 or production-gce-c8 is in reserve mode/
|
||||
)
|
||||
})
|
||||
|
||||
test('verifies a 1,000-cap cell against both runtime and director telemetry', () => {
|
||||
const expected = deployment(
|
||||
{
|
||||
...topology().target,
|
||||
connection_hard_cap: 1_000,
|
||||
connection_unobserved_bound: 60
|
||||
},
|
||||
'target'
|
||||
)
|
||||
const capacity = {
|
||||
hardCap: 1_000,
|
||||
controlRebindReserve: 100,
|
||||
ordinaryConnectionLimit: 900,
|
||||
unobservedBound: 60,
|
||||
normalAdmissionPause: 840
|
||||
}
|
||||
assert.doesNotThrow(() =>
|
||||
assertDeploymentConnectionCapacity(expected, capacity, {
|
||||
...capacity,
|
||||
heartbeatFresh: true
|
||||
})
|
||||
)
|
||||
assert.throws(
|
||||
() =>
|
||||
assertDeploymentConnectionCapacity(expected, capacity, {
|
||||
...capacity,
|
||||
hardCap: 600,
|
||||
heartbeatFresh: true
|
||||
}),
|
||||
/differs from Terraform/
|
||||
)
|
||||
})
|
||||
|
||||
test('verifies a 3,000-cap regional cell with a 2,840 placement boundary', () => {
|
||||
const expected = deployment(
|
||||
{
|
||||
...topology().target,
|
||||
connection_hard_cap: 3_000,
|
||||
connection_unobserved_bound: 60
|
||||
},
|
||||
'target'
|
||||
)
|
||||
const capacity = {
|
||||
hardCap: 3_000,
|
||||
controlRebindReserve: 100,
|
||||
ordinaryConnectionLimit: 2_900,
|
||||
unobservedBound: 60,
|
||||
normalAdmissionPause: 2_840
|
||||
}
|
||||
|
||||
assert.doesNotThrow(() =>
|
||||
assertDeploymentConnectionCapacity(expected, capacity, {
|
||||
...capacity,
|
||||
heartbeatFresh: true
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
function topology() {
|
||||
return {
|
||||
source: {
|
||||
origin: 'https://c1.relay.example.com',
|
||||
zone: 'us-central1-b',
|
||||
mig_name: 'relay-c1',
|
||||
instance_group: 'https://compute.example/instanceGroups/relay-c1',
|
||||
backend_name: 'relay-c1',
|
||||
backend_id: 'https://compute.example/backendServices/relay-c1',
|
||||
image: `us-central1-docker.pkg.dev/project/repo/relay@${digestA}`,
|
||||
capacity_requests: 4_000,
|
||||
initially_enabled: true
|
||||
},
|
||||
target: {
|
||||
origin: 'https://c2.relay.example.com',
|
||||
zone: 'us-central1-c',
|
||||
mig_name: 'relay-c2',
|
||||
instance_group: 'https://compute.example/instanceGroups/relay-c2',
|
||||
backend_name: 'relay-c2',
|
||||
backend_id: 'https://compute.example/backendServices/relay-c2',
|
||||
image: `us-central1-docker.pkg.dev/project/repo/relay@${digestB}`,
|
||||
capacity_requests: 4_000,
|
||||
initially_enabled: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function withTopology(operation) {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-gce-candidate-'))
|
||||
const file = join(directory, 'topology.json')
|
||||
writeFileSync(file, JSON.stringify(topology()))
|
||||
return Promise.resolve(operation(file)).finally(() => rmSync(directory, { recursive: true }))
|
||||
}
|
||||
|
||||
function config(topologyFile, mode = 'preflight') {
|
||||
return {
|
||||
project: 'test-project',
|
||||
directorOrigin: 'https://relay.example.com',
|
||||
adminAudience: 'https://relay.example.com/v1/admin/drain',
|
||||
topologyFile,
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target',
|
||||
runtimeServiceAccount,
|
||||
mode,
|
||||
batchSize: 100,
|
||||
drainGraceMs: 120_000,
|
||||
pollIntervalMs: 1,
|
||||
timeoutMs: 1_000
|
||||
}
|
||||
}
|
||||
|
||||
function response(body, status = 200) {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'content-type': 'application/json' }
|
||||
})
|
||||
}
|
||||
|
||||
function migrationResponse(body) {
|
||||
return response({
|
||||
registeredSourceActive: 0,
|
||||
registeredCompletable: 0,
|
||||
registeredTargetInactive: 0,
|
||||
expiredUnregistered: 0,
|
||||
repairableExpiredUnregistered: 0,
|
||||
abortableExpiredUnregistered: 0,
|
||||
blockedExpiredUnregistered: 0,
|
||||
blockedExpiredOnNewerTargetAssignment: 0,
|
||||
...body
|
||||
})
|
||||
}
|
||||
|
||||
function fakeCommand(args) {
|
||||
const name = args[args.indexOf('describe') + 1] ?? args[args.indexOf('list-instances') + 1]
|
||||
if (args.includes('list-instances')) {
|
||||
return [
|
||||
{
|
||||
instance: `https://compute.example/instances/${name}-vm`,
|
||||
instanceStatus: 'RUNNING',
|
||||
currentAction: 'NONE'
|
||||
}
|
||||
]
|
||||
}
|
||||
if (args.includes('instance-groups')) {
|
||||
return {
|
||||
targetSize: 1,
|
||||
updatePolicy: {
|
||||
replacementMethod: 'RECREATE',
|
||||
maxSurge: { fixed: 0 },
|
||||
maxUnavailable: { fixed: 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (args.includes('instances')) {
|
||||
return {
|
||||
networkInterfaces: [{ networkIP: '10.42.0.2' }],
|
||||
serviceAccounts: [{ email: runtimeServiceAccount }]
|
||||
}
|
||||
}
|
||||
const cell = name === 'relay-c1' ? topology().source : topology().target
|
||||
return { protocol: 'HTTP', timeoutSec: 86_400, backends: [{ group: cell.instance_group }] }
|
||||
}
|
||||
|
||||
function harness({
|
||||
failTargetEnable = false,
|
||||
failDrain = false,
|
||||
failAfterRegistration = false,
|
||||
failCompletionResponse = false,
|
||||
sourceEnabled = true,
|
||||
targetEnabled = false,
|
||||
targetAssignments = 0,
|
||||
migrationInProgress = 0,
|
||||
migrationTargetRegistered = 0,
|
||||
migrationTargetInactive = 0,
|
||||
transientCompletionFailures = 0,
|
||||
dormantSourceAssignments = 0,
|
||||
selectorGeneration = 0
|
||||
} = {}) {
|
||||
const state = {
|
||||
source: {
|
||||
enabled: selectorGeneration > 0 ? false : sourceEnabled,
|
||||
assignments: 2,
|
||||
activityLeases: 2
|
||||
},
|
||||
target: {
|
||||
enabled: selectorGeneration > 0 ? true : targetEnabled,
|
||||
assignments: targetAssignments,
|
||||
activityLeases: targetAssignments
|
||||
}
|
||||
}
|
||||
const events = []
|
||||
const stateChanges = []
|
||||
let batch = 0
|
||||
let migrationCompleted = false
|
||||
let remainingTransientCompletionFailures = transientCompletionFailures
|
||||
const fetch = async (url, options = {}) => {
|
||||
const parsed = new URL(url)
|
||||
if (parsed.pathname === '/health' || parsed.pathname === '/ready') return response({ ok: true })
|
||||
const body = JSON.parse(options.body ?? '{}')
|
||||
if (parsed.pathname === '/v1/admin/runtime-status') {
|
||||
const target = parsed.origin.includes('c2.')
|
||||
return response({
|
||||
v: 1,
|
||||
role: 'cell',
|
||||
cellId: target ? 'target' : 'source',
|
||||
cellUrl: parsed.origin,
|
||||
imageDigest: target ? digestB : digestA
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/admission-selector/status') {
|
||||
return response({
|
||||
v: 1,
|
||||
selector: {
|
||||
generation: selectorGeneration,
|
||||
attemptId: null,
|
||||
membership: {
|
||||
existingOnly:
|
||||
selectorGeneration > 0
|
||||
? ['source']
|
||||
: Object.keys(state).filter((cellId) => !state[cellId].enabled),
|
||||
migrationOnly: selectorGeneration > 0 ? ['target'] : [],
|
||||
general:
|
||||
selectorGeneration > 0
|
||||
? []
|
||||
: Object.keys(state).filter((cellId) => state[cellId].enabled)
|
||||
}
|
||||
},
|
||||
intent: null
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/cell-status') {
|
||||
const cell = state[body.cellId]
|
||||
return response({
|
||||
v: 1,
|
||||
status: {
|
||||
cellId: body.cellId,
|
||||
cellUrl: topology()[body.cellId].origin,
|
||||
enabled: cell.enabled,
|
||||
admissionState:
|
||||
selectorGeneration > 0
|
||||
? body.cellId === 'source'
|
||||
? 'existing-only'
|
||||
: 'migration-only'
|
||||
: cell.enabled
|
||||
? 'general'
|
||||
: 'existing-only',
|
||||
assignments: cell.assignments,
|
||||
activityLeases: cell.activityLeases,
|
||||
activityRequestUnits: cell.activityLeases,
|
||||
reservedRequests: cell.activityLeases,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0,
|
||||
runtime: {
|
||||
cellUrl: topology()[body.cellId].origin,
|
||||
ready: true,
|
||||
heartbeatFresh: true,
|
||||
observedRequests: cell.activityLeases
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/evacuation-capacity') {
|
||||
return response({ sourceAssignments: 2, requiredTargetUnits: 4, availableTargetUnits: 4_000 })
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/cell-state') {
|
||||
stateChanges.push([body.cellId, body.enabled])
|
||||
if (failTargetEnable && body.cellId === 'target' && body.enabled) {
|
||||
return response({ error: 'injected_enable_failure' }, 409)
|
||||
}
|
||||
state[body.cellId].enabled = body.enabled
|
||||
return response({ ok: true })
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/evacuate-cell') {
|
||||
if (failAfterRegistration && batch > 0) {
|
||||
return response({ error: 'injected_batch_failure' }, 503)
|
||||
}
|
||||
const started = batch++ === 0 ? 2 : 0
|
||||
return response({ v: 1, started })
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/evacuation-status') {
|
||||
if (failAfterRegistration) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 1,
|
||||
registeredSourceActive: 1,
|
||||
completed: 0,
|
||||
blocked: 1
|
||||
})
|
||||
}
|
||||
if (failDrain) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 0,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (body.completeReady) {
|
||||
state.source.assignments = dormantSourceAssignments
|
||||
state.source.activityLeases = 0
|
||||
state.target.assignments = 2
|
||||
state.target.activityLeases = 2
|
||||
if (remainingTransientCompletionFailures > 0) {
|
||||
remainingTransientCompletionFailures--
|
||||
throw new TypeError('injected transient fetch failure')
|
||||
}
|
||||
if (migrationTargetInactive > 0) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: migrationTargetInactive,
|
||||
targetRegistered: migrationTargetInactive,
|
||||
registeredTargetInactive: migrationTargetInactive,
|
||||
completed: 0,
|
||||
blocked: migrationTargetInactive
|
||||
})
|
||||
}
|
||||
migrationCompleted = true
|
||||
if (failCompletionResponse) {
|
||||
return response({ error: 'injected_completion_response_failure' }, 503)
|
||||
}
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 0,
|
||||
targetRegistered: 0,
|
||||
completed: 2,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (migrationCompleted) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 0,
|
||||
targetRegistered: 0,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (batch === 0) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: migrationInProgress,
|
||||
targetRegistered: migrationTargetRegistered,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 2,
|
||||
registeredCompletable: 2,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/drain') {
|
||||
return failDrain ? response({ error: 'injected_drain_failure' }, 503) : response({ ok: true })
|
||||
}
|
||||
return response({ error: 'unexpected_request' }, 500)
|
||||
}
|
||||
return {
|
||||
overrides: {
|
||||
commandJson: fakeCommand,
|
||||
identityToken: () => 'secret-token-never-emitted',
|
||||
fetch,
|
||||
emit: (event) => events.push(event),
|
||||
wait: async () => undefined,
|
||||
random: () => 0
|
||||
},
|
||||
events,
|
||||
stateChanges,
|
||||
state
|
||||
}
|
||||
}
|
||||
|
||||
test('requires explicit dry-run/execute inputs and a distinct disabled candidate', () => {
|
||||
assert.throws(() => parseArguments([]), /missing --project/)
|
||||
assert.throws(
|
||||
() =>
|
||||
parseArguments([
|
||||
'--project',
|
||||
'project',
|
||||
'--director-origin',
|
||||
'https://relay.example.com',
|
||||
'--admin-audience',
|
||||
'https://relay.example.com/not-drain',
|
||||
'--topology-file',
|
||||
'topology.json',
|
||||
'--source-cell-id',
|
||||
'source',
|
||||
'--target-cell-id',
|
||||
'target',
|
||||
'--runtime-service-account',
|
||||
runtimeServiceAccount,
|
||||
'--mode',
|
||||
'preflight'
|
||||
]),
|
||||
/director drain URL/
|
||||
)
|
||||
assert.throws(() => selectDeployments(topology(), 'source', 'source'), /must differ/)
|
||||
const overlapping = topology()
|
||||
overlapping.target.backend_id = overlapping.source.backend_id
|
||||
assert.throws(() => selectDeployments(overlapping, 'source', 'target'), /backendId overlap/)
|
||||
const enabled = topology()
|
||||
enabled.target.initially_enabled = true
|
||||
assert.throws(() => selectDeployments(enabled, 'source', 'target'), /initially disabled/)
|
||||
})
|
||||
|
||||
test('accepts only a bounded JWT-shaped supplied admin identity token', () => {
|
||||
assert.equal(suppliedAdminIdentityToken({}), null)
|
||||
assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc')
|
||||
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' }))
|
||||
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' }))
|
||||
assert.throws(() =>
|
||||
suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` })
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects unsafe fixed-one topology, public IPs, and backend overlap', () => {
|
||||
const expected = selectDeployments(topology(), 'source', 'target').target
|
||||
assert.throws(() => validateMig({ targetSize: 2 }, [], expected), /fixed-one/)
|
||||
assert.throws(
|
||||
() =>
|
||||
validateInstance(
|
||||
{
|
||||
networkInterfaces: [{ accessConfigs: [{ natIP: '203.0.113.1' }] }],
|
||||
serviceAccounts: [{ email: runtimeServiceAccount }]
|
||||
},
|
||||
expected,
|
||||
runtimeServiceAccount
|
||||
),
|
||||
/public IP/
|
||||
)
|
||||
assert.throws(
|
||||
() => validateBackend({ protocol: 'HTTP', timeoutSec: 86_400, backends: [] }, expected),
|
||||
/topology mismatch/
|
||||
)
|
||||
})
|
||||
|
||||
test('preflights exact served digests and survivor headroom without mutating admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness()
|
||||
await runCandidateDeployment(config(file), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.equal(events[0].event, 'candidate_preflight')
|
||||
assert.equal(events[0].targetDigest, digestB)
|
||||
assert.equal(JSON.stringify(events).includes('secret-token'), false)
|
||||
})
|
||||
})
|
||||
|
||||
test('audits a partially committed migration without changing admission or completing rows', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'audit'), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(events, [
|
||||
{
|
||||
event: 'candidate_audit',
|
||||
source: {
|
||||
cellId: 'source',
|
||||
enabled: false,
|
||||
assignments: 2,
|
||||
activityLeases: 2,
|
||||
activityRequestUnits: 2,
|
||||
reservedRequests: 2,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0,
|
||||
runtimeReady: true,
|
||||
heartbeatFresh: true,
|
||||
observedRequests: 2
|
||||
},
|
||||
target: {
|
||||
cellId: 'target',
|
||||
enabled: true,
|
||||
assignments: 1,
|
||||
activityLeases: 1,
|
||||
activityRequestUnits: 1,
|
||||
reservedRequests: 1,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0,
|
||||
runtimeReady: true,
|
||||
heartbeatFresh: true,
|
||||
observedRequests: 1
|
||||
},
|
||||
migration: {
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 2,
|
||||
registeredSourceActive: 0,
|
||||
registeredCompletable: 0,
|
||||
registeredTargetInactive: 0,
|
||||
completed: 0,
|
||||
blocked: 0,
|
||||
expiredUnregistered: 0,
|
||||
repairableExpiredUnregistered: 0,
|
||||
abortableExpiredUnregistered: 0,
|
||||
blockedExpiredUnregistered: 0,
|
||||
blockedExpiredOnNewerTargetAssignment: 0
|
||||
}
|
||||
}
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
test('preflights with source admission disabled but still refuses execution', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ sourceEnabled: false })
|
||||
await runCandidateDeployment(config(file), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.equal(events.at(-1).event, 'candidate_preflight')
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/source cell is not enabled/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('explicitly resets only an empty declared candidate to disabled admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides)
|
||||
assert.deepEqual(stateChanges, [['target', false]])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'candidate_admission_reset',
|
||||
targetCellId: 'target',
|
||||
changed: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses to reset candidate admission while it owns durable activity', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness({ targetEnabled: true, targetAssignments: 1 })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides),
|
||||
/requires zero durable activity/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('disables only new admission while preserving durable candidate activity', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'disable-cell'), overrides)
|
||||
assert.deepEqual(stateChanges, [['target', false]])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'cell_admission_disabled',
|
||||
targetCellId: 'target',
|
||||
changed: true,
|
||||
assignments: 1,
|
||||
activityLeases: 1,
|
||||
reservedRequests: 1,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('explicitly enables only an empty preflighted cell for admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ sourceEnabled: false })
|
||||
await runCandidateDeployment(config(file, 'enable-empty-cell'), overrides)
|
||||
assert.deepEqual(stateChanges, [['target', true]])
|
||||
assert.equal(events.at(-2).event, 'candidate_preflight')
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'cell_admission_enabled',
|
||||
targetCellId: 'target',
|
||||
changed: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses to enable cell admission while it owns durable activity', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness({ targetAssignments: 1 })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'enable-empty-cell'), overrides),
|
||||
/requires zero durable activity/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('executes target-first evacuation and verifies aggregate drained counts', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness()
|
||||
await runCandidateDeployment(config(file, 'execute'), overrides)
|
||||
assert.deepEqual(stateChanges.slice(0, 2), [
|
||||
['source', false],
|
||||
['target', true]
|
||||
])
|
||||
assert.equal(events.at(-1).event, 'candidate_complete')
|
||||
assert.equal(events.at(-1).targetAssignments, 2)
|
||||
})
|
||||
})
|
||||
|
||||
test('executes within selector membership without legacy admission writes', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const testHarness = harness({ selectorGeneration: 1 })
|
||||
await runCandidateDeployment(config(file, 'execute'), testHarness.overrides)
|
||||
assert.deepEqual(testHarness.stateChanges, [])
|
||||
assert.equal(testHarness.state.source.enabled, false)
|
||||
assert.equal(testHarness.state.target.enabled, true)
|
||||
})
|
||||
})
|
||||
|
||||
test('never restores legacy general admission after selector-era failure', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const testHarness = harness({
|
||||
selectorGeneration: 1,
|
||||
failAfterRegistration: true
|
||||
})
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), testHarness.overrides),
|
||||
/injected_batch_failure/
|
||||
)
|
||||
assert.deepEqual(testHarness.stateChanges, [])
|
||||
assert.equal(testHarness.state.source.enabled, false)
|
||||
})
|
||||
})
|
||||
|
||||
test('continues a partial evacuation without resetting target admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: true,
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1,
|
||||
migrationInProgress: 1,
|
||||
migrationTargetRegistered: 1
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'continue-evacuation'), overrides)
|
||||
assert.deepEqual(stateChanges, [['source', false]])
|
||||
assert.equal(events.at(-1).event, 'candidate_complete')
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses continued evacuation unless target admission is already enabled', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness()
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'continue-evacuation'), overrides),
|
||||
/requires enabled target/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('preserves partial target admission when continued batching fails', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
failAfterRegistration: true,
|
||||
sourceEnabled: true,
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1,
|
||||
migrationInProgress: 1,
|
||||
migrationTargetRegistered: 1
|
||||
})
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'continue-evacuation'), overrides),
|
||||
/injected_batch_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [['source', false]])
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
|
||||
})
|
||||
})
|
||||
|
||||
test('resumes only a committed forward migration and preserves dormant source assignments', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2,
|
||||
dormantSourceAssignments: 7
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'recover-forward'), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'candidate_forward_recovered',
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target',
|
||||
dormantSourceAssignments: 7,
|
||||
targetAssignments: 2,
|
||||
targetActivityLeases: 2,
|
||||
targetReservedRequests: 2
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('retries a transient idempotent completion request without reversing admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2,
|
||||
transientCompletionFailures: 1
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'recover-forward'), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(
|
||||
events.filter(({ event }) => event === 'candidate_admin_retry'),
|
||||
[
|
||||
{
|
||||
event: 'candidate_admin_retry',
|
||||
path: '/v1/admin/evacuation-status',
|
||||
attempt: 1,
|
||||
reason: 'transport'
|
||||
}
|
||||
]
|
||||
)
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovered')
|
||||
})
|
||||
})
|
||||
|
||||
test('stops forward recovery promptly when only registered offline targets remain', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2,
|
||||
migrationTargetInactive: 2
|
||||
})
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'recover-forward'), overrides),
|
||||
/pending for inactive target controls/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'candidate_forward_pending',
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target',
|
||||
inProgress: 2,
|
||||
registeredSourceActive: 0,
|
||||
registeredCompletable: 0,
|
||||
registeredTargetInactive: 2
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses forward recovery unless source and target admission match committed direction', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness()
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'recover-forward'), overrides),
|
||||
/requires disabled source and enabled target/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('re-enables an intact source when candidate admission fails before migration', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness({ failTargetEnable: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_enable_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [
|
||||
['source', false],
|
||||
['target', true],
|
||||
['source', true],
|
||||
['target', false]
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
test('re-enables the source and waits for lease rollback when no target registered', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ failDrain: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_drain_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges.slice(-1), [['source', true]])
|
||||
assert.equal(events.at(-1).event, 'candidate_rollback_waiting_for_lease_expiry')
|
||||
})
|
||||
})
|
||||
|
||||
test('preserves both routes for forward recovery after a target registration', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ failAfterRegistration: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_batch_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [
|
||||
['source', false],
|
||||
['target', true]
|
||||
])
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
|
||||
assert.equal(events.at(-1).targetRegistered, 1)
|
||||
})
|
||||
})
|
||||
|
||||
test('does not reverse admission after completion commits but its response is lost', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ failCompletionResponse: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_completion_response_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [
|
||||
['source', false],
|
||||
['target', true]
|
||||
])
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
|
||||
assert.equal(events.at(-1).targetRegistered, 0)
|
||||
})
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,136 +0,0 @@
|
||||
const REQUEST_TIMEOUT_MS = 15_000
|
||||
const JWT_PATTERN = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/
|
||||
|
||||
export async function requestGitHubSmokeTokens(
|
||||
authOrigin,
|
||||
fetchImpl = fetch,
|
||||
environment = process.env,
|
||||
options = {}
|
||||
) {
|
||||
const origin = canonicalHttpsOrigin(authOrigin)
|
||||
const requestUrl = environment.ACTIONS_ID_TOKEN_REQUEST_URL
|
||||
const requestToken = environment.ACTIONS_ID_TOKEN_REQUEST_TOKEN
|
||||
if (!requestUrl || !requestToken) throw new Error('GitHub OIDC request context is unavailable')
|
||||
const audience = `${origin}/v1/internal/github-smoke-token`
|
||||
const oidcUrl = new URL(requestUrl)
|
||||
oidcUrl.searchParams.set('audience', audience)
|
||||
const oidcResponse = await fetchImpl(oidcUrl, {
|
||||
headers: { authorization: `Bearer ${requestToken}` },
|
||||
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
|
||||
})
|
||||
const oidc = await readJson(oidcResponse, 'GitHub OIDC request')
|
||||
if (!oidcResponse.ok || !validJwt(oidc.value)) {
|
||||
throw new Error(`GitHub OIDC request failed with ${oidcResponse.status}`)
|
||||
}
|
||||
const exchangeResponse = await fetchImpl(audience, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${oidc.value}`,
|
||||
...(options.relayAsiaLoad ? { 'content-type': 'application/json' } : {})
|
||||
},
|
||||
...(options.relayAsiaLoad
|
||||
? { body: JSON.stringify({ relayAsiaLoad: parseRelayAsiaLoadOptions(options.relayAsiaLoad) }) }
|
||||
: {}),
|
||||
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
|
||||
})
|
||||
const exchange = await readJson(exchangeResponse, 'Orca smoke identity exchange')
|
||||
if (!exchangeResponse.ok) {
|
||||
throw new Error(`Orca smoke identity exchange failed with ${exchangeResponse.status}`)
|
||||
}
|
||||
return {
|
||||
...parseAccessTokens(exchange.accessTokens),
|
||||
...(options.relayAsiaLoad
|
||||
? {
|
||||
relayAsiaLoadPrincipals: parseRelayAsiaLoadPrincipals(
|
||||
exchange.relayAsiaLoadPrincipals,
|
||||
options.relayAsiaLoad.principalCount
|
||||
)
|
||||
}
|
||||
: {})
|
||||
}
|
||||
}
|
||||
|
||||
// Staging auth mints Asia load principal sets for this many shards.
|
||||
export const RELAY_ASIA_LOAD_PRINCIPAL_SHARDS = 4
|
||||
|
||||
function parseRelayAsiaLoadOptions(value) {
|
||||
if (
|
||||
!value || typeof value !== 'object' ||
|
||||
!Number.isSafeInteger(value.shardIndex) || value.shardIndex < 0 || value.shardIndex >= RELAY_ASIA_LOAD_PRINCIPAL_SHARDS ||
|
||||
!Number.isSafeInteger(value.principalCount) || value.principalCount < 1 ||
|
||||
value.principalCount > 32
|
||||
) throw new Error('Relay Asia load principal request is invalid')
|
||||
return { v: 1, shardIndex: value.shardIndex, principalCount: value.principalCount }
|
||||
}
|
||||
|
||||
function canonicalHttpsOrigin(value) {
|
||||
const url = new URL(value)
|
||||
if (url.protocol !== 'https:' || url.pathname !== '/' || url.search || url.hash) {
|
||||
throw new Error('auth origin must be canonical HTTPS')
|
||||
}
|
||||
return url.origin
|
||||
}
|
||||
|
||||
function validJwt(value) {
|
||||
return typeof value === 'string' && value.length <= 8192 && JWT_PATTERN.test(value)
|
||||
}
|
||||
|
||||
function parseAccessTokens(value) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error('Orca smoke identity response is malformed')
|
||||
}
|
||||
const expected = ['outsider', 'owner', 'recipient']
|
||||
if (Object.keys(value).sort().join(',') !== expected.join(',')) {
|
||||
throw new Error('Orca smoke identity response principals are invalid')
|
||||
}
|
||||
return Object.fromEntries(
|
||||
expected.map((name) => {
|
||||
const principal = value[name]
|
||||
if (
|
||||
!principal ||
|
||||
typeof principal !== 'object' ||
|
||||
typeof principal.userId !== 'string' ||
|
||||
!/^[A-Za-z0-9_-]{1,128}$/.test(principal.userId) ||
|
||||
typeof principal.accessToken !== 'string' ||
|
||||
!validJwt(principal.accessToken) ||
|
||||
typeof principal.expiresAt !== 'number' ||
|
||||
principal.expiresAt <= Date.now() ||
|
||||
principal.expiresAt > Date.now() + 610_000
|
||||
) {
|
||||
throw new Error('Orca smoke identity response principal is malformed')
|
||||
}
|
||||
return [name, principal]
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
function parseRelayAsiaLoadPrincipals(value, expectedCount) {
|
||||
if (!Array.isArray(value) || value.length !== expectedCount) {
|
||||
throw new Error('Relay Asia load principal response is invalid')
|
||||
}
|
||||
const userIds = new Set()
|
||||
return value.map((principal, principalIndex) => {
|
||||
if (
|
||||
!principal || typeof principal !== 'object' ||
|
||||
principal.principalIndex !== principalIndex ||
|
||||
typeof principal.userId !== 'string' ||
|
||||
!/^usr_relay_asia_load_[A-Za-z0-9_-]{32}$/.test(principal.userId) ||
|
||||
typeof principal.profileId !== 'string' ||
|
||||
principal.profileId !== principal.userId.replace(/^usr_/, 'prof_') ||
|
||||
userIds.has(principal.userId) ||
|
||||
typeof principal.accessToken !== 'string' || !validJwt(principal.accessToken) ||
|
||||
typeof principal.expiresAt !== 'number' || principal.expiresAt <= Date.now() ||
|
||||
principal.expiresAt > Date.now() + 610_000
|
||||
) throw new Error('Relay Asia load principal response is malformed')
|
||||
userIds.add(principal.userId)
|
||||
return principal
|
||||
})
|
||||
}
|
||||
|
||||
async function readJson(response, label) {
|
||||
try {
|
||||
return await response.json()
|
||||
} catch {
|
||||
throw new Error(`${label} did not return JSON`)
|
||||
}
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { requestGitHubSmokeTokens } from './github-smoke-token.mjs'
|
||||
|
||||
const jwt = (value) => `${value}.${value}.${value}`
|
||||
|
||||
test('exchanges the runner OIDC token without returning request credentials', async () => {
|
||||
const requests = []
|
||||
const fetchImpl = async (url, init) => {
|
||||
requests.push({ url: String(url), init })
|
||||
if (requests.length === 1) return Response.json({ value: jwt('github') })
|
||||
return Response.json({
|
||||
accessTokens: Object.fromEntries(
|
||||
['owner', 'recipient', 'outsider'].map((name) => [
|
||||
name,
|
||||
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
|
||||
])
|
||||
)
|
||||
})
|
||||
}
|
||||
const result = await requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
fetchImpl,
|
||||
{
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token?api-version=1',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
|
||||
}
|
||||
)
|
||||
assert.equal(result.owner.userId, 'usr_owner')
|
||||
assert.match(requests[0].url, /audience=https%3A%2F%2Fauth-staging\.onorca\.dev/)
|
||||
assert.equal(requests[0].init.headers.authorization, 'Bearer runner-request-token')
|
||||
assert.equal(requests[1].init.headers.authorization, `Bearer ${jwt('github')}`)
|
||||
})
|
||||
|
||||
test('fails with bounded errors and never includes credentials', async () => {
|
||||
await assert.rejects(
|
||||
requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
async () => new Response('denied', { status: 403 }),
|
||||
{
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'private-request-token'
|
||||
}
|
||||
),
|
||||
(error) => {
|
||||
assert.doesNotMatch(String(error), /private-request-token|denied/)
|
||||
return true
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('requests and validates an exact Relay Asia principal batch', async () => {
|
||||
const requests = []
|
||||
const principals = Array.from({ length: 32 }, (_, principalIndex) => {
|
||||
const suffix = String(principalIndex).padStart(32, 'a')
|
||||
return {
|
||||
principalIndex,
|
||||
userId: `usr_relay_asia_load_${suffix}`,
|
||||
profileId: `prof_relay_asia_load_${suffix}`,
|
||||
accessToken: jwt(`load${principalIndex}`),
|
||||
expiresAt: Date.now() + 600_000
|
||||
}
|
||||
})
|
||||
const result = await requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
async (url, init) => {
|
||||
requests.push({ url: String(url), init })
|
||||
return requests.length === 1
|
||||
? Response.json({ value: jwt('github') })
|
||||
: Response.json({
|
||||
accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [
|
||||
name,
|
||||
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
|
||||
])),
|
||||
relayAsiaLoadPrincipals: principals
|
||||
})
|
||||
},
|
||||
{
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
|
||||
},
|
||||
{ relayAsiaLoad: { shardIndex: 3, principalCount: 32 } }
|
||||
)
|
||||
assert.equal(result.relayAsiaLoadPrincipals.length, 32)
|
||||
assert.deepEqual(JSON.parse(requests[1].init.body), {
|
||||
relayAsiaLoad: { v: 1, shardIndex: 3, principalCount: 32 }
|
||||
})
|
||||
assert.equal(requests[1].init.headers['content-type'], 'application/json')
|
||||
})
|
||||
|
||||
test('rejects malformed or duplicate Relay Asia principal batches', async () => {
|
||||
const environment = {
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
|
||||
}
|
||||
let request = 0
|
||||
await assert.rejects(requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
async () => ++request === 1
|
||||
? Response.json({ value: jwt('github') })
|
||||
: Response.json({
|
||||
accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [
|
||||
name,
|
||||
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
|
||||
])),
|
||||
relayAsiaLoadPrincipals: []
|
||||
}),
|
||||
environment,
|
||||
{ relayAsiaLoad: { shardIndex: 0, principalCount: 32 } }
|
||||
), /principal response is invalid/)
|
||||
})
|
||||
@@ -1,146 +0,0 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { homedir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { assertStagingRelayAwake } from './staging-relay-apply-guard.mjs'
|
||||
|
||||
// The relay root keeps its historical path so every existing caller — 9 workflows, the fence
|
||||
// broker, and the infra:* scripts — is unchanged when --root is omitted. The foundation and apps
|
||||
// roots stay in the private repository with the services they own.
|
||||
const ROOT_DIRECTORIES = {
|
||||
relay: join('infra', 'terraform')
|
||||
}
|
||||
|
||||
const command = process.argv[2]
|
||||
const environment = readEnvironment(process.argv.slice(3))
|
||||
const root = readRoot(process.argv.slice(3))
|
||||
const tool = process.env.IAC_TOOL || findTool()
|
||||
|
||||
if (!command || !['init', 'plan', 'apply'].includes(command)) {
|
||||
exitWithUsage()
|
||||
}
|
||||
|
||||
if (!environment) {
|
||||
exitWithUsage('Missing --env staging|production')
|
||||
}
|
||||
|
||||
if (!root) {
|
||||
exitWithUsage(`Unknown --root; expected one of ${Object.keys(ROOT_DIRECTORIES).join('|')}`)
|
||||
}
|
||||
|
||||
const rootDirectory = ROOT_DIRECTORIES[root]
|
||||
const terraformDir = join(process.cwd(), rootDirectory)
|
||||
const backendConfig = join(terraformDir, 'backend', `${environment}.hcl`)
|
||||
const varFile = join(terraformDir, 'environments', `${environment}.tfvars`)
|
||||
|
||||
if (!existsSync(backendConfig)) {
|
||||
throw new Error(`Backend config not found: ${backendConfig}`)
|
||||
}
|
||||
|
||||
if (!existsSync(varFile)) {
|
||||
throw new Error(`Variable file not found: ${varFile}`)
|
||||
}
|
||||
|
||||
const chdir = `-chdir=${rootDirectory}`
|
||||
|
||||
if (command === 'init') {
|
||||
run([chdir, 'init', `-backend-config=backend/${environment}.hcl`])
|
||||
} else if (command === 'plan') {
|
||||
run([
|
||||
chdir,
|
||||
'plan',
|
||||
`-var-file=environments/${environment}.tfvars`,
|
||||
`-out=${environment}.tfplan`
|
||||
])
|
||||
} else {
|
||||
// A normal staging apply must not implicitly wake or partially mutate a sleeping data plane.
|
||||
// Only the relay root can touch that data plane; the guard would refuse app work for no reason.
|
||||
if (environment === 'staging' && root === 'relay') assertStagingRelayAwake()
|
||||
run([chdir, 'apply', `${environment}.tfplan`])
|
||||
}
|
||||
|
||||
function readEnvironment(args) {
|
||||
const envIndex = args.indexOf('--env')
|
||||
if (envIndex >= 0) {
|
||||
return args[envIndex + 1]
|
||||
}
|
||||
|
||||
return process.env.ORCA_CLOUD_ENV
|
||||
}
|
||||
|
||||
function readRoot(args) {
|
||||
const rootIndex = args.indexOf('--root')
|
||||
const requested = rootIndex >= 0 ? args[rootIndex + 1] : 'relay'
|
||||
return requested in ROOT_DIRECTORIES ? requested : undefined
|
||||
}
|
||||
|
||||
function findTool() {
|
||||
for (const candidate of ['tofu', 'terraform']) {
|
||||
try {
|
||||
execFileSync(candidate, ['version'], { stdio: 'ignore' })
|
||||
return candidate
|
||||
} catch {
|
||||
// Try the next compatible IaC binary.
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Install Terraform or OpenTofu, or set IAC_TOOL.')
|
||||
}
|
||||
|
||||
function run(args) {
|
||||
execFileSync(tool, args, { env: terraformEnv(), stdio: 'inherit' })
|
||||
}
|
||||
|
||||
function terraformEnv() {
|
||||
if (
|
||||
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
|
||||
process.env.GOOGLE_CREDENTIALS ||
|
||||
process.env.GOOGLE_OAUTH_ACCESS_TOKEN
|
||||
) {
|
||||
return process.env
|
||||
}
|
||||
|
||||
const token = readGcloudAccessToken()
|
||||
if (!token) {
|
||||
return process.env
|
||||
}
|
||||
|
||||
// Local convenience: Terraform uses ADC, while engineers often only have
|
||||
// gcloud CLI auth. CI should use Workload Identity instead.
|
||||
return { ...process.env, GOOGLE_OAUTH_ACCESS_TOKEN: token }
|
||||
}
|
||||
|
||||
function readGcloudAccessToken() {
|
||||
for (const candidate of gcloudCandidates()) {
|
||||
try {
|
||||
return execFileSync(candidate, ['auth', 'print-access-token'], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'ignore']
|
||||
}).trim()
|
||||
} catch {
|
||||
// Try the next common gcloud location.
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
function gcloudCandidates() {
|
||||
return [
|
||||
process.env.GCLOUD_PATH,
|
||||
'gcloud',
|
||||
join(homedir(), 'Downloads', 'google-cloud-sdk', 'bin', 'gcloud'),
|
||||
join(homedir(), 'google-cloud-sdk', 'bin', 'gcloud')
|
||||
].filter(Boolean)
|
||||
}
|
||||
|
||||
function exitWithUsage(message) {
|
||||
if (message) {
|
||||
console.error(message)
|
||||
}
|
||||
|
||||
console.error(
|
||||
'Usage: pnpm infra:<init|plan|apply> --env staging|production [--root relay]'
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const repository = fileURLToPath(new URL('../../', import.meta.url))
|
||||
const script = 'dev/scripts/infra.mjs'
|
||||
|
||||
// IAC_TOOL=echo prints the argv the real binary would have received, so the root a flag selects
|
||||
// is observable without running Terraform.
|
||||
function invoke(args) {
|
||||
return execFileSync('node', [script, ...args], {
|
||||
cwd: repository,
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, IAC_TOOL: 'echo' }
|
||||
}).trim()
|
||||
}
|
||||
|
||||
function rejects(args) {
|
||||
try {
|
||||
execFileSync('node', [script, ...args], {
|
||||
cwd: repository,
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
env: { ...process.env, IAC_TOOL: 'echo' }
|
||||
})
|
||||
} catch (error) {
|
||||
return error.stderr
|
||||
}
|
||||
throw new Error(`expected ${args.join(' ')} to exit non-zero`)
|
||||
}
|
||||
|
||||
// Why: 9 relay workflows, the fence broker, and the three infra:* package scripts all invoke this
|
||||
// without --root. If the default ever moves off infra/terraform they break silently at the plan.
|
||||
test('omitting --root keeps every existing caller on the relay root', () => {
|
||||
for (const environment of ['staging', 'production']) {
|
||||
assert.equal(
|
||||
invoke(['init', '--env', environment]),
|
||||
`-chdir=infra/terraform init -backend-config=backend/${environment}.hcl`
|
||||
)
|
||||
assert.equal(
|
||||
invoke(['plan', '--env', environment]),
|
||||
`-chdir=infra/terraform plan -var-file=environments/${environment}.tfvars -out=${environment}.tfplan`
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
// Only the relay root ships here; the foundation and apps roots stay in the private repository.
|
||||
test('each root name selects exactly its own directory', () => {
|
||||
const directories = { relay: 'infra/terraform' }
|
||||
for (const [root, directory] of Object.entries(directories)) {
|
||||
for (const environment of ['staging', 'production']) {
|
||||
assert.equal(
|
||||
invoke(['init', '--env', environment, '--root', root]),
|
||||
`-chdir=${directory} init -backend-config=backend/${environment}.hcl`
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('an unknown root fails closed rather than falling back to the relay root', () => {
|
||||
const stderr = rejects(['plan', '--env', 'staging', '--root', 'releay'])
|
||||
assert.match(stderr, /Unknown --root/)
|
||||
assert.doesNotMatch(stderr, /infra\/terraform /)
|
||||
})
|
||||
|
||||
test('a missing environment still fails before any root is resolved', () => {
|
||||
assert.match(rejects(['plan']), /Missing --env/)
|
||||
})
|
||||
|
||||
// Why: the guard refuses a staging apply while the relay data plane is asleep. Applying it to the
|
||||
// app or foundation roots would block work that never touches that data plane.
|
||||
test('the sleeping staging relay guard is scoped to the relay root', () => {
|
||||
const source = readFileSync(new URL('./infra.mjs', import.meta.url), 'utf8')
|
||||
assert.match(source, /environment === 'staging' && root === 'relay'/)
|
||||
})
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user