mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
Jinwoo Hong
eb2f2d52ae
* refactor(cloud): share PostgreSQL schema startup between services * feat(cloud): add durable native push notification gateway * infra(push): define dedicated gateway resources and operational checks * fix(push): bound cross-host admission and simplify gateway configuration * fix(push): validate deploy configuration and preserve topic-error registrations
567 lines
30 KiB
YAML
567 lines
30 KiB
YAML
name: Deploy Push Gateway Production
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
source_sha:
|
|
description: Full reviewed commit SHA to build (feature may remain unmerged)
|
|
required: true
|
|
type: string
|
|
confirmation:
|
|
description: Enter DEPLOY_PUSH_GATEWAY to shift production traffic
|
|
required: true
|
|
type: string
|
|
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
|
|
# Serialize push traffic changes independently of Relay and the shared database.
|
|
concurrency:
|
|
group: production-push-rollout
|
|
cancel-in-progress: false
|
|
|
|
defaults:
|
|
run:
|
|
working-directory: cloud
|
|
|
|
jobs:
|
|
deploy:
|
|
if: >-
|
|
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
|
|
github.ref == 'refs/heads/main' }}
|
|
runs-on: blacksmith-2vcpu-ubuntu-2204
|
|
environment: production
|
|
env:
|
|
GCP_PROJECT_ID: onorca-cloud
|
|
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
|
SERVICE_NAME: orca-cloud-push
|
|
REPOSITORY_ID: orca-cloud
|
|
IMAGE_NAME: push
|
|
PUSH_ORIGIN: https://push.onorca.dev
|
|
PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud.iam.gserviceaccount.com
|
|
# Scaling the serving revision must already hold, matching push_min_instances and
|
|
# push_max_instances. Terraform owns both, and the candidate inherits them from the
|
|
# service, so this deploy never passes a scaling flag: doing so would write a
|
|
# Terraform-owned field that `lifecycle.ignore_changes` does not cover, and a later
|
|
# `push_max_instances` raise would then be reverted by every deploy. These two values
|
|
# are the expected shape, asserted before the candidate is created and again on the
|
|
# candidate itself, so a deploy that would change the gateway's Cloud SQL draw fails.
|
|
PUSH_MIN_INSTANCES: 1
|
|
PUSH_MAX_INSTANCES: 2
|
|
CONFIRMATION: ${{ inputs.confirmation }}
|
|
SOURCE_SHA: ${{ inputs.source_sha }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Require the explicit deploy confirmation
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
test "${CONFIRMATION}" = DEPLOY_PUSH_GATEWAY
|
|
[[ "${SOURCE_SHA}" =~ ^[a-f0-9]{40}$ ]]
|
|
|
|
# Keep the workflow and rollout lease on main; only the Docker build uses candidate code.
|
|
- name: Fetch the immutable gateway source
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
git fetch --no-tags origin "${SOURCE_SHA}"
|
|
test "$(git rev-parse FETCH_HEAD)" = "${SOURCE_SHA}"
|
|
mkdir -p "${RUNNER_TEMP}/push-source"
|
|
git -C "${GITHUB_WORKSPACE}" archive "${SOURCE_SHA}" cloud \
|
|
| tar -x -C "${RUNNER_TEMP}/push-source"
|
|
|
|
- uses: google-github-actions/auth@v2
|
|
with:
|
|
workload_identity_provider: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
|
service_account: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT }}
|
|
|
|
- uses: google-github-actions/setup-gcloud@v2
|
|
|
|
- uses: docker/setup-buildx-action@v3
|
|
|
|
- name: Configure Docker auth
|
|
run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
|
|
|
|
# Building an image does not need the deployment lease.
|
|
- name: Build and publish the immutable gateway image
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
image_tag="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${SOURCE_SHA}"
|
|
docker buildx build --push --platform linux/amd64 --provenance=false --metadata-file "${RUNNER_TEMP}/push-image.json" \
|
|
-f "${RUNNER_TEMP}/push-source/cloud/apps/push/Dockerfile" \
|
|
-t "${image_tag}" "${RUNNER_TEMP}/push-source/cloud"
|
|
digest="$(jq -er '."containerimage.digest"' "${RUNNER_TEMP}/push-image.json")"
|
|
[[ "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
|
|
echo "IMAGE=${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${digest}" \
|
|
>> "${GITHUB_ENV}"
|
|
echo "IMAGE_DIGEST=${digest}" >> "${GITHUB_ENV}"
|
|
|
|
# Refuse older images before they ever boot against production.
|
|
- name: Require image support for inert validation
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
docker run --rm --network none --entrypoint node "${IMAGE}" --input-type=module -e '
|
|
import { loadPushConfig } from "./apps/push/dist/config.js";
|
|
const env = { ORCA_PUSH_PUBLIC_URL: "https://push.onorca.dev", ORCA_PUSH_MODE: "validation", ORCA_PUSH_FCM_PROJECT_ID: "onorca-cloud" };
|
|
if (loadPushConfig(env).mode !== "validation") throw new Error("validation_mode_unsupported");
|
|
let rejected = false;
|
|
try { loadPushConfig({ ...env, ORCA_PUSH_MODE: "invalid" }); } catch { rejected = true; }
|
|
if (!rejected) throw new Error("validation_mode_not_fail_closed");
|
|
'
|
|
|
|
# Held across the deploy, not just a separate schema step: the gateway opens its pool and
|
|
# applies its schema while the new revision starts, so the revision is the schema step.
|
|
- uses: ./.github/actions/cloud-sql-rollout-lease
|
|
with:
|
|
bucket: onorca-cloud-terraform-state
|
|
object: terraform/state/push-rollout/production.lock
|
|
|
|
# Why: the candidate inherits the serving revision's scaling. A serving revision that has
|
|
# drifted below the floor would hand the candidate a cold start on every notification, and
|
|
# one that has drifted above the ceiling would hand it a larger Cloud SQL draw than the
|
|
# rollout lease was taken for. Refuse to inherit either rather than latch it.
|
|
- name: Record the serving revision and require its Terraform-owned scaling
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
|
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
|
|
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
|
test -n "${serving}"
|
|
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
|
|
if test "${revisions}" != "${serving}"; then
|
|
echo 'Retire leftover revisions under the rollout lease before deploying; three pools are the limit.' >&2
|
|
exit 1
|
|
fi
|
|
floor="$(gcloud run revisions describe "${serving}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
|
|
if [[ "${floor:-0}" -lt "${PUSH_MIN_INSTANCES}" ]]; then
|
|
echo "serving revision ${serving} holds ${floor:-0} minimum instances," \
|
|
"below ${PUSH_MIN_INSTANCES}; deploying would inherit and latch it." >&2
|
|
echo "Restore the floor first: gcloud run services update ${SERVICE_NAME}" \
|
|
"--region ${GCP_REGION} --min-instances=${PUSH_MIN_INSTANCES}" >&2
|
|
exit 1
|
|
fi
|
|
ceiling="$(gcloud run revisions describe "${serving}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
|
|
test "${ceiling}" = "${PUSH_MAX_INSTANCES}"
|
|
echo "serving revision ${serving} holds ${floor} minimum and ${ceiling} maximum instances"
|
|
echo "ROLLBACK_REVISION=${serving}" >> "${GITHUB_ENV}"
|
|
gcloud run revisions describe "${serving}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
|
> "${RUNNER_TEMP}/push-rollback-revision.json"
|
|
image="$(jq -er '.status.imageDigest' "${RUNNER_TEMP}/push-rollback-revision.json")"
|
|
[[ "${image}" =~ @sha256:[a-f0-9]{64}$ ]]
|
|
echo "ROLLBACK_IMAGE=${image}" >> "${GITHUB_ENV}"
|
|
jq -e 'all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE" or .value == "active")' \
|
|
"${RUNNER_TEMP}/push-rollback-revision.json" > /dev/null
|
|
|
|
# Validation has no schema writes, HTTP mutations, worker, or pruners; tags alone do not
|
|
# isolate background consumers from production.
|
|
- name: Deploy the candidate revision with no traffic
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
tag="c${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
|
{
|
|
echo "VALIDATION_DEPLOY_ATTEMPTED=true"
|
|
echo "VALIDATION_REVISION=${SERVICE_NAME}-${tag}"
|
|
echo "VALIDATION_TAG=${tag}"
|
|
echo "CANDIDATE_TAG=${tag}"
|
|
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
|
|
} >> "${GITHUB_ENV}"
|
|
gcloud run deploy "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" \
|
|
--region "${GCP_REGION}" \
|
|
--image "${IMAGE}" \
|
|
--tag "${tag}" \
|
|
--revision-suffix "${tag}" \
|
|
--no-traffic \
|
|
--update-env-vars ORCA_PUSH_MODE=validation \
|
|
--quiet
|
|
candidate="$(gcloud run services describe "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
|
| jq -er --arg tag "${tag}" \
|
|
'[.status.traffic[] | select(.tag == $tag)]
|
|
| if length == 1 then .[0] else error("tagged candidate is not unique") end')"
|
|
test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
|
|
echo "CANDIDATE_URL=$(jq -r '.url' <<< "${candidate}")" >> "${GITHUB_ENV}"
|
|
|
|
# A tagged revision is directly addressable and sits outside the service-wide cap, so the
|
|
# candidate and the serving revision each draw up to the ceiling during the probe window.
|
|
# Successor creation later requires three revision pools; assert the inherited ceiling.
|
|
- name: Require the candidate to serve the exact image and inherited scaling
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
served="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--format='value(spec.containers[0].image)')"
|
|
test "${served}" = "${IMAGE}"
|
|
test "${CANDIDATE_REVISION}" != "${ROLLBACK_REVISION}"
|
|
candidate_ceiling="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
|
|
test "${candidate_ceiling}" = "${PUSH_MAX_INSTANCES}"
|
|
|
|
- name: Probe the candidate readiness endpoint
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
[[ "${CANDIDATE_URL}" =~ ^https://[^/]+$ ]]
|
|
for attempt in $(seq 1 30); do
|
|
code="$(curl -sS -o "${RUNNER_TEMP}/push-ready.json" -w '%{http_code}' \
|
|
--max-time 10 "${CANDIDATE_URL}/ready" || true)"
|
|
if test "${code}" = 200; then
|
|
jq -e . < "${RUNNER_TEMP}/push-ready.json" > /dev/null
|
|
curl --fail --silent --show-error --max-time 10 "${CANDIDATE_URL}/health" \
|
|
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "validation"' > /dev/null
|
|
echo "candidate ${CANDIDATE_REVISION} is ready after ${attempt} attempt(s)"
|
|
exit 0
|
|
fi
|
|
echo "attempt ${attempt}: /ready returned ${code}"
|
|
sleep 5
|
|
done
|
|
echo "candidate ${CANDIDATE_REVISION} never reported ready" >&2
|
|
exit 1
|
|
|
|
# Why: a gateway that boots and answers /ready can still be unable to send. This proves the
|
|
# runtime account's FCM grant end to end without delivering anything: validate_only stops
|
|
# Google before any push, and the deliberately invalid token means a healthy credential
|
|
# answers INVALID_ARGUMENT. PERMISSION_DENIED is the failure this step exists to catch.
|
|
#
|
|
# Only the four verdicts below are conclusive. A 429, a 5xx, or a transport failure says
|
|
# nothing about the credential, so it is retried rather than treated as either answer; a
|
|
# denied credential still fails on the first attempt, without burning the retries.
|
|
- name: Prove the runtime identity can reach FCM
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
token="$(gcloud auth print-access-token \
|
|
--impersonate-service-account "${PUSH_RUNTIME_SERVICE_ACCOUNT}")"
|
|
test -n "${token}"
|
|
echo "::add-mask::${token}"
|
|
body='{"validate_only":true,"message":{"token":"orca-push-deploy-probe-invalid-token","notification":{"title":"Orca","body":"deploy probe"}}}'
|
|
for attempt in $(seq 1 5); do
|
|
code="$(curl -sS -o "${RUNNER_TEMP}/push-fcm.json" -w '%{http_code}' --max-time 20 \
|
|
-X POST "https://fcm.googleapis.com/v1/projects/${GCP_PROJECT_ID}/messages:send" \
|
|
-H "Authorization: Bearer ${token}" \
|
|
-H 'Content-Type: application/json' \
|
|
--data "${body}" || true)"
|
|
status="$(jq -r '.error.status // empty' < "${RUNNER_TEMP}/push-fcm.json" || true)"
|
|
echo "attempt ${attempt}: FCM validate-only send returned HTTP ${code} status ${status:-OK}"
|
|
if test "${status}" = PERMISSION_DENIED || test "${status}" = INVALID_ARGUMENT ||
|
|
test "${code}" = 401 || test "${code}" = 403; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
if test "${status}" = PERMISSION_DENIED || test "${code}" = 401 || test "${code}" = 403; then
|
|
echo "the push runtime identity cannot send through FCM" >&2
|
|
exit 1
|
|
fi
|
|
test "${status}" = INVALID_ARGUMENT
|
|
|
|
# Cloud Run requires a successor before the latest revision can be deleted.
|
|
- name: Retire inert validation and activate the verified image
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
tag="a${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
|
{
|
|
echo "CANDIDATE_TAG=${tag}"
|
|
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
|
|
echo "ACTIVATION_ATTEMPTED=true"
|
|
} >> "${GITHUB_ENV}"
|
|
# This is the production-effect boundary: schema, pruners and workers start here.
|
|
gcloud run deploy "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--image "${IMAGE}" --tag "${tag}" --revision-suffix "${tag}" \
|
|
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
|
|
gcloud run services update-traffic "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--remove-tags "${VALIDATION_TAG}" --quiet
|
|
gcloud run revisions delete "${VALIDATION_REVISION}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
|
echo "VALIDATION_RETIRED=true" >> "${GITHUB_ENV}"
|
|
revision="$(gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
|
|
jq -e --arg image "${IMAGE}" --arg account "${PUSH_RUNTIME_SERVICE_ACCOUNT}" \
|
|
--arg ceiling "${PUSH_MAX_INSTANCES}" --arg floor "${PUSH_MIN_INSTANCES}" \
|
|
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" '
|
|
def shape: del(.containers[0].image) |
|
|
.containers[0].env = ((.containers[0].env // []) |
|
|
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
|
|
.spec.containers[0].image == $image and .spec.serviceAccountName == $account and
|
|
all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
|
|
(.spec | shape) == ($prior[0].spec | shape) and
|
|
.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
|
|
(.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber)' \
|
|
<<< "${revision}" > /dev/null
|
|
candidate="$(gcloud run services describe "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
|
| jq -er --arg tag "${tag}" '[.status.traffic[] | select(.tag == $tag)]
|
|
| if length == 1 then .[0] else error("active candidate is not unique") end')"
|
|
test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
|
|
url="$(jq -er '.url' <<< "${candidate}")"
|
|
[[ "${url}" =~ ^https://[^/]+$ ]]
|
|
curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
|
|
curl --fail --silent --show-error --max-time 10 "${url}/health" \
|
|
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
|
|
|
|
- name: Shift all traffic to the verified candidate
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
echo "TRAFFIC_SHIFT_ATTEMPTED=true" >> "${GITHUB_ENV}"
|
|
gcloud run services update-traffic "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" \
|
|
--region "${GCP_REGION}" \
|
|
--to-revisions "${CANDIDATE_REVISION}=100" \
|
|
--quiet
|
|
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
|
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
|
|
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
|
test "${serving}" = "${CANDIDATE_REVISION}"
|
|
echo "TRAFFIC_SHIFTED=true" >> "${GITHUB_ENV}"
|
|
|
|
# Why: the summary is written before the origin check, not after it. Once traffic has
|
|
# moved, the rollback target is the single thing an operator needs, and a summary that only
|
|
# appeared on success would be missing in exactly the run that needs it.
|
|
- name: Publish the rollout summary
|
|
if: ${{ always() && env.CANDIDATE_REVISION != '' && env.ROLLBACK_REVISION != '' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
{
|
|
echo '### Push gateway rollout'
|
|
echo
|
|
echo "Source: ${SOURCE_SHA}"
|
|
echo
|
|
echo "Revision: \`${CANDIDATE_REVISION}\`"
|
|
echo
|
|
echo "Image: \`${IMAGE_DIGEST}\`"
|
|
echo "Known-good image: \`${ROLLBACK_IMAGE}\`"
|
|
echo
|
|
echo "Recovery: deploy \`${ROLLBACK_IMAGE}\` as a new revision with" \
|
|
"\`--remove-env-vars ORCA_PUSH_MODE --no-traffic --tag <unique-recovery-tag> --revision-suffix <unique-recovery-suffix>\`."
|
|
echo 'Verify its exact digest, configuration, readiness and active mode, then promote and check the public origin.'
|
|
echo 'Only then remove obsolete tags and delete rejected/previous revisions; never delete the latest revision.'
|
|
echo 'The previous revision is retired after public checks; retain this immutable image for recovery under the rollout lease.'
|
|
echo "Activation attempted: ${ACTIVATION_ATTEMPTED:-false}; traffic rollback cannot undo schema or deliveries."
|
|
} >> "${GITHUB_STEP_SUMMARY}"
|
|
|
|
- name: Verify the public origin after the shift
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
for attempt in $(seq 1 30); do
|
|
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \
|
|
"${PUSH_ORIGIN}/ready" || true)"
|
|
if test "${code}" = 200; then
|
|
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
|
|
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' > /dev/null
|
|
echo "ROLLOUT_VERIFIED=true" >> "${GITHUB_ENV}"
|
|
echo "${PUSH_ORIGIN} is ready after ${attempt} attempt(s)"
|
|
exit 0
|
|
fi
|
|
echo "attempt ${attempt}: ${PUSH_ORIGIN}/ready returned ${code}"
|
|
sleep 5
|
|
done
|
|
echo "${PUSH_ORIGIN} never reported ready after the shift" >&2
|
|
exit 1
|
|
|
|
# Why: everything after the shift runs with production on the candidate. A failure there
|
|
# is not a failure to deploy, it is a live gateway that has to go back, so the traffic move
|
|
# is undone here rather than left to whoever reads the run.
|
|
- name: Roll traffic back to the previous revision
|
|
if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
test -n "${ROLLBACK_REVISION:-}"
|
|
gcloud run services update-traffic "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" \
|
|
--region "${GCP_REGION}" \
|
|
--to-revisions "${ROLLBACK_REVISION}=100" \
|
|
--quiet
|
|
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
|
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
|
|
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
|
test "${serving}" = "${ROLLBACK_REVISION}"
|
|
echo "TRAFFIC_ROLLED_BACK=true" >> "${GITHUB_ENV}"
|
|
{
|
|
echo
|
|
echo '### Push gateway rolled back'
|
|
echo
|
|
echo "Traffic returned to \`${ROLLBACK_REVISION}\`; the candidate" \
|
|
"\`${CANDIDATE_REVISION}\` no longer serves HTTP; deletion below must stop its workers."
|
|
} >> "${GITHUB_STEP_SUMMARY}"
|
|
|
|
# Deleting a revision does not restore the service template that Terraform reconciles.
|
|
- name: Restore the known-good service template
|
|
if: ${{ (failure() || cancelled()) && env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
test "${TRAFFIC_SHIFT_ATTEMPTED:-false}" != true || test "${TRAFFIC_ROLLED_BACK:-false}" = true
|
|
test -n "${ROLLBACK_IMAGE}"
|
|
# A partial activation may leave validation plus active; free one slot before recovery.
|
|
latest="$(gcloud run services describe "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--format='value(status.latestCreatedRevisionName)')"
|
|
test -n "${latest}"
|
|
if test "${VALIDATION_RETIRED:-false}" != true && test "${latest}" != "${VALIDATION_REVISION}"; then
|
|
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--filter "metadata.name=${VALIDATION_REVISION}" --format='value(metadata.name)')"
|
|
if test -n "${existing}"; then
|
|
test "${existing}" = "${VALIDATION_REVISION}"
|
|
gcloud run services update-traffic "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--remove-tags "${VALIDATION_TAG}" --quiet
|
|
gcloud run revisions delete "${VALIDATION_REVISION}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
|
fi
|
|
fi
|
|
tag="r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
|
echo "RECOVERY_TAG=${tag}" >> "${GITHUB_ENV}"
|
|
echo "TEMPLATE_RECOVERY_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
|
|
echo "Template recovery attempted: ${SERVICE_NAME}-${tag}, image ${ROLLBACK_IMAGE}." \
|
|
>> "${GITHUB_STEP_SUMMARY}"
|
|
# Known-good schema/workers can run here even though HTTP stays on the old revision.
|
|
gcloud run deploy "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--image "${ROLLBACK_IMAGE}" --revision-suffix "${tag}" --tag "${tag}" \
|
|
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
|
|
gcloud run services describe "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
|
> "${RUNNER_TEMP}/push-recovered-service.json"
|
|
gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
|
> "${RUNNER_TEMP}/push-recovery-revision.json"
|
|
jq -e --arg image "${ROLLBACK_IMAGE}" --arg serving "${ROLLBACK_REVISION}" \
|
|
--arg floor "${PUSH_MIN_INSTANCES}" --arg ceiling "${PUSH_MAX_INSTANCES}" \
|
|
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" \
|
|
--slurpfile recovered "${RUNNER_TEMP}/push-recovery-revision.json" '
|
|
def shape: del(.containers[0].image) |
|
|
.containers[0].env = ((.containers[0].env // []) |
|
|
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
|
|
.spec.template.spec.containers[0].image == $image and
|
|
all(.spec.template.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
|
|
$recovered[0].spec.containers[0].image == $image and
|
|
all($recovered[0].spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
|
|
($recovered[0].spec | shape) == ($prior[0].spec | shape) and
|
|
.spec.template.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
|
|
(.spec.template.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber) and
|
|
([.status.traffic[] | select((.percent // 0) > 0)] |
|
|
length == 1 and .[0].revisionName == $serving and .[0].percent == 100)
|
|
' "${RUNNER_TEMP}/push-recovered-service.json" > /dev/null
|
|
echo "TEMPLATE_RESTORED=true" >> "${GITHUB_ENV}"
|
|
echo 'Known-good image and normal mode restored; previous revision still serves HTTP.' \
|
|
>> "${GITHUB_STEP_SUMMARY}"
|
|
|
|
- name: Promote and verify the known-good recovery revision
|
|
if: ${{ always() && env.TEMPLATE_RESTORED == 'true' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
url="$(jq -er --arg tag "${RECOVERY_TAG}" \
|
|
'.status.traffic[] | select(.tag == $tag) | .url' "${RUNNER_TEMP}/push-recovered-service.json")"
|
|
[[ "${url}" =~ ^https://[^/]+$ ]]
|
|
curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
|
|
curl --fail --silent --show-error --max-time 10 "${url}/health" \
|
|
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
|
|
gcloud run services update-traffic "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--to-revisions "${TEMPLATE_RECOVERY_REVISION}=100" --quiet
|
|
serving="$(gcloud run services describe "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
|
|
| jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
|
|
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
|
|
test "${serving}" = "${TEMPLATE_RECOVERY_REVISION}"
|
|
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/ready" | jq -e '.ok == true'
|
|
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
|
|
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
|
|
echo "RECOVERY_VERIFIED=true" >> "${GITHUB_ENV}"
|
|
echo "Recovery revision ${TEMPLATE_RECOVERY_REVISION} now serves the known-good image." \
|
|
>> "${GITHUB_STEP_SUMMARY}"
|
|
|
|
- name: Delete the rejected candidate revision
|
|
if: ${{ always() && env.RECOVERY_VERIFIED == 'true' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
if test -z "${CANDIDATE_REVISION:-}"; then
|
|
echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
|
|
exit 0
|
|
fi
|
|
if test -n "${CANDIDATE_TAG:-}"; then
|
|
gcloud run services update-traffic "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" \
|
|
--region "${GCP_REGION}" \
|
|
--remove-tags "${CANDIDATE_TAG}" \
|
|
--quiet
|
|
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
|
|
fi
|
|
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
|
|
--filter "metadata.name=${CANDIDATE_REVISION}" --format='value(metadata.name)')"
|
|
if test -n "${existing}"; then
|
|
test "${existing}" = "${CANDIDATE_REVISION}"
|
|
gcloud run revisions delete "${CANDIDATE_REVISION}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
|
fi
|
|
echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
|
|
echo "candidate revision ${CANDIDATE_REVISION} is absent"
|
|
|
|
# Public checks commit the serving revision; cleanup failures must not roll it back.
|
|
- name: Retire previous consumers after public checks
|
|
if: ${{ always() && (env.ROLLOUT_VERIFIED == 'true' || (env.RECOVERY_VERIFIED == 'true' && env.CANDIDATE_DELETED == 'true')) }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
serving="${CANDIDATE_REVISION}"
|
|
if test "${RECOVERY_VERIFIED:-false}" = true; then
|
|
serving="${TEMPLATE_RECOVERY_REVISION}"
|
|
fi
|
|
gcloud run services update-traffic "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --clear-tags --quiet
|
|
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
|
|
while IFS= read -r revision; do
|
|
test -n "${revision}" || continue
|
|
test "${revision}" != "${serving}" || continue
|
|
case "${revision}" in
|
|
"${ROLLBACK_REVISION}"|"${VALIDATION_REVISION}"|"${CANDIDATE_REVISION}") ;;
|
|
*) echo "Unexpected revision ${revision}; manual retirement required." >&2; exit 1 ;;
|
|
esac
|
|
gcloud run revisions delete "${revision}" \
|
|
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
|
|
done <<< "${revisions}"
|
|
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
|
|
echo 'Obsolete revision resources retired; verify actual SQL session drain operationally.' \
|
|
>> "${GITHUB_STEP_SUMMARY}"
|
|
|
|
- name: Drop the candidate traffic tag
|
|
if: always()
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
test -n "${CANDIDATE_TAG:-}" || exit 0
|
|
gcloud run services update-traffic "${SERVICE_NAME}" \
|
|
--project "${GCP_PROJECT_ID}" \
|
|
--region "${GCP_REGION}" \
|
|
--remove-tags "${CANDIDATE_TAG}" \
|
|
--quiet
|