From 48fb41b32453c6fe3be60f54db753ee910e5f1d8 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Mon, 7 Sep 2026 21:57:05 -0400 Subject: [PATCH] fix(push): verify successor-first Cloud Run recovery --- .github/workflows/cloud-push-deploy.yml | 234 ++++--- cloud/dev/scripts/push-cloud-run-model.mjs | 140 +++++ .../scripts/push-gateway-recovery.test.mjs | 592 ++++++++++-------- .../scripts/push-gateway-workflow.test.mjs | 15 +- .../scripts/push-validation-workflow.test.mjs | 4 +- .../relay-cloud-sql-connection-budget.mjs | 4 +- ...relay-cloud-sql-connection-budget.test.mjs | 12 +- cloud/docs/push-database-cutover.md | 21 +- cloud/docs/push-gateway.md | 100 +-- cloud/infra/terraform/push-gateway.tf | 4 +- 10 files changed, 724 insertions(+), 402 deletions(-) create mode 100644 cloud/dev/scripts/push-cloud-run-model.mjs diff --git a/.github/workflows/cloud-push-deploy.yml b/.github/workflows/cloud-push-deploy.yml index e99a28d5cf6..d92e2eb3df8 100644 --- a/.github/workflows/cloud-push-deploy.yml +++ b/.github/workflows/cloud-push-deploy.yml @@ -94,7 +94,7 @@ jobs: run: | set -euo pipefail image_tag="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${SOURCE_SHA}" - docker buildx build --push --metadata-file "${RUNNER_TEMP}/push-image.json" \ + docker buildx build --push --platform linux/amd64 --provenance=false --metadata-file "${RUNNER_TEMP}/push-image.json" \ -f "${RUNNER_TEMP}/push-source/cloud/apps/push/Dockerfile" \ -t "${image_tag}" "${RUNNER_TEMP}/push-source/cloud" digest="$(jq -er '."containerimage.digest"' "${RUNNER_TEMP}/push-image.json")" @@ -137,6 +137,12 @@ jobs: | jq -r '[.status.traffic[] | select((.percent // 0) > 0)] | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" test -n "${serving}" + revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')" + if test "${revisions}" != "${serving}"; then + echo 'Retire leftover revisions under the rollout lease before deploying; three pools are the limit.' >&2 + exit 1 + fi floor="$(gcloud run revisions describe "${serving}" \ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ --format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")" @@ -169,9 +175,13 @@ jobs: run: | set -euo pipefail tag="c${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - echo "VALIDATION_DEPLOY_ATTEMPTED=true" >> "${GITHUB_ENV}" - echo "CANDIDATE_TAG=${tag}" >> "${GITHUB_ENV}" - echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}" + { + echo "VALIDATION_DEPLOY_ATTEMPTED=true" + echo "VALIDATION_REVISION=${SERVICE_NAME}-${tag}" + echo "VALIDATION_TAG=${tag}" + echo "CANDIDATE_TAG=${tag}" + echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}" + } >> "${GITHUB_ENV}" gcloud run deploy "${SERVICE_NAME}" \ --project "${GCP_PROJECT_ID}" \ --region "${GCP_REGION}" \ @@ -191,8 +201,7 @@ jobs: # A tagged revision is directly addressable and sits outside the service-wide cap, so the # candidate and the serving revision each draw up to the ceiling during the probe window. - # The lease is taken for exactly that doubling; a candidate that inherited a wider ceiling - # would exceed it, so the inherited scaling is asserted here too. + # Successor creation later requires three revision pools; assert the inherited ceiling. - name: Require the candidate to serve the exact image and inherited scaling shell: bash run: | @@ -265,34 +274,39 @@ jobs: fi test "${status}" = INVALID_ARGUMENT - # Delete validation before activation so only two revision pools can overlap. + # Cloud Run requires a successor before the latest revision can be deleted. - name: Retire inert validation and activate the verified image shell: bash run: | set -euo pipefail - gcloud run services update-traffic "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --remove-tags "${CANDIDATE_TAG}" --quiet - gcloud run revisions delete "${CANDIDATE_REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet - echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}" - echo "CANDIDATE_REVISION=" >> "${GITHUB_ENV}" - # Allow the deleted instance's bounded shutdown to release its pool. - sleep 10 tag="a${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - echo "CANDIDATE_TAG=${tag}" >> "${GITHUB_ENV}" - echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}" - echo "ACTIVATION_ATTEMPTED=true" >> "${GITHUB_ENV}" + { + echo "CANDIDATE_TAG=${tag}" + echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}" + echo "ACTIVATION_ATTEMPTED=true" + } >> "${GITHUB_ENV}" # This is the production-effect boundary: schema, pruners and workers start here. gcloud run deploy "${SERVICE_NAME}" \ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ --image "${IMAGE}" --tag "${tag}" --revision-suffix "${tag}" \ --remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet + gcloud run services update-traffic "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ + --remove-tags "${VALIDATION_TAG}" --quiet + gcloud run revisions delete "${VALIDATION_REVISION}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet + echo "VALIDATION_RETIRED=true" >> "${GITHUB_ENV}" revision="$(gcloud run revisions describe "${SERVICE_NAME}-${tag}" \ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" jq -e --arg image "${IMAGE}" --arg account "${PUSH_RUNTIME_SERVICE_ACCOUNT}" \ --arg ceiling "${PUSH_MAX_INSTANCES}" --arg floor "${PUSH_MIN_INSTANCES}" \ - '.spec.containers[0].image == $image and .spec.serviceAccountName == $account and + --slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" ' + def shape: del(.containers[0].image) | + .containers[0].env = ((.containers[0].env // []) | + map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name)); + .spec.containers[0].image == $image and .spec.serviceAccountName == $account and + all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and + (.spec | shape) == ($prior[0].spec | shape) and .metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and (.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber)' \ <<< "${revision}" > /dev/null @@ -342,10 +356,11 @@ jobs: echo "Image: \`${IMAGE_DIGEST}\`" echo "Known-good image: \`${ROLLBACK_IMAGE}\`" echo - echo "Rollback: \`gcloud run services update-traffic ${SERVICE_NAME}" \ - "--project ${GCP_PROJECT_ID} --region ${GCP_REGION} --to-revisions ${ROLLBACK_REVISION}=100\`" - echo - echo "Then remove tag \`${CANDIDATE_TAG:-none}\` and delete revision \`${CANDIDATE_REVISION}\` to stop workers." + echo "Recovery: deploy \`${ROLLBACK_IMAGE}\` as a new revision with" \ + "\`--remove-env-vars ORCA_PUSH_MODE --no-traffic --tag --revision-suffix \`." + echo 'Verify its exact digest, configuration, readiness and active mode, then promote and check the public origin.' + echo 'Only then remove obsolete tags and delete rejected/previous revisions; never delete the latest revision.' + echo 'The previous revision is retired after public checks; retain this immutable image for recovery under the rollout lease.' echo "Activation attempted: ${ACTIVATION_ATTEMPTED:-false}; traffic rollback cannot undo schema or deliveries." } >> "${GITHUB_STEP_SUMMARY}" @@ -359,6 +374,7 @@ jobs: if test "${code}" = 200; then curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \ | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' > /dev/null + echo "ROLLOUT_VERIFIED=true" >> "${GITHUB_ENV}" echo "${PUSH_ORIGIN} is ready after ${attempt} attempt(s)" exit 0 fi @@ -372,7 +388,7 @@ jobs: # is not a failure to deploy, it is a live gateway that has to go back, so the traffic move # is undone here rather than left to whoever reads the run. - name: Roll traffic back to the previous revision - if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' }} + if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }} shell: bash run: | set -euo pipefail @@ -396,12 +412,97 @@ jobs: "\`${CANDIDATE_REVISION}\` no longer serves HTTP; deletion below must stop its workers." } >> "${GITHUB_STEP_SUMMARY}" - # Why: a candidate that never took traffic is a revision holding a warm floor and a Cloud - # SQL pool for nothing. Its tag comes off first, because Cloud Run refuses to delete a - # revision a traffic target still names, and clearing CANDIDATE_TAG makes the always() tag - # step below a no-op rather than a second failure. + # Deleting a revision does not restore the service template that Terraform reconciles. + - name: Restore the known-good service template + if: ${{ (failure() || cancelled()) && env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }} + shell: bash + run: | + set -euo pipefail + test "${TRAFFIC_SHIFT_ATTEMPTED:-false}" != true || test "${TRAFFIC_ROLLED_BACK:-false}" = true + test -n "${ROLLBACK_IMAGE}" + # A partial activation may leave validation plus active; free one slot before recovery. + latest="$(gcloud run services describe "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ + --format='value(status.latestCreatedRevisionName)')" + test -n "${latest}" + if test "${VALIDATION_RETIRED:-false}" != true && test "${latest}" != "${VALIDATION_REVISION}"; then + existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ + --filter "metadata.name=${VALIDATION_REVISION}" --format='value(metadata.name)')" + if test -n "${existing}"; then + test "${existing}" = "${VALIDATION_REVISION}" + gcloud run services update-traffic "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ + --remove-tags "${VALIDATION_TAG}" --quiet + gcloud run revisions delete "${VALIDATION_REVISION}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet + fi + fi + tag="r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "RECOVERY_TAG=${tag}" >> "${GITHUB_ENV}" + echo "TEMPLATE_RECOVERY_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}" + echo "Template recovery attempted: ${SERVICE_NAME}-${tag}, image ${ROLLBACK_IMAGE}." \ + >> "${GITHUB_STEP_SUMMARY}" + # Known-good schema/workers can run here even though HTTP stays on the old revision. + gcloud run deploy "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ + --image "${ROLLBACK_IMAGE}" --revision-suffix "${tag}" --tag "${tag}" \ + --remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet + gcloud run services describe "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ + > "${RUNNER_TEMP}/push-recovered-service.json" + gcloud run revisions describe "${SERVICE_NAME}-${tag}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ + > "${RUNNER_TEMP}/push-recovery-revision.json" + jq -e --arg image "${ROLLBACK_IMAGE}" --arg serving "${ROLLBACK_REVISION}" \ + --arg floor "${PUSH_MIN_INSTANCES}" --arg ceiling "${PUSH_MAX_INSTANCES}" \ + --slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" \ + --slurpfile recovered "${RUNNER_TEMP}/push-recovery-revision.json" ' + def shape: del(.containers[0].image) | + .containers[0].env = ((.containers[0].env // []) | + map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name)); + .spec.template.spec.containers[0].image == $image and + all(.spec.template.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and + $recovered[0].spec.containers[0].image == $image and + all($recovered[0].spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and + ($recovered[0].spec | shape) == ($prior[0].spec | shape) and + .spec.template.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and + (.spec.template.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber) and + ([.status.traffic[] | select((.percent // 0) > 0)] | + length == 1 and .[0].revisionName == $serving and .[0].percent == 100) + ' "${RUNNER_TEMP}/push-recovered-service.json" > /dev/null + echo "TEMPLATE_RESTORED=true" >> "${GITHUB_ENV}" + echo 'Known-good image and normal mode restored; previous revision still serves HTTP.' \ + >> "${GITHUB_STEP_SUMMARY}" + + - name: Promote and verify the known-good recovery revision + if: ${{ always() && env.TEMPLATE_RESTORED == 'true' }} + shell: bash + run: | + set -euo pipefail + url="$(jq -er --arg tag "${RECOVERY_TAG}" \ + '.status.traffic[] | select(.tag == $tag) | .url' "${RUNNER_TEMP}/push-recovered-service.json")" + [[ "${url}" =~ ^https://[^/]+$ ]] + curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true' + curl --fail --silent --show-error --max-time 10 "${url}/health" \ + | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' + gcloud run services update-traffic "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ + --to-revisions "${TEMPLATE_RECOVERY_REVISION}=100" --quiet + serving="$(gcloud run services describe "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ + | jq -er '[.status.traffic[] | select((.percent // 0) > 0)] | + if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" + test "${serving}" = "${TEMPLATE_RECOVERY_REVISION}" + curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/ready" | jq -e '.ok == true' + curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \ + | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' + echo "RECOVERY_VERIFIED=true" >> "${GITHUB_ENV}" + echo "Recovery revision ${TEMPLATE_RECOVERY_REVISION} now serves the known-good image." \ + >> "${GITHUB_STEP_SUMMARY}" + - name: Delete the rejected candidate revision - if: ${{ (failure() || cancelled()) && (env.TRAFFIC_SHIFT_ATTEMPTED != 'true' || env.TRAFFIC_ROLLED_BACK == 'true') }} + if: ${{ always() && env.RECOVERY_VERIFIED == 'true' }} shell: bash run: | set -euo pipefail @@ -428,61 +529,32 @@ jobs: echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}" echo "candidate revision ${CANDIDATE_REVISION} is absent" - # Deleting a revision does not restore the service template that Terraform reconciles. - - name: Restore the known-good service template - if: ${{ (failure() || cancelled()) && env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.CANDIDATE_DELETED == 'true' }} + # Public checks commit the serving revision; cleanup failures must not roll it back. + - name: Retire previous consumers after public checks + if: ${{ always() && (env.ROLLOUT_VERIFIED == 'true' || (env.RECOVERY_VERIFIED == 'true' && env.CANDIDATE_DELETED == 'true')) }} shell: bash run: | set -euo pipefail - test "${TRAFFIC_SHIFT_ATTEMPTED:-false}" != true || test "${TRAFFIC_ROLLED_BACK:-false}" = true - test -n "${ROLLBACK_IMAGE}" - # Shutdown allowance, not a measurement of PostgreSQL connection drain. - sleep 10 - tag="r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - echo "TEMPLATE_RECOVERY_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}" - echo "Template recovery attempted: ${SERVICE_NAME}-${tag}, image ${ROLLBACK_IMAGE}." \ - >> "${GITHUB_STEP_SUMMARY}" - # Known-good schema/workers can run here even though HTTP stays on the old revision. - gcloud run deploy "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --image "${ROLLBACK_IMAGE}" --revision-suffix "${tag}" \ - --remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet - gcloud run services describe "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - > "${RUNNER_TEMP}/push-recovered-service.json" - jq -e --arg image "${ROLLBACK_IMAGE}" --arg serving "${ROLLBACK_REVISION}" \ - --arg floor "${PUSH_MIN_INSTANCES}" --arg ceiling "${PUSH_MAX_INSTANCES}" \ - --slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" ' - def shape: del(.containers[0].image) | - .containers[0].env = ((.containers[0].env // []) | - map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name)); - .spec.template.spec.containers[0].image == $image and - all(.spec.template.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and - (.spec.template.spec | shape) == ($prior[0].spec | shape) and - .spec.template.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and - (.spec.template.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber) and - ([.status.traffic[] | select((.percent // 0) > 0)] | - length == 1 and .[0].revisionName == $serving and .[0].percent == 100) - ' "${RUNNER_TEMP}/push-recovered-service.json" > /dev/null - echo "TEMPLATE_RESTORED=true" >> "${GITHUB_ENV}" - echo 'Known-good image and normal mode restored; previous revision still serves HTTP.' \ - >> "${GITHUB_STEP_SUMMARY}" - - - name: Retire the template recovery revision - if: ${{ always() && env.TEMPLATE_RECOVERY_REVISION != '' }} - shell: bash - run: | - set -euo pipefail - existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --filter "metadata.name=${TEMPLATE_RECOVERY_REVISION}" --format='value(metadata.name)')" - if test -n "${existing}"; then - test "${existing}" = "${TEMPLATE_RECOVERY_REVISION}" - gcloud run revisions delete "${TEMPLATE_RECOVERY_REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet + serving="${CANDIDATE_REVISION}" + if test "${RECOVERY_VERIFIED:-false}" = true; then + serving="${TEMPLATE_RECOVERY_REVISION}" fi - sleep 10 - echo 'Template recovery revision retired; SQL connection drain still needs operational verification.' \ + gcloud run services update-traffic "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --clear-tags --quiet + revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')" + while IFS= read -r revision; do + test -n "${revision}" || continue + test "${revision}" != "${serving}" || continue + case "${revision}" in + "${ROLLBACK_REVISION}"|"${VALIDATION_REVISION}"|"${CANDIDATE_REVISION}") ;; + *) echo "Unexpected revision ${revision}; manual retirement required." >&2; exit 1 ;; + esac + gcloud run revisions delete "${revision}" \ + --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet + done <<< "${revisions}" + echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}" + echo 'Obsolete revision resources retired; verify actual SQL session drain operationally.' \ >> "${GITHUB_STEP_SUMMARY}" - name: Drop the candidate traffic tag diff --git a/cloud/dev/scripts/push-cloud-run-model.mjs b/cloud/dev/scripts/push-cloud-run-model.mjs new file mode 100644 index 00000000000..fea782e625d --- /dev/null +++ b/cloud/dev/scripts/push-cloud-run-model.mjs @@ -0,0 +1,140 @@ +import { readFileSync, writeFileSync } from 'node:fs' + +// Executable fake gcloud: revision deletion obeys the platform's latest/traffic constraints. +const path = process.env.MODEL_STATE +const state = JSON.parse(readFileSync(path, 'utf8')) +const args = process.argv.slice(2) +const option = (name) => args[args.indexOf(name) + 1] +const has = (name) => args.includes(name) +const fail = (message) => { + throw new Error(message) +} +const persist = () => writeFileSync(path, JSON.stringify(state)) +const output = (value) => console.log(typeof value === 'string' ? value : JSON.stringify(value)) +const revision = (name) => state.revisions[name] ?? fail(`missing revision ${name}`) +const traffic = () => [ + { revisionName: state.serving, percent: 100 }, + ...Object.entries(state.tags).map(([tag, name]) => ({ + tag, + revisionName: name, + url: `https://${tag}.test` + })) +] +state.trace.push(args.join(' ')) +try { + if (args[0] === 'curl') { + if (state.failure === 'public' && args.some((arg) => arg.includes('https://public.test'))) { + fail('public check failed') + } + const url = args.find((arg) => arg.startsWith('https://')) + const tag = new URL(url).hostname.split('.')[0] + const name = state.tags[tag] ?? state.serving + if (has('-w')) { + output('200') + } else { + output({ + ok: true, + deliveryProtocol: 2, + mode: revision(name).spec.containers[0].env.some((entry) => entry.value === 'validation') + ? 'validation' + : 'active' + }) + } + } else if (args.slice(0, 2).join(' ') === 'run deploy') { + const name = `${option('deploy')}-${option('--revision-suffix')}` + if (state.failure === 'deploy-before') { + fail('deploy failed before create') + } + const item = structuredClone(revision(state.latest)) + item.metadata.name = name + item.spec.containers[0].image = option('--image') + item.status.imageDigest = option('--image') + item.spec.containers[0].env = item.spec.containers[0].env.filter( + (entry) => entry.name !== 'ORCA_PUSH_MODE' + ) + if (has('--update-env-vars')) { + item.spec.containers[0].env.push({ name: 'ORCA_PUSH_MODE', value: 'validation' }) + } + state.revisions[name] = item + state.latest = name + if (has('--tag')) { + state.tags[option('--tag')] = name + } + state.peak = Math.max(state.peak, Object.keys(state.revisions).length) + if (state.peak > 3) { + fail('three-revision budget exceeded') + } + if (state.failure === 'deploy-after') { + fail('deploy failed after create') + } + } else if (args.slice(0, 3).join(' ') === 'run services describe') { + if (state.failure === 'describe') { + fail('describe failed') + } + if (args.some((arg) => arg.includes('value(status.latestCreatedRevisionName)'))) { + output(state.latest) + } else { + const template = structuredClone(revision(state.latest)) + delete template.spec.containers[0].name + output({ + spec: { template }, + status: { latestCreatedRevisionName: state.latest, traffic: traffic() } + }) + } + } else if (args.slice(0, 3).join(' ') === 'run services update-traffic') { + if (has('--to-revisions')) { + const name = option('--to-revisions').split('=')[0] + revision(name) + state.serving = name + } + if (has('--remove-tags')) { + for (const tag of option('--remove-tags').split(',')) { + delete state.tags[tag] + } + } + if (has('--clear-tags')) { + state.tags = {} + } + if (state.failure === 'traffic-after') { + fail('traffic changed but response failed') + } + } else if (args.slice(0, 3).join(' ') === 'run revisions list') { + const names = Object.keys(state.revisions) + output( + (has('--filter') + ? names.filter((name) => name === option('--filter').split('=')[1]) + : names + ).join('\n') + ) + } else if (args.slice(0, 3).join(' ') === 'run revisions describe') { + const item = revision(args[3]) + const format = args.find((arg) => arg.startsWith('--format=')) ?? option('--format') + if (format.includes('minScale')) { + output(item.metadata.annotations['autoscaling.knative.dev/minScale']) + } else if (format.includes('maxScale')) { + output(item.metadata.annotations['autoscaling.knative.dev/maxScale']) + } else { + output(item) + } + } else if (args.slice(0, 3).join(' ') === 'run revisions delete') { + const name = args[3] + if (name === state.latest) { + fail('FAILED_PRECONDITION: latest created Revision cannot be directly deleted') + } + if (name === state.serving || Object.values(state.tags).includes(name)) { + fail('revision has traffic or tags') + } + if (state.failure === 'delete') { + fail('delete failed') + } + revision(name) + delete state.revisions[name] + } else { + fail(`unmodeled gcloud call ${args.join(' ')}`) + } +} catch (error) { + console.error(error.message) + process.exitCode = 1 +} finally { + persist() +} diff --git a/cloud/dev/scripts/push-gateway-recovery.test.mjs b/cloud/dev/scripts/push-gateway-recovery.test.mjs index 6a3e4d48bac..97d9c4b5539 100644 --- a/cloud/dev/scripts/push-gateway-recovery.test.mjs +++ b/cloud/dev/scripts/push-gateway-recovery.test.mjs @@ -1,7 +1,8 @@ import assert from 'node:assert/strict' -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { fileURLToPath } from 'node:url' import { spawnSync } from 'node:child_process' import test from 'node:test' import { readRelayWorkflow } from './relay-repository.mjs' @@ -12,272 +13,361 @@ function step(name) { assert.notEqual(start, -1) const end = workflow.indexOf('\n - ', start + 1) const block = workflow.slice(start, end === -1 ? undefined : end) - return block.slice(block.indexOf(' run: |\n') + ' run: |\n'.length) - .split('\n').filter((line) => line.startsWith(' ')).map((line) => line.slice(10)).join('\n') + return block + .slice(block.indexOf(' run: |\n') + ' run: |\n'.length) + .split('\n') + .filter((line) => line.startsWith(' ')) + .map((line) => line.slice(10)) + .join('\n') } -const candidate = step('Deploy the candidate revision with no traffic') -const shift = step('Shift all traffic to the verified candidate') -const rollback = step('Roll traffic back to the previous revision') -const cleanup = step('Delete the rejected candidate revision') -const env = { SERVICE_NAME: 'push-test', GCP_PROJECT_ID: 'test', GCP_REGION: 'test', - GITHUB_RUN_ID: '123', GITHUB_RUN_ATTEMPT: '1', IMAGE: 'synthetic-image', - CANDIDATE_REVISION: 'push-test-c123-1', ROLLBACK_REVISION: 'push-test-old', - ROLLBACK_IMAGE: 'registry/push@sha256:' + 'a'.repeat(64), PUSH_MIN_INSTANCES: '1', PUSH_MAX_INSTANCES: '2' } - -function exercise(body, setup = () => {}) { +const names = { + preflight: 'Record the serving revision and require its Terraform-owned scaling', + candidate: 'Deploy the candidate revision with no traffic', + activate: 'Retire inert validation and activate the verified image', + shift: 'Shift all traffic to the verified candidate', + public: 'Verify the public origin after the shift', + rollback: 'Roll traffic back to the previous revision', + restore: 'Restore the known-good service template', + promoteRecovery: 'Promote and verify the known-good recovery revision', + cleanup: 'Delete the rejected candidate revision', + retire: 'Retire previous consumers after public checks' +} +const image = `registry/push@sha256:${'a'.repeat(64)}` +const spec = { + serviceAccountName: 'runtime@test', + containerConcurrency: 40, + containers: [ + { + image, + name: 'push-test-1', + env: [ + { + name: 'ORCA_PUSH_DATABASE_URL', + valueFrom: { secretKeyRef: { name: 'database', key: '7' } } + }, + { name: 'ORCA_PUSH_DATABASE_POOL_MAX', value: '2' } + ] + } + ] +} +const prior = { + metadata: { + name: 'push-test-old', + annotations: { + 'autoscaling.knative.dev/minScale': '1', + 'autoscaling.knative.dev/maxScale': '2' + } + }, + spec, + status: { imageDigest: image } +} +const model = fileURLToPath(new URL('./push-cloud-run-model.mjs', import.meta.url)) +const options = { skip: process.platform === 'win32' } +function exercise(callback) { const dir = mkdtempSync(join(tmpdir(), 'push-workflow-')) + const statePath = join(dir, 'state.json') + writeFileSync( + statePath, + JSON.stringify({ + revisions: { 'push-test-old': prior }, + latest: 'push-test-old', + serving: 'push-test-old', + tags: {}, + peak: 1, + trace: [] + }) + ) + writeFileSync(join(dir, 'env'), '') + const env = { + ...process.env, + SERVICE_NAME: 'push-test', + GCP_PROJECT_ID: 'test', + GCP_REGION: 'test', + GITHUB_RUN_ID: '123', + GITHUB_RUN_ATTEMPT: '1', + IMAGE: `registry/push@sha256:${'b'.repeat(64)}`, + PUSH_MIN_INSTANCES: '1', + PUSH_MAX_INSTANCES: '2', + PUSH_RUNTIME_SERVICE_ACCOUNT: 'runtime@test', + PUSH_ORIGIN: 'https://public.test', + MODEL_STATE: statePath, + MODEL_SCRIPT: model, + RUNNER_TEMP: dir, + GITHUB_ENV: join(dir, 'env'), + GITHUB_STEP_SUMMARY: join(dir, 'summary') + } + const state = () => JSON.parse(readFileSync(statePath, 'utf8')) + const change = (edit) => { + const value = state() + edit(value) + writeFileSync(statePath, JSON.stringify(value)) + } + const run = (key, ok = true, extra = '') => { + const result = spawnSync( + 'bash', + [ + '-c', + ` + set -a + source "$GITHUB_ENV" + gcloud() { node "$MODEL_SCRIPT" "$@"; } + curl() { node "$MODEL_SCRIPT" curl "$@"; } + sleep() { :; } + ${extra} + ${names[key] ? step(names[key]) : key} + ` + ], + { cwd: dir, env, encoding: 'utf8', timeout: 30000 } + ) + assert.equal(result.status === 0, ok, `${key}: ${result.stderr}\n${result.stdout}`) + return result + } try { - setup(dir) - const run = spawnSync('bash', ['-c', body], { encoding: 'utf8', timeout: 10000, cwd: dir, - env: { ...process.env, ...env, RUNNER_TEMP: dir, GITHUB_ENV: join(dir, 'env'), GITHUB_STEP_SUMMARY: join(dir, 'summary'), - TRACE: join(dir, 'trace'), STATE: join(dir, 'state') } }) - assert.equal(run.status, 0, run.stderr) - } finally { rmSync(dir, { recursive: true, force: true }) } + callback({ run, state, change, dir }) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} +function recover(h) { + h.change((state) => { + delete state.failure + }) + h.run('restore') + h.run('promoteRecovery') + h.run('cleanup') + h.run('retire') + const state = h.state() + assert.equal(state.serving, 'push-test-r123-1') + assert.equal(state.latest, state.serving) + assert.deepEqual(Object.keys(state.revisions), [state.serving]) + assert.equal(state.revisions[state.serving].spec.containers[0].image, image) + assert.ok(state.peak <= 3) } -// Workflow shell behavior is Linux-specific; these tests never call a real cloud CLI. -test('failed candidate discovery retains enough state to remove tag and revision', { skip: process.platform === 'win32' }, () => { - exercise(` - gcloud() { - case "$*" in - 'run deploy '*) echo deployed > "$STATE" ;; - 'run services describe '*) return 1 ;; - 'run revisions list '*) echo "$CANDIDATE_REVISION" ;; - *) echo "$*" >> "$TRACE" ;; - esac - } - jq() { return 1; } - ( ${candidate} ) - test "$?" != 0 || exit 1 - source "$GITHUB_ENV" - test "$CANDIDATE_TAG" = c123-1 || exit 1 - test "$CANDIDATE_REVISION" = push-test-c123-1 || exit 1 - ( ${cleanup} ) || exit 1 - grep -q -- '--remove-tags c123-1' "$TRACE" || exit 1 - grep -q 'run revisions delete push-test-c123-1' "$TRACE" || exit 1 - `) -}) +test( + 'the executable Cloud Run model rejects deleting latest even without tags or traffic', + options, + () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.run('gcloud run services update-traffic "$SERVICE_NAME" --clear-tags') + const result = h.run('gcloud run revisions delete "$CANDIDATE_REVISION"', false) + assert.match(result.stderr, /FAILED_PRECONDITION: latest created Revision/) + }) +) -test('failed post-promotion read retains intent and restores previous traffic', { skip: process.platform === 'win32' }, () => { - exercise(` - gcloud() { - case "$*" in - 'run services update-traffic '*) echo "$*" >> "$TRACE" ;; - 'run services describe '*) return 1 ;; - esac - } - jq() { return 1; } - ( ${shift} ) - test "$?" != 0 || exit 1 - source "$GITHUB_ENV" - test "$TRAFFIC_SHIFT_ATTEMPTED" = true || exit 1 - gcloud() { - case "$*" in - 'run services update-traffic '*) echo "$*" >> "$TRACE" ;; - 'run services describe '*) echo '{}' ;; - esac - } - jq() { echo "$ROLLBACK_REVISION"; } - ( ${rollback} ) || exit 1 - source "$GITHUB_ENV" - test "$TRAFFIC_ROLLED_BACK" = true || exit 1 - grep -q -- '--to-revisions push-test-old=100' "$TRACE" || exit 1 - `) -}) +test( + 'success creates successor before retirement and repeated rollouts retain one consumer', + options, + () => + exercise((h) => { + for (const attempt of ['1', '2']) { + if (attempt === '2') { + writeFileSync(join(h.dir, 'env'), 'GITHUB_RUN_ATTEMPT=2\n') + } + for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public', 'retire']) { + h.run(key) + } + const state = h.state() + assert.equal(state.serving, `push-test-a123-${attempt}`) + assert.deepEqual(Object.keys(state.revisions), [state.serving]) + assert.equal(state.peak, 3) + } + }) +) -test('ambiguous promotion failure also leaves rollback intent', { skip: process.platform === 'win32' }, () => { - exercise(` - gcloud() { return 1; } - ( ${shift} ) - test "$?" != 0 || exit 1 - source "$GITHUB_ENV" - test "$TRAFFIC_SHIFT_ATTEMPTED" = true - `) -}) +for (const failure of ['deploy-before', 'deploy-after', 'describe']) { + test(`validation ${failure} recovers without deleting latest`, options, () => + exercise((h) => { + h.run('preflight') + h.change((state) => { + state.failure = failure + }) + h.run('candidate', false) + recover(h) + }) + ) +} +for (const failure of ['deploy-before', 'deploy-after', 'delete', 'describe']) { + test( + `activation ${failure} frees validation slot before recovery and stays within three`, + options, + () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.change((state) => { + state.failure = failure + }) + h.run('activate', false) + recover(h) + }) + ) +} -const activate = step('Retire inert validation and activate the verified image') -test('partial activation records the new revision before deploy and deletes its consumer', { skip: process.platform === 'win32' }, () => { - exercise(` - CANDIDATE_TAG=c123-1 - sleep() { :; } - gcloud() { - echo "$*" >> "$TRACE" - case "$*" in - 'run deploy '*) return 1 ;; - 'run revisions list '*) echo "$CANDIDATE_REVISION" ;; - esac - } - ( ${activate} ) - test "$?" != 0 || exit 1 - source "$GITHUB_ENV" - test "$ACTIVATION_ATTEMPTED" = true || exit 1 - test "$CANDIDATE_REVISION" = push-test-a123-1 || exit 1 - test "$CANDIDATE_TAG" = a123-1 || exit 1 - ( ${cleanup} ) || exit 1 - grep -q 'run revisions delete push-test-c123-1' "$TRACE" || exit 1 - grep -q 'run revisions delete push-test-a123-1' "$TRACE" || exit 1 - grep -q -- '--remove-env-vars ORCA_PUSH_MODE' "$TRACE" || exit 1 - test "$(grep -n 'run revisions delete push-test-c123-1' "$TRACE" | cut -d: -f1)" -lt \ - "$(grep -n 'run deploy' "$TRACE" | cut -d: -f1)" || exit 1 - `) -}) +test( + 'ambiguous traffic shift records intent before mutation, rolls back and recovers', + options, + () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.run('activate') + h.change((state) => { + state.failure = 'traffic-after' + }) + h.run('shift', false) + assert.match(readFileSync(join(h.dir, 'env'), 'utf8'), /TRAFFIC_SHIFT_ATTEMPTED=true/) + h.change((state) => { + delete state.failure + }) + h.run('rollback') + recover(h) + }) +) -test('failed validation retirement never activates another consumer', { skip: process.platform === 'win32' }, () => { - exercise(` - CANDIDATE_TAG=c123-1 - gcloud() { - echo "$*" >> "$TRACE" - case "$*" in - 'run revisions delete '*) return 1 ;; - esac +test('failed public check rolls back and recovers', options, () => + exercise((h) => { + for (const key of ['preflight', 'candidate', 'activate', 'shift']) { + h.run(key) } - ( ${activate} ) - test "$?" != 0 || exit 1 - ! grep -q 'run deploy' "$TRACE" || exit 1 - ! grep -q ACTIVATION_ATTEMPTED "$GITHUB_ENV" 2>/dev/null || exit 1 - `) -}) + h.change((state) => { + state.failure = 'public' + }) + h.run('public', false) + h.change((state) => { + delete state.failure + }) + h.run('rollback') + recover(h) + }) +) + +for (const defect of [ + 'runtime', + 'secret', + 'mode', + 'image', + 'traffic', + 'scaling', + 'deploy-before', + 'deploy-after' +]) { + test(`recovery rejects ${defect} and preserves partial-create state`, options, () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.change((state) => { + const revision = state.revisions[state.latest] + if (defect === 'runtime') { + revision.spec.serviceAccountName = 'wrong@test' + } + if (defect === 'secret') { + revision.spec.containers[0].env[0].valueFrom.secretKeyRef.key = '8' + } + if (defect === 'scaling') { + revision.metadata.annotations['autoscaling.knative.dev/maxScale'] = '3' + } + if (defect === 'traffic') { + state.serving = state.latest + } + if (defect.startsWith('deploy-')) { + state.failure = defect + } + }) + // Corrupt the recovery response after the modeled deploy while keeping real jq assertions. + const extra = ['mode', 'image'].includes(defect) + ? ` + gcloud() { + node "$MODEL_SCRIPT" "$@" > "$RUNNER_TEMP/out" || return $? + if [[ "$*" == 'run services describe '* && "$*" == *'--format=json'* ]]; then + jq '${defect === 'mode' ? '.spec.template.spec.containers[0].env += [{name:"ORCA_PUSH_MODE",value:"validation"}]' : '.spec.template.spec.containers[0].image = "wrong"'}' "$RUNNER_TEMP/out" + else cat "$RUNNER_TEMP/out"; fi + }` + : '' + h.run('restore', false, extra) + const recorded = readFileSync(join(h.dir, 'env'), 'utf8') + assert.match(recorded, /TEMPLATE_RECOVERY_REVISION=push-test-r123-1/) + assert.doesNotMatch(recorded, /TEMPLATE_RESTORED=true/) + assert.ok(h.state().peak <= 3) + }) + ) +} + +test('failed validation retirement blocks a fourth revision during recovery', options, () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.change((state) => { + state.failure = 'delete' + }) + h.run('activate', false) + h.run('restore', false) + assert.equal(h.state().peak, 3) + assert.equal(h.state().revisions['push-test-r123-1'], undefined) + }) +) + +test( + 'failed retirement after public checks leaves verified serving and blocks the next run', + options, + () => + exercise((h) => { + for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public']) { + h.run(key) + } + h.change((state) => { + state.failure = 'delete' + }) + h.run('retire', false) + assert.equal(h.state().serving, 'push-test-a123-1') + h.run('preflight', false) + assert.match(workflow, /env.ROLLOUT_VERIFIED != 'true'/) + }) +) + +test( + 'recovery promotion failure keeps consumers for operator diagnosis and blocks new rollout', + options, + () => + exercise((h) => { + h.run('preflight') + h.run('candidate') + h.run('restore') + h.change((state) => { + state.failure = 'public' + }) + h.run('promoteRecovery', false) + assert.doesNotMatch(readFileSync(join(h.dir, 'env'), 'utf8'), /RECOVERY_VERIFIED=true/) + h.run('preflight', false) + }) +) const capability = step('Require image support for inert validation') -for (const [label, source, expected] of [ - ['old image', 'export function loadPushConfig() { return {}; }', 1], - ['invalid mode accepted', 'export function loadPushConfig(env) { return { mode: env.ORCA_PUSH_MODE }; }', 1], - ['validation supported', `export function loadPushConfig(env) { +for (const [label, source, ok] of [ + ['old image', 'export function loadPushConfig() { return {}; }', false], + [ + 'invalid mode accepted', + 'export function loadPushConfig(env) { return { mode: env.ORCA_PUSH_MODE }; }', + false + ], + [ + 'validation supported', + `export function loadPushConfig(env) { if (env.ORCA_PUSH_MODE !== 'validation') throw new Error('invalid mode'); return { mode: 'validation' }; - }`, 0] + }`, + true + ] ]) { - test(`pre-production image smoke: ${label}`, { skip: process.platform === 'win32' }, () => { - exercise(` - docker() { node "\${@: -3}"; } - ( ${capability} ) - test "$?" = ${expected} - `, (dir) => { - const dist = join(dir, 'apps', 'push', 'dist') + test(`pre-production image smoke: ${label}`, options, () => + exercise((h) => { + const dist = join(h.dir, 'apps', 'push', 'dist') mkdirSync(dist, { recursive: true }) - writeFileSync(join(dir, 'package.json'), '{"type":"module"}') + writeFileSync(join(h.dir, 'package.json'), '{"type":"module"}') writeFileSync(join(dist, 'config.js'), source) + h.run(capability, ok, 'docker() { node "${@: -3}"; }') }) - }) + ) } - -const restore = step('Restore the known-good service template') -const priorSpec = { serviceAccountName: 'runtime@test', containerConcurrency: 40, - containers: [{ image: env.ROLLBACK_IMAGE, env: [ - { name: 'ORCA_PUSH_DATABASE_URL', valueFrom: { secretKeyRef: { name: 'database', key: '7' } } }, - { name: 'ORCA_PUSH_DATABASE_POOL_MAX', value: '2' } - ] }] } -const recoveredService = { spec: { template: { spec: priorSpec, metadata: { annotations: { - 'autoscaling.knative.dev/minScale': '1', 'autoscaling.knative.dev/maxScale': '2' -} } } }, status: { traffic: [{ revisionName: env.ROLLBACK_REVISION, percent: 100 }] } } -function recoveryFiles(dir, service = recoveredService) { - writeFileSync(join(dir, 'push-rollback-revision.json'), JSON.stringify({ spec: priorSpec })) - writeFileSync(join(dir, 'recovered.json'), JSON.stringify(service)) -} - -test('recovery requires cleanup success and restores no traffic before verified rollback', { skip: process.platform === 'win32' }, () => { - const block = workflow.slice(workflow.indexOf('- name: Restore the known-good service template')) - assert.match(block, /env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.CANDIDATE_DELETED == 'true'/) - assert.ok(workflow.indexOf('- name: Delete the rejected candidate revision') < - workflow.indexOf('- name: Restore the known-good service template')) - exercise(` - TRAFFIC_SHIFT_ATTEMPTED=true - gcloud() { echo unexpected >> "$TRACE"; } - ( ${restore} ) - test "$?" != 0 || exit 1 - test ! -e "$TRACE" - `) -}) - -for (const absent of [false, true]) { - test(`failed validation restores known-good template after candidate ${absent ? 'was never created' : 'deletion'}`, { skip: process.platform === 'win32' }, () => { - exercise(` - CANDIDATE_TAG=c123-1 - sleep() { echo shutdown-allowance >> "$TRACE"; } - gcloud() { - echo "$*" >> "$TRACE" - case "$*" in - 'run revisions list '*) ${absent ? ':' : 'echo "$CANDIDATE_REVISION"'} ;; - 'run services describe '*) cat "$RUNNER_TEMP/recovered.json" ;; - esac - } - ( ${cleanup} ) || exit 1 - source "$GITHUB_ENV" - test "$CANDIDATE_DELETED" = true || exit 1 - ( ${restore} ) || exit 1 - source "$GITHUB_ENV" - test "$TEMPLATE_RESTORED" = true || exit 1 - grep -q -- '--image registry/push@sha256:' "$TRACE" || exit 1 - grep -q -- '--remove-env-vars ORCA_PUSH_MODE --no-traffic' "$TRACE" || exit 1 - test "$(grep -n shutdown-allowance "$TRACE" | cut -d: -f1)" -lt \ - "$(grep -n 'run deploy' "$TRACE" | cut -d: -f1)" - `, recoveryFiles) - }) -} - -test('failed candidate deletion does not authorize template recovery', { skip: process.platform === 'win32' }, () => { - exercise(` - gcloud() { - case "$*" in - 'run revisions list '*) echo "$CANDIDATE_REVISION" ;; - 'run revisions delete '*) return 1 ;; - esac - } - ( ${cleanup} ) - test "$?" != 0 || exit 1 - ! grep -q CANDIDATE_DELETED=true "$GITHUB_ENV" 2>/dev/null - `) -}) - -for (const defect of ['runtime', 'secret', 'mode', 'image', 'traffic', 'scaling', 'deploy']) { - test(`template recovery rejects ${defect} failure and records attempted revision`, { skip: process.platform === 'win32' }, () => { - const service = structuredClone(recoveredService) - if (defect === 'runtime') service.spec.template.spec.serviceAccountName = 'wrong@test' - if (defect === 'secret') service.spec.template.spec.containers[0].env[0].valueFrom.secretKeyRef.key = '8' - if (defect === 'mode') service.spec.template.spec.containers[0].env.push({ name: 'ORCA_PUSH_MODE', value: 'validation' }) - if (defect === 'image') service.spec.template.spec.containers[0].image = 'rejected' - if (defect === 'traffic') service.status.traffic[0].revisionName = 'rejected' - if (defect === 'scaling') service.spec.template.metadata.annotations['autoscaling.knative.dev/maxScale'] = '3' - exercise(` - sleep() { :; } - gcloud() { - case "$*" in - 'run deploy '*) ${defect === 'deploy' ? 'return 1' : ':'} ;; - 'run services describe '*) cat "$RUNNER_TEMP/recovered.json" ;; - esac - } - ( ${restore} ) - test "$?" != 0 || exit 1 - source "$GITHUB_ENV" - test "$TEMPLATE_RECOVERY_REVISION" = push-test-r123-1 || exit 1 - test -z "\${TEMPLATE_RESTORED:-}" - `, (dir) => recoveryFiles(dir, service)) - }) -} - -const retireRecovery = step('Retire the template recovery revision') -for (const absent of [false, true]) { - test(`recovery retirement handles ${absent ? 'partial creation without a revision' : 'an existing recovery consumer'}`, { skip: process.platform === 'win32' }, () => { - exercise(` - TEMPLATE_RECOVERY_REVISION=push-test-r123-1 - sleep() { echo shutdown-allowance >> "$TRACE"; } - gcloud() { - echo "$*" >> "$TRACE" - case "$*" in - 'run revisions list '*) ${absent ? ':' : 'echo "$TEMPLATE_RECOVERY_REVISION"'} ;; - esac - } - ( ${retireRecovery} ) || exit 1 - ${absent ? '!' : ''} grep -q 'run revisions delete' "$TRACE" || exit 1 - grep -q shutdown-allowance "$TRACE" - `) - }) -} - -test('recovery retirement runs after success or failure without mutating the restored template', () => { - const block = workflow.slice(workflow.indexOf('- name: Retire the template recovery revision'), - workflow.indexOf('- name: Drop the candidate traffic tag')) - assert.match(block, /always\(\) && env.TEMPLATE_RECOVERY_REVISION != ''/) - assert.doesNotMatch(retireRecovery, /run (deploy|services update|services replace)/) - assert.match(workflow, /image="\$\(jq -er '\.status.imageDigest'/) -}) diff --git a/cloud/dev/scripts/push-gateway-workflow.test.mjs b/cloud/dev/scripts/push-gateway-workflow.test.mjs index b208f5f7db7..d0a7efd3386 100644 --- a/cloud/dev/scripts/push-gateway-workflow.test.mjs +++ b/cloud/dev/scripts/push-gateway-workflow.test.mjs @@ -240,7 +240,7 @@ test('the summary is written before anything that can fail after the shift', () const summary = indexOfStep('Publish the rollout summary') assert.ok(summary > indexOfStep('Shift all traffic to the verified candidate')) assert.ok(summary < indexOfStep('Verify the public origin after the shift')) - assert.match(workflow, /--to-revisions \$\{ROLLBACK_REVISION\}=100/) + assert.match(workflow, /Known-good image:/) assert.match(workflow, /GITHUB_STEP_SUMMARY/) }) @@ -262,7 +262,7 @@ test('a failure after the shift rolls production back automatically', () => { ) assert.match( body, - /if: \$\{\{ \(failure\(\) \|\| cancelled\(\)\) && env\.TRAFFIC_SHIFT_ATTEMPTED == 'true' \}\}/, + /if: \$\{\{ \(failure\(\) \|\| cancelled\(\)\) && env\.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env\.ROLLOUT_VERIFIED != 'true' \}\}/, 'the rollback must be conditioned on both failure and the shift marker' ) assert.match(body, /test -n "\$\{ROLLBACK_REVISION:-\}"/) @@ -273,15 +273,15 @@ test('a failure after the shift rolls production back automatically', () => { // Why: a candidate that never took traffic still holds a warm instance and a Cloud SQL pool. Its // tag comes off first, because Cloud Run refuses to delete a revision a traffic target names. -test('a failure before the shift deletes the candidate it created', () => { +test('verified recovery authorizes rejected candidate deletion', () => { const body = workflow.slice( workflow.indexOf('- name: Delete the rejected candidate revision'), workflow.indexOf('- name: Drop the candidate traffic tag') ) assert.match( body, - /env\.TRAFFIC_SHIFT_ATTEMPTED != 'true' \|\| env\.TRAFFIC_ROLLED_BACK == 'true'/, - 'the cleanup must be conditioned on both failure and the absence of the shift marker' + /env\.RECOVERY_VERIFIED == 'true'/, + 'cleanup must wait for verified recovery traffic and public checks' ) assert.match(body, /if test -z "\$\{CANDIDATE_REVISION:-\}"; then/) assert.ok( @@ -308,3 +308,8 @@ test('push credentials cannot assume the shared Relay deploy identity', () => { assert.doesNotMatch(workflow, /PRODUCTION_GCP_RELAY_DEPLOY_/) assert.doesNotMatch(terraform('push-gateway.tf'), /member\s*=\s*local\.relay_github_deploy_service_account_member/) }) + +// A latest revision needs a successor even when validation is inert. +test('dedicated database admits three simultaneous revision pools', () => { + assert.match(terraform('push-gateway.tf'), /var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/) +}) diff --git a/cloud/dev/scripts/push-validation-workflow.test.mjs b/cloud/dev/scripts/push-validation-workflow.test.mjs index 510d5b64fcf..e26769cfeb3 100644 --- a/cloud/dev/scripts/push-validation-workflow.test.mjs +++ b/cloud/dev/scripts/push-validation-workflow.test.mjs @@ -15,7 +15,7 @@ const activation = position('Retire inert validation and activate the verified i const shift = position('Shift all traffic to the verified candidate') test('the exact build digest must support validation before production boot', () => { - assert.match(workflow, /docker buildx build --push --metadata-file/) + assert.match(workflow, /docker buildx build --push --platform linux\/amd64 --provenance=false --metadata-file/) assert.match(workflow, /containerimage\.digest/) assert.doesNotMatch(workflow, /gcloud artifacts docker images describe/) assert.ok(capability < deploy) @@ -30,7 +30,7 @@ test('inert validation and credential checks precede deliberate activation of th assert.match(workflow.slice(deploy, activation), /\.mode == "validation"/) assert.ok(position('Prove the runtime identity can reach FCM') < activation) const active = workflow.slice(activation, shift) - assert.ok(active.indexOf('gcloud run revisions delete') < active.indexOf('gcloud run deploy')) + assert.ok(active.indexOf('gcloud run deploy') < active.indexOf('gcloud run revisions delete')) assert.match(active, /--image "\$\{IMAGE\}"/) assert.match(active, /--remove-env-vars ORCA_PUSH_MODE/) assert.match(active, /\.spec\.containers\[0\]\.image == \$image/) diff --git a/cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs b/cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs index 2ad1d653007..9f0ee3d7eaa 100644 --- a/cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs +++ b/cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs @@ -73,8 +73,8 @@ export function calculateRelayCloudSqlConnectionBudget(inputs) { relayDirectorCandidate: retainedDirectorRollback * 2, apiCandidate: retainedDirectorRollback + inputs.apiInstances * inputs.apiPoolMax, authCandidate: retainedDirectorRollback + inputs.authInstances * inputs.authPoolMax, - // Serving push pools are already in configuredMaximum; the tagged candidate adds one copy. - pushCandidate: retainedDirectorRollback + pushDraw, + // Serving is already counted; validation/rejected and its successor add two pools. + pushCandidate: retainedDirectorRollback + pushDraw * 2, relayCells: retainedDirectorRollback } const rolloutOverlap = Math.max(...Object.values(candidateOverlap)) diff --git a/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs b/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs index a1613eb2f43..40f46cdf4cf 100644 --- a/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs +++ b/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs @@ -22,7 +22,7 @@ test('production plus the push gateway keeps allowance and reserve below the cei assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30) assert.equal(report.rolloutOverlap.apiCandidate, 65) assert.equal(report.rolloutOverlap.authCandidate, 35) - assert.equal(report.rolloutOverlap.pushCandidate, 19) + assert.equal(report.rolloutOverlap.pushCandidate, 23) assert.equal(report.rolloutOverlap.relayCells, 15) assert.equal(report.rolloutOverlap.retainedDirectorRollback, 15) // The gateway does not set the maximum; the API candidate does, as it did before it existed. @@ -65,10 +65,10 @@ test('the same relay shape without the gateway stays inside the ceiling', () => assert.equal(report.withinBudget, true) }) -// Why: a tagged candidate is directly addressable and sits outside the service-wide cap, so both +// Why: a tagged candidate is directly addressable and sits outside the service-wide cap, so all three // push revisions can reach the ceiling at once. The API and auth candidates add one copy; this // one adds two, like the director candidate. -test('the push rollout scenario doubles the gateway draw over the retained director', () => { +test('the push rollout scenario triples the gateway draw over the retained director', () => { const report = calculateRelayCloudSqlConnectionBudget({ cellPoolTotal: 0, asiaCellCount: 0, @@ -87,8 +87,8 @@ test('the push rollout scenario doubles the gateway draw over the retained direc }) assert.equal(report.consumers.push, 4) - // 15 retained director rollback, plus the 4-connection draw counted twice. - assert.equal(report.rolloutOverlap.pushCandidate, 19) + // Serving is in the base; overlap adds 15 retained director plus two 4-connection pools. + assert.equal(report.rolloutOverlap.pushCandidate, 23) }) test('fails closed when pool growth consumes the explicit reserve', () => { @@ -176,7 +176,7 @@ test('a tfvars push_max_instances override wins over the variable default', () = }) assert.equal(report.consumers.push, 6) - assert.equal(report.rolloutOverlap.pushCandidate, 9) + assert.equal(report.rolloutOverlap.pushCandidate, 15) }) test('requires strict headroom below the physical ceiling', () => { diff --git a/cloud/docs/push-database-cutover.md b/cloud/docs/push-database-cutover.md index 7218d301b15..3bc60d96dca 100644 --- a/cloud/docs/push-database-cutover.md +++ b/cloud/docs/push-database-cutover.md @@ -45,7 +45,7 @@ tier, backup policy, and regional availability. No Cloud Run service changes in Do not apply a plan that would shift traffic or revert runtime configuration. 4. Dispatch `cloud-push-deploy.yml` from main with the exact reviewed source SHA. It creates an inert, read-only candidate inheriting the dedicated attachment, probes readiness and provider - access, then deletes it and deliberately activates the same digest under the same lease. + access, then deliberately creates an active successor of the same digest before deleting validation. Activation starts schema writes and workers before HTTP promotion. Verify the candidate's SQL attachment and pinned secret reference as well as its image and health. 5. Register a test phone against the deployed origin and prove real APNs delivery. Check @@ -53,11 +53,16 @@ tier, backup policy, and regional availability. No Cloud Run service changes in connections have drained. Leave the old database intact; do not delete shared resources. If activation fails before promotion, the existing HTTP serving revision is unchanged, but -activated workers may already have sent notifications or mutated the queue. Delete the rejected -revision to stop those workers; traffic rollback alone does not stop consumers. After cleanup, -the workflow restores the known-good image and normal mode in the service template, verifies -runtime settings and secret references, and retires the untagged recovery revision. Recovery -can run known-good schema/workers; it does not undo earlier queue or schema changes. +activated workers may already have sent notifications or mutated the queue. Cloud Run will not +delete the latest created revision, even untagged at zero traffic. Recovery creates a known-good +successor first, verifies its template/runtime/secret shape and health, promotes and verifies it, +then deletes rejected and previous consumers. The recovery successor remains serving; it can run +known-good schema/workers before promotion and does not undo earlier queue or schema changes. +A partial activation leaving three resources must retire non-latest inert validation before +recovery creates another; failed retirement stops automation. Every deploy requires a single +serving revision resource at admission, so review and retire historical/leftover revisions under +the lease before dispatch. A Terraform attachment update can itself create such a revision: +verify/promote that known-good image and attachment and retire the former revision before dispatch. The deploy workflow can roll traffic back on failure; in this internal reset rollout, that may discard registrations created during the probe window. After successful activation, application rollback should retain the dedicated attachment and deploy an older compatible @@ -67,8 +72,8 @@ not a lossless rollback. Future public migrations require a separately rehearsed ## Capacity and resizing The initial gateway keeps its existing two-connection pool and two-instance maximum. -Dedicated database rollout pools are capped at 64 total connections across serving and -candidate revisions, leaving room for maintenance and operators; this is an admission +Dedicated database rollout pools are capped at 64 total configured pool connections across three simultaneous +revision resources (serving, validation/rejected, and active/recovery successor), leaving room for maintenance and operators; this is an admission budget, not a throughput claim. Increase the pool only after measuring deployed contention. Keep the shared database allocation reserved until source connections have drained. diff --git a/cloud/docs/push-gateway.md b/cloud/docs/push-gateway.md index f2fb245f105..677312b139c 100644 --- a/cloud/docs/push-gateway.md +++ b/cloud/docs/push-gateway.md @@ -36,7 +36,7 @@ coalescing window lives in instance memory, so the floor is what keeps a notific ceiling is a different question, answered below. The maximum and the pool are set by the connection budget, not by the gateway's own appetite. Two -instances times a two-connection pool is a draw of 4, and a rollout doubles it to 8, because the +instances times a two-connection pool is a draw of 4, and a rollout triples it to 12, because the tagged candidate is directly addressable and sits outside the service-wide cap. The shared Cloud SQL instance's 400 connections were already spoken for by the relay cells, the directors, auth, and the API, which left five. Four is the whole of the room there was, and the gateway fits in @@ -181,61 +181,70 @@ asserts Terraform-owned scaling. It deploys a tagged, zero-traffic validation re prove schema compatibility, provider delivery, or active-worker readiness. Container probes can still use `/health` without treating an inert process as unhealthy. +The build explicitly targets `linux/amd64` with provenance disabled so build metadata records a +single manifest digest, rather than an OCI index that Cloud Run resolves to a different digest. The workflow verifies the exact image and scaling, probes readiness and mode, and checks the -runtime identity with a validate-only FCM request. It then removes the tag and deletes validation, -allowing ten seconds for shutdown before starting another revision. This orders the rollout -within the two-revision connection budget; the delay is not proof of SQL connection drain. -Verify revision termination and SQL sessions during controlled rollout acceptance. +runtime identity with a validate-only FCM request. Cloud Run rejects deletion of the latest +created revision even when it has no tag or traffic. Activation therefore creates a successor +before removing the validation tag and deleting validation. The dedicated 64-connection budget +and legacy shared budget reserve three simultaneous revision pools: serving, validation/rejected, +and active/recovery successor (12 configured pool connections at the current two-by-two shape). +Revision deletion is not proof of physical SQL session drain; verify termination and SQL sessions +in controlled rollout acceptance. There is no shutdown sleep used as a drain gate. -**The next step deliberately activates production effects.** It deploys a distinct revision of -the exact validated digest, removing the validation override so the default `active` mode applies. -Schema setup runs on its existing one-connection untimed pool, followed by workers and pruners. -These can mutate production and send notifications **before HTTP traffic moves**. The workflow -checks the active revision's digest, runtime identity, scaling, readiness and mode, then moves all -HTTP traffic and checks the public origin. The summary records activation intent and rollback. -Terraform continues to own configuration and scaling; the temporary validation environment entry -is removed on activation, so no new ignored Terraform field is needed. +**Activation deliberately starts production effects.** The distinct active revision uses the exact +validated digest with the validation override removed. Schema setup runs on its existing +one-connection untimed pool, followed by workers and pruners, before HTTP promotion. The workflow +checks digest, full runtime spec and secret-reference shape, scaling, readiness and active mode, +then moves HTTP traffic and checks the public origin. Those checks commit the new serving revision; +subsequent retirement failures do not trigger rollback to a possibly deleted previous revision. +The previous consumer is retired and all tags are cleared. Retain the previous immutable image +from the summary: later recovery redeploys that digest, because the previous revision is deleted. -Failure before activation deletes the inert candidate. Failure during activation also deletes the -partially created active revision when traffic has not moved. After any attempted traffic shift, -the workflow first restores and verifies previous traffic, then removes the candidate tag and -deletes the rejected revision. It then restores the service template with the previous serving -revision’s resolved image digest and no validation override, leaving traffic on the old revision. -This creates an untagged recovery revision: known-good schema and workers can execute before -it is retired, even with no HTTP traffic. The workflow verifies the template’s runtime settings, -secret references, scaling and normal mode, then deletes the recovery revision under the same -lease. Deletion leaves the safe service template in place for later Terraform reconciliation. -If candidate deletion fails, no recovery revision is created; failed recovery attempts still -record their revision name for retirement and operator diagnosis. +Before any candidate creation, the workflow requires exactly one revision resource, the sole HTTP +serving revision. Existing historical revisions or leftovers from interrupted runs require explicit +operator review and cleanup under the lease first; the workflow does not blindly delete them. +This gate and retirement after every successful rollout prevent repeated runs accumulating workers. +Terraform still owns configuration and scaling; removing validation mode adds no ignored field. -Traffic restoration alone does **not** stop queue consumers. If rollback, template restoration -or deletion fails, operator recovery must complete under the rollout lease; do not call -the rollout recovered merely because the old origin answers. A canceled runner can also require -manual cleanup. Successful rollout removes the active candidate tag. +On failure before public checks pass, any attempted traffic shift is first rolled back and verified. +If partial activation created a successor, recovery retires non-latest validation first; deletion +failure stops recovery before a fourth resource can be created. Recovery then deploys the captured +known-good digest as a tagged, zero-traffic successor with normal mode. It verifies template shape, +secret references and scaling, probes tagged readiness and active mode, promotes the recovery +revision, verifies traffic and public health, and only then deletes rejected and previous revisions. +The latest recovery revision remains serving. Known-good recovery schema and workers can execute +before promotion; neither recovery nor traffic rollback undoes schema changes or sent notifications. -Manual rollback must restore traffic, retire the rejected revision, and restore the service -template under the rollout lease. Use the exact names and known-good image digest from the summary: +Partial creates record deterministic names before mutation. Failed recovery or deletion requires +operator cleanup under the lease; the next automated run refuses leftover resources. A canceled +runner can require the same intervention. Traffic restoration alone does not stop queue consumers. + +Manual recovery must preserve the three-resource bound and keep the successor serving: ```sh -gcloud run services update-traffic orca-cloud-push \ - --project onorca-cloud --region us-central1 --to-revisions =100 -gcloud run services update-traffic orca-cloud-push \ - --project onorca-cloud --region us-central1 --remove-tags -gcloud run revisions delete \ - --project onorca-cloud --region us-central1 -# Verify termination/drain before creating another revision. +# Hold the rollout lease; inspect latest, traffic, tags and existing revisions first. +# If three resources remain after partial activation, retire non-latest inert validation first. +# Restore previous traffic if its revision still exists and a failed candidate took traffic. gcloud run deploy orca-cloud-push \ --project onorca-cloud --region us-central1 --image \ - --remove-env-vars ORCA_PUSH_MODE --no-traffic --revision-suffix -# Verify template image/mode/runtime/secret references/scaling and unchanged traffic, then retire it. -gcloud run revisions delete \ + --remove-env-vars ORCA_PUSH_MODE --no-traffic \ + --tag --revision-suffix +# Verify exact digest, template spec/secret references/scaling, tagged /ready and active /health. +gcloud run services update-traffic orca-cloud-push \ + --project onorca-cloud --region us-central1 --to-revisions =100 +# Verify traffic and public /ready and /health before retiring old consumers. +gcloud run services update-traffic orca-cloud-push \ + --project onorca-cloud --region us-central1 --clear-tags +gcloud run revisions delete \ --project onorca-cloud --region us-central1 +# Repeat only for reviewed obsolete revisions; retain the latest serving recovery revision. ``` Never merely remove validation mode while the template still holds a rejected image. Terraform owns environment configuration but ignores the image, so that would activate rejected code. -Remove a tag only if it remains present. Verify the old revision is serving, the template is safe, -and both rejected/recovery revision deletion and connection drain completed; +Remove a tag only if it remains present. Verify the recovery revision is serving, the template is safe, +and obsolete revision deletion and connection drain completed; already accepted provider sends cannot be undone. Activation-time schema changes must be additive and compatible with the rollback image: rollback does not reverse migrations or queue mutations. The inert phase intentionally cannot validate a new schema by applying it to production. Review @@ -351,9 +360,10 @@ issuance and breaks Cloud Run host routing. Candidate tags and deterministic revision names are recorded before deployment. Promotion intent is recorded before changing traffic, so a failed verification or ambiguous mutation result still triggers -rollback. Failed candidates are deleted only before attempted promotion or after verified rollback. -A known-good template is restored after successful cleanup, and its untagged recovery revision -is retired so it cannot remain an extra consumer. The summary runs even if candidate discovery or traffic verification fails. +rollback. A known-good successor must exist before the rejected latest revision can be deleted. +After verified recovery promotion and public checks, rejected and previous consumers are retired; +the recovery revision remains serving. Failed cleanup blocks subsequent rollout admission. +The summary runs even if candidate discovery or traffic verification fails. Push uses the relay's schema-startup retry implementation through `@orca-cloud/postgres-schema`. Session replacement is serialized per host and a unique host index upgrades older databases by diff --git a/cloud/infra/terraform/push-gateway.tf b/cloud/infra/terraform/push-gateway.tf index 4608d6c0e72..cfb27bce0a6 100644 --- a/cloud/infra/terraform/push-gateway.tf +++ b/cloud/infra/terraform/push-gateway.tf @@ -310,8 +310,8 @@ resource "google_cloud_run_v2_service" "push" { } precondition { - condition = !var.push_dedicated_database_active || var.push_max_instances * var.push_database_pool_max * 2 <= 64 - error_message = "Dedicated push serving and candidate pools must fit the 64-connection rollout budget." + condition = !var.push_dedicated_database_active || var.push_max_instances * var.push_database_pool_max * 3 <= 64 + error_message = "Dedicated push serving, validation/rejected and successor pools must fit the 64-connection rollout budget." } ignore_changes = [