starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-09-29 00:02:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(push): verify successor-first Cloud Run recovery

Browse Source
This commit is contained in:
Jinwoo-H
2026-09-07 21:57:05 -04:00
parent 8fd9947c1a
commit 48fb41b324
10 changed files with 724 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/cloud-push-deploy.yml
+153 -81
View File
@@ -94,7 +94,7 @@ jobs:
run: |
set -euo pipefail
image_tag="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${SOURCE_SHA}"
docker buildx build --push --metadata-file "${RUNNER_TEMP}/push-image.json" \
docker buildx build --push --platform linux/amd64 --provenance=false --metadata-file "${RUNNER_TEMP}/push-image.json" \
-f "${RUNNER_TEMP}/push-source/cloud/apps/push/Dockerfile" \
-t "${image_tag}" "${RUNNER_TEMP}/push-source/cloud"
digest="$(jq -er '."containerimage.digest"' "${RUNNER_TEMP}/push-image.json")"
@@ -137,6 +137,12 @@ jobs:
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test -n "${serving}"
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
if test "${revisions}" != "${serving}"; then
echo 'Retire leftover revisions under the rollout lease before deploying; three pools are the limit.' >&2
exit 1
fi
floor="$(gcloud run revisions describe "${serving}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
@@ -169,9 +175,13 @@ jobs:
run: |
set -euo pipefail
tag="c${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
echo "VALIDATION_DEPLOY_ATTEMPTED=true" >> "${GITHUB_ENV}"
echo "CANDIDATE_TAG=${tag}" >> "${GITHUB_ENV}"
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
{
echo "VALIDATION_DEPLOY_ATTEMPTED=true"
echo "VALIDATION_REVISION=${SERVICE_NAME}-${tag}"
echo "VALIDATION_TAG=${tag}"
echo "CANDIDATE_TAG=${tag}"
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
} >> "${GITHUB_ENV}"
gcloud run deploy "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
@@ -191,8 +201,7 @@ jobs:
# A tagged revision is directly addressable and sits outside the service-wide cap, so the
# candidate and the serving revision each draw up to the ceiling during the probe window.
# The lease is taken for exactly that doubling; a candidate that inherited a wider ceiling
# would exceed it, so the inherited scaling is asserted here too.
# Successor creation later requires three revision pools; assert the inherited ceiling.
- name: Require the candidate to serve the exact image and inherited scaling
shell: bash
run: |
@@ -265,34 +274,39 @@ jobs:
fi
test "${status}" = INVALID_ARGUMENT
# Delete validation before activation so only two revision pools can overlap.
# Cloud Run requires a successor before the latest revision can be deleted.
- name: Retire inert validation and activate the verified image
shell: bash
run: |
set -euo pipefail
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--remove-tags "${CANDIDATE_TAG}" --quiet
gcloud run revisions delete "${CANDIDATE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
echo "CANDIDATE_REVISION=" >> "${GITHUB_ENV}"
# Allow the deleted instance's bounded shutdown to release its pool.
sleep 10
tag="a${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
echo "CANDIDATE_TAG=${tag}" >> "${GITHUB_ENV}"
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
echo "ACTIVATION_ATTEMPTED=true" >> "${GITHUB_ENV}"
{
echo "CANDIDATE_TAG=${tag}"
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
echo "ACTIVATION_ATTEMPTED=true"
} >> "${GITHUB_ENV}"
# This is the production-effect boundary: schema, pruners and workers start here.
gcloud run deploy "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--image "${IMAGE}" --tag "${tag}" --revision-suffix "${tag}" \
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--remove-tags "${VALIDATION_TAG}" --quiet
gcloud run revisions delete "${VALIDATION_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
echo "VALIDATION_RETIRED=true" >> "${GITHUB_ENV}"
revision="$(gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
jq -e --arg image "${IMAGE}" --arg account "${PUSH_RUNTIME_SERVICE_ACCOUNT}" \
--arg ceiling "${PUSH_MAX_INSTANCES}" --arg floor "${PUSH_MIN_INSTANCES}" \
'.spec.containers[0].image == $image and .spec.serviceAccountName == $account and
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" '
def shape: del(.containers[0].image) |
.containers[0].env = ((.containers[0].env // []) |
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
.spec.containers[0].image == $image and .spec.serviceAccountName == $account and
all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
(.spec | shape) == ($prior[0].spec | shape) and
.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
(.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber)' \
<<< "${revision}" > /dev/null
@@ -342,10 +356,11 @@ jobs:
echo "Image: \`${IMAGE_DIGEST}\`"
echo "Known-good image: \`${ROLLBACK_IMAGE}\`"
echo
echo "Rollback: \`gcloud run services update-traffic ${SERVICE_NAME}" \
"--project ${GCP_PROJECT_ID} --region ${GCP_REGION} --to-revisions ${ROLLBACK_REVISION}=100\`"
echo
echo "Then remove tag \`${CANDIDATE_TAG:-none}\` and delete revision \`${CANDIDATE_REVISION}\` to stop workers."
echo "Recovery: deploy \`${ROLLBACK_IMAGE}\` as a new revision with" \
"\`--remove-env-vars ORCA_PUSH_MODE --no-traffic --tag <unique-recovery-tag> --revision-suffix <unique-recovery-suffix>\`."
echo 'Verify its exact digest, configuration, readiness and active mode, then promote and check the public origin.'
echo 'Only then remove obsolete tags and delete rejected/previous revisions; never delete the latest revision.'
echo 'The previous revision is retired after public checks; retain this immutable image for recovery under the rollout lease.'
echo "Activation attempted: ${ACTIVATION_ATTEMPTED:-false}; traffic rollback cannot undo schema or deliveries."
} >> "${GITHUB_STEP_SUMMARY}"
@@ -359,6 +374,7 @@ jobs:
if test "${code}" = 200; then
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' > /dev/null
echo "ROLLOUT_VERIFIED=true" >> "${GITHUB_ENV}"
echo "${PUSH_ORIGIN} is ready after ${attempt} attempt(s)"
exit 0
fi
@@ -372,7 +388,7 @@ jobs:
# is not a failure to deploy, it is a live gateway that has to go back, so the traffic move
# is undone here rather than left to whoever reads the run.
- name: Roll traffic back to the previous revision
if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' }}
if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
shell: bash
run: |
set -euo pipefail
@@ -396,12 +412,97 @@ jobs:
"\`${CANDIDATE_REVISION}\` no longer serves HTTP; deletion below must stop its workers."
} >> "${GITHUB_STEP_SUMMARY}"
# Why: a candidate that never took traffic is a revision holding a warm floor and a Cloud
# SQL pool for nothing. Its tag comes off first, because Cloud Run refuses to delete a
# revision a traffic target still names, and clearing CANDIDATE_TAG makes the always() tag
# step below a no-op rather than a second failure.
# Deleting a revision does not restore the service template that Terraform reconciles.
- name: Restore the known-good service template
if: ${{ (failure() || cancelled()) && env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
shell: bash
run: |
set -euo pipefail
test "${TRAFFIC_SHIFT_ATTEMPTED:-false}" != true || test "${TRAFFIC_ROLLED_BACK:-false}" = true
test -n "${ROLLBACK_IMAGE}"
# A partial activation may leave validation plus active; free one slot before recovery.
latest="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format='value(status.latestCreatedRevisionName)')"
test -n "${latest}"
if test "${VALIDATION_RETIRED:-false}" != true && test "${latest}" != "${VALIDATION_REVISION}"; then
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--filter "metadata.name=${VALIDATION_REVISION}" --format='value(metadata.name)')"
if test -n "${existing}"; then
test "${existing}" = "${VALIDATION_REVISION}"
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--remove-tags "${VALIDATION_TAG}" --quiet
gcloud run revisions delete "${VALIDATION_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
fi
fi
tag="r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
echo "RECOVERY_TAG=${tag}" >> "${GITHUB_ENV}"
echo "TEMPLATE_RECOVERY_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
echo "Template recovery attempted: ${SERVICE_NAME}-${tag}, image ${ROLLBACK_IMAGE}." \
>> "${GITHUB_STEP_SUMMARY}"
# Known-good schema/workers can run here even though HTTP stays on the old revision.
gcloud run deploy "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--image "${ROLLBACK_IMAGE}" --revision-suffix "${tag}" --tag "${tag}" \
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
> "${RUNNER_TEMP}/push-recovered-service.json"
gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
> "${RUNNER_TEMP}/push-recovery-revision.json"
jq -e --arg image "${ROLLBACK_IMAGE}" --arg serving "${ROLLBACK_REVISION}" \
--arg floor "${PUSH_MIN_INSTANCES}" --arg ceiling "${PUSH_MAX_INSTANCES}" \
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" \
--slurpfile recovered "${RUNNER_TEMP}/push-recovery-revision.json" '
def shape: del(.containers[0].image) |
.containers[0].env = ((.containers[0].env // []) |
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
.spec.template.spec.containers[0].image == $image and
all(.spec.template.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
$recovered[0].spec.containers[0].image == $image and
all($recovered[0].spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
($recovered[0].spec | shape) == ($prior[0].spec | shape) and
.spec.template.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
(.spec.template.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber) and
([.status.traffic[] | select((.percent // 0) > 0)] |
length == 1 and .[0].revisionName == $serving and .[0].percent == 100)
' "${RUNNER_TEMP}/push-recovered-service.json" > /dev/null
echo "TEMPLATE_RESTORED=true" >> "${GITHUB_ENV}"
echo 'Known-good image and normal mode restored; previous revision still serves HTTP.' \
>> "${GITHUB_STEP_SUMMARY}"
- name: Promote and verify the known-good recovery revision
if: ${{ always() && env.TEMPLATE_RESTORED == 'true' }}
shell: bash
run: |
set -euo pipefail
url="$(jq -er --arg tag "${RECOVERY_TAG}" \
'.status.traffic[] | select(.tag == $tag) | .url' "${RUNNER_TEMP}/push-recovered-service.json")"
[[ "${url}" =~ ^https://[^/]+$ ]]
curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
curl --fail --silent --show-error --max-time 10 "${url}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--to-revisions "${TEMPLATE_RECOVERY_REVISION}=100" --quiet
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test "${serving}" = "${TEMPLATE_RECOVERY_REVISION}"
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/ready" | jq -e '.ok == true'
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
echo "RECOVERY_VERIFIED=true" >> "${GITHUB_ENV}"
echo "Recovery revision ${TEMPLATE_RECOVERY_REVISION} now serves the known-good image." \
>> "${GITHUB_STEP_SUMMARY}"
- name: Delete the rejected candidate revision
if: ${{ (failure() || cancelled()) && (env.TRAFFIC_SHIFT_ATTEMPTED != 'true' || env.TRAFFIC_ROLLED_BACK == 'true') }}
if: ${{ always() && env.RECOVERY_VERIFIED == 'true' }}
shell: bash
run: |
set -euo pipefail
@@ -428,61 +529,32 @@ jobs:
echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
echo "candidate revision ${CANDIDATE_REVISION} is absent"
# Deleting a revision does not restore the service template that Terraform reconciles.
- name: Restore the known-good service template
if: ${{ (failure() || cancelled()) && env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.CANDIDATE_DELETED == 'true' }}
# Public checks commit the serving revision; cleanup failures must not roll it back.
- name: Retire previous consumers after public checks
if: ${{ always() && (env.ROLLOUT_VERIFIED == 'true' || (env.RECOVERY_VERIFIED == 'true' && env.CANDIDATE_DELETED == 'true')) }}
shell: bash
run: |
set -euo pipefail
test "${TRAFFIC_SHIFT_ATTEMPTED:-false}" != true || test "${TRAFFIC_ROLLED_BACK:-false}" = true
test -n "${ROLLBACK_IMAGE}"
# Shutdown allowance, not a measurement of PostgreSQL connection drain.
sleep 10
tag="r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
echo "TEMPLATE_RECOVERY_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
echo "Template recovery attempted: ${SERVICE_NAME}-${tag}, image ${ROLLBACK_IMAGE}." \
>> "${GITHUB_STEP_SUMMARY}"
# Known-good schema/workers can run here even though HTTP stays on the old revision.
gcloud run deploy "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--image "${ROLLBACK_IMAGE}" --revision-suffix "${tag}" \
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
> "${RUNNER_TEMP}/push-recovered-service.json"
jq -e --arg image "${ROLLBACK_IMAGE}" --arg serving "${ROLLBACK_REVISION}" \
--arg floor "${PUSH_MIN_INSTANCES}" --arg ceiling "${PUSH_MAX_INSTANCES}" \
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" '
def shape: del(.containers[0].image) |
.containers[0].env = ((.containers[0].env // []) |
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
.spec.template.spec.containers[0].image == $image and
all(.spec.template.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
(.spec.template.spec | shape) == ($prior[0].spec | shape) and
.spec.template.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
(.spec.template.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber) and
([.status.traffic[] | select((.percent // 0) > 0)] |
length == 1 and .[0].revisionName == $serving and .[0].percent == 100)
' "${RUNNER_TEMP}/push-recovered-service.json" > /dev/null
echo "TEMPLATE_RESTORED=true" >> "${GITHUB_ENV}"
echo 'Known-good image and normal mode restored; previous revision still serves HTTP.' \
>> "${GITHUB_STEP_SUMMARY}"
- name: Retire the template recovery revision
if: ${{ always() && env.TEMPLATE_RECOVERY_REVISION != '' }}
shell: bash
run: |
set -euo pipefail
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--filter "metadata.name=${TEMPLATE_RECOVERY_REVISION}" --format='value(metadata.name)')"
if test -n "${existing}"; then
test "${existing}" = "${TEMPLATE_RECOVERY_REVISION}"
gcloud run revisions delete "${TEMPLATE_RECOVERY_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
serving="${CANDIDATE_REVISION}"
if test "${RECOVERY_VERIFIED:-false}" = true; then
serving="${TEMPLATE_RECOVERY_REVISION}"
fi
sleep 10
echo 'Template recovery revision retired; SQL connection drain still needs operational verification.' \
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --clear-tags --quiet
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
while IFS= read -r revision; do
test -n "${revision}" || continue
test "${revision}" != "${serving}" || continue
case "${revision}" in
"${ROLLBACK_REVISION}"|"${VALIDATION_REVISION}"|"${CANDIDATE_REVISION}") ;;
*) echo "Unexpected revision ${revision}; manual retirement required." >&2; exit 1 ;;
esac
gcloud run revisions delete "${revision}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
done <<< "${revisions}"
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
echo 'Obsolete revision resources retired; verify actual SQL session drain operationally.' \
>> "${GITHUB_STEP_SUMMARY}"
- name: Drop the candidate traffic tag
cloud/dev/scripts/push-cloud-run-model.mjs
+140
View File
@@ -0,0 +1,140 @@
import { readFileSync, writeFileSync } from 'node:fs'
// Executable fake gcloud: revision deletion obeys the platform's latest/traffic constraints.
const path = process.env.MODEL_STATE
const state = JSON.parse(readFileSync(path, 'utf8'))
const args = process.argv.slice(2)
const option = (name) => args[args.indexOf(name) + 1]
const has = (name) => args.includes(name)
const fail = (message) => {
throw new Error(message)
}
const persist = () => writeFileSync(path, JSON.stringify(state))
const output = (value) => console.log(typeof value === 'string' ? value : JSON.stringify(value))
const revision = (name) => state.revisions[name] ?? fail(`missing revision ${name}`)
const traffic = () => [
{ revisionName: state.serving, percent: 100 },
...Object.entries(state.tags).map(([tag, name]) => ({
tag,
revisionName: name,
url: `https://${tag}.test`
}))
]
state.trace.push(args.join(' '))
try {
if (args[0] === 'curl') {
if (state.failure === 'public' && args.some((arg) => arg.includes('https://public.test'))) {
fail('public check failed')
}
const url = args.find((arg) => arg.startsWith('https://'))
const tag = new URL(url).hostname.split('.')[0]
const name = state.tags[tag] ?? state.serving
if (has('-w')) {
output('200')
} else {
output({
ok: true,
deliveryProtocol: 2,
mode: revision(name).spec.containers[0].env.some((entry) => entry.value === 'validation')
? 'validation'
: 'active'
})
}
} else if (args.slice(0, 2).join(' ') === 'run deploy') {
const name = `${option('deploy')}-${option('--revision-suffix')}`
if (state.failure === 'deploy-before') {
fail('deploy failed before create')
}
const item = structuredClone(revision(state.latest))
item.metadata.name = name
item.spec.containers[0].image = option('--image')
item.status.imageDigest = option('--image')
item.spec.containers[0].env = item.spec.containers[0].env.filter(
(entry) => entry.name !== 'ORCA_PUSH_MODE'
)
if (has('--update-env-vars')) {
item.spec.containers[0].env.push({ name: 'ORCA_PUSH_MODE', value: 'validation' })
}
state.revisions[name] = item
state.latest = name
if (has('--tag')) {
state.tags[option('--tag')] = name
}
state.peak = Math.max(state.peak, Object.keys(state.revisions).length)
if (state.peak > 3) {
fail('three-revision budget exceeded')
}
if (state.failure === 'deploy-after') {
fail('deploy failed after create')
}
} else if (args.slice(0, 3).join(' ') === 'run services describe') {
if (state.failure === 'describe') {
fail('describe failed')
}
if (args.some((arg) => arg.includes('value(status.latestCreatedRevisionName)'))) {
output(state.latest)
} else {
const template = structuredClone(revision(state.latest))
delete template.spec.containers[0].name
output({
spec: { template },
status: { latestCreatedRevisionName: state.latest, traffic: traffic() }
})
}
} else if (args.slice(0, 3).join(' ') === 'run services update-traffic') {
if (has('--to-revisions')) {
const name = option('--to-revisions').split('=')[0]
revision(name)
state.serving = name
}
if (has('--remove-tags')) {
for (const tag of option('--remove-tags').split(',')) {
delete state.tags[tag]
}
}
if (has('--clear-tags')) {
state.tags = {}
}
if (state.failure === 'traffic-after') {
fail('traffic changed but response failed')
}
} else if (args.slice(0, 3).join(' ') === 'run revisions list') {
const names = Object.keys(state.revisions)
output(
(has('--filter')
? names.filter((name) => name === option('--filter').split('=')[1])
: names
).join('\n')
)
} else if (args.slice(0, 3).join(' ') === 'run revisions describe') {
const item = revision(args[3])
const format = args.find((arg) => arg.startsWith('--format=')) ?? option('--format')
if (format.includes('minScale')) {
output(item.metadata.annotations['autoscaling.knative.dev/minScale'])
} else if (format.includes('maxScale')) {
output(item.metadata.annotations['autoscaling.knative.dev/maxScale'])
} else {
output(item)
}
} else if (args.slice(0, 3).join(' ') === 'run revisions delete') {
const name = args[3]
if (name === state.latest) {
fail('FAILED_PRECONDITION: latest created Revision cannot be directly deleted')
}
if (name === state.serving || Object.values(state.tags).includes(name)) {
fail('revision has traffic or tags')
}
if (state.failure === 'delete') {
fail('delete failed')
}
revision(name)
delete state.revisions[name]
} else {
fail(`unmodeled gcloud call ${args.join(' ')}`)
}
} catch (error) {
console.error(error.message)
process.exitCode = 1
} finally {
persist()
}
cloud/dev/scripts/push-gateway-recovery.test.mjs
+341 -251
View File
@@ -1,7 +1,8 @@
import assert from 'node:assert/strict'
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { spawnSync } from 'node:child_process'
import test from 'node:test'
import { readRelayWorkflow } from './relay-repository.mjs'
@@ -12,272 +13,361 @@ function step(name) {
assert.notEqual(start, -1)
const end = workflow.indexOf('\n - ', start + 1)
const block = workflow.slice(start, end === -1 ? undefined : end)
return block.slice(block.indexOf(' run: |\n') + ' run: |\n'.length)
.split('\n').filter((line) => line.startsWith(' ')).map((line) => line.slice(10)).join('\n')
return block
.slice(block.indexOf(' run: |\n') + ' run: |\n'.length)
.split('\n')
.filter((line) => line.startsWith(' '))
.map((line) => line.slice(10))
.join('\n')
}
const candidate = step('Deploy the candidate revision with no traffic')
const shift = step('Shift all traffic to the verified candidate')
const rollback = step('Roll traffic back to the previous revision')
const cleanup = step('Delete the rejected candidate revision')
const env = { SERVICE_NAME: 'push-test', GCP_PROJECT_ID: 'test', GCP_REGION: 'test',
GITHUB_RUN_ID: '123', GITHUB_RUN_ATTEMPT: '1', IMAGE: 'synthetic-image',
CANDIDATE_REVISION: 'push-test-c123-1', ROLLBACK_REVISION: 'push-test-old',
ROLLBACK_IMAGE: 'registry/push@sha256:' + 'a'.repeat(64), PUSH_MIN_INSTANCES: '1', PUSH_MAX_INSTANCES: '2' }
function exercise(body, setup = () => {}) {
const names = {
preflight: 'Record the serving revision and require its Terraform-owned scaling',
candidate: 'Deploy the candidate revision with no traffic',
activate: 'Retire inert validation and activate the verified image',
shift: 'Shift all traffic to the verified candidate',
public: 'Verify the public origin after the shift',
rollback: 'Roll traffic back to the previous revision',
restore: 'Restore the known-good service template',
promoteRecovery: 'Promote and verify the known-good recovery revision',
cleanup: 'Delete the rejected candidate revision',
retire: 'Retire previous consumers after public checks'
}
const image = `registry/push@sha256:${'a'.repeat(64)}`
const spec = {
serviceAccountName: 'runtime@test',
containerConcurrency: 40,
containers: [
{
image,
name: 'push-test-1',
env: [
{
name: 'ORCA_PUSH_DATABASE_URL',
valueFrom: { secretKeyRef: { name: 'database', key: '7' } }
},
{ name: 'ORCA_PUSH_DATABASE_POOL_MAX', value: '2' }
]
}
]
}
const prior = {
metadata: {
name: 'push-test-old',
annotations: {
'autoscaling.knative.dev/minScale': '1',
'autoscaling.knative.dev/maxScale': '2'
}
},
spec,
status: { imageDigest: image }
}
const model = fileURLToPath(new URL('./push-cloud-run-model.mjs', import.meta.url))
const options = { skip: process.platform === 'win32' }
function exercise(callback) {
const dir = mkdtempSync(join(tmpdir(), 'push-workflow-'))
const statePath = join(dir, 'state.json')
writeFileSync(
statePath,
JSON.stringify({
revisions: { 'push-test-old': prior },
latest: 'push-test-old',
serving: 'push-test-old',
tags: {},
peak: 1,
trace: []
})
)
writeFileSync(join(dir, 'env'), '')
const env = {
...process.env,
SERVICE_NAME: 'push-test',
GCP_PROJECT_ID: 'test',
GCP_REGION: 'test',
GITHUB_RUN_ID: '123',
GITHUB_RUN_ATTEMPT: '1',
IMAGE: `registry/push@sha256:${'b'.repeat(64)}`,
PUSH_MIN_INSTANCES: '1',
PUSH_MAX_INSTANCES: '2',
PUSH_RUNTIME_SERVICE_ACCOUNT: 'runtime@test',
PUSH_ORIGIN: 'https://public.test',
MODEL_STATE: statePath,
MODEL_SCRIPT: model,
RUNNER_TEMP: dir,
GITHUB_ENV: join(dir, 'env'),
GITHUB_STEP_SUMMARY: join(dir, 'summary')
}
const state = () => JSON.parse(readFileSync(statePath, 'utf8'))
const change = (edit) => {
const value = state()
edit(value)
writeFileSync(statePath, JSON.stringify(value))
}
const run = (key, ok = true, extra = '') => {
const result = spawnSync(
'bash',
[
'-c',
`
set -a
source "$GITHUB_ENV"
gcloud() { node "$MODEL_SCRIPT" "$@"; }
curl() { node "$MODEL_SCRIPT" curl "$@"; }
sleep() { :; }
${extra}
${names[key] ? step(names[key]) : key}
`
],
{ cwd: dir, env, encoding: 'utf8', timeout: 30000 }
)
assert.equal(result.status === 0, ok, `${key}: ${result.stderr}\n${result.stdout}`)
return result
}
try {
setup(dir)
const run = spawnSync('bash', ['-c', body], { encoding: 'utf8', timeout: 10000, cwd: dir,
env: { ...process.env, ...env, RUNNER_TEMP: dir, GITHUB_ENV: join(dir, 'env'), GITHUB_STEP_SUMMARY: join(dir, 'summary'),
TRACE: join(dir, 'trace'), STATE: join(dir, 'state') } })
assert.equal(run.status, 0, run.stderr)
} finally { rmSync(dir, { recursive: true, force: true }) }
callback({ run, state, change, dir })
} finally {
rmSync(dir, { recursive: true, force: true })
}
}
function recover(h) {
h.change((state) => {
delete state.failure
})
h.run('restore')
h.run('promoteRecovery')
h.run('cleanup')
h.run('retire')
const state = h.state()
assert.equal(state.serving, 'push-test-r123-1')
assert.equal(state.latest, state.serving)
assert.deepEqual(Object.keys(state.revisions), [state.serving])
assert.equal(state.revisions[state.serving].spec.containers[0].image, image)
assert.ok(state.peak <= 3)
}
// Workflow shell behavior is Linux-specific; these tests never call a real cloud CLI.
test('failed candidate discovery retains enough state to remove tag and revision', { skip: process.platform === 'win32' }, () => {
exercise(`
gcloud() {
case "$*" in
'run deploy '*) echo deployed > "$STATE" ;;
'run services describe '*) return 1 ;;
'run revisions list '*) echo "$CANDIDATE_REVISION" ;;
*) echo "$*" >> "$TRACE" ;;
esac
}
jq() { return 1; }
( ${candidate} )
test "$?" != 0 || exit 1
source "$GITHUB_ENV"
test "$CANDIDATE_TAG" = c123-1 || exit 1
test "$CANDIDATE_REVISION" = push-test-c123-1 || exit 1
( ${cleanup} ) || exit 1
grep -q -- '--remove-tags c123-1' "$TRACE" || exit 1
grep -q 'run revisions delete push-test-c123-1' "$TRACE" || exit 1
`)
})
test(
'the executable Cloud Run model rejects deleting latest even without tags or traffic',
options,
() =>
exercise((h) => {
h.run('preflight')
h.run('candidate')
h.run('gcloud run services update-traffic "$SERVICE_NAME" --clear-tags')
const result = h.run('gcloud run revisions delete "$CANDIDATE_REVISION"', false)
assert.match(result.stderr, /FAILED_PRECONDITION: latest created Revision/)
})
)
test('failed post-promotion read retains intent and restores previous traffic', { skip: process.platform === 'win32' }, () => {
exercise(`
gcloud() {
case "$*" in
'run services update-traffic '*) echo "$*" >> "$TRACE" ;;
'run services describe '*) return 1 ;;
esac
}
jq() { return 1; }
( ${shift} )
test "$?" != 0 || exit 1
source "$GITHUB_ENV"
test "$TRAFFIC_SHIFT_ATTEMPTED" = true || exit 1
gcloud() {
case "$*" in
'run services update-traffic '*) echo "$*" >> "$TRACE" ;;
'run services describe '*) echo '{}' ;;
esac
}
jq() { echo "$ROLLBACK_REVISION"; }
( ${rollback} ) || exit 1
source "$GITHUB_ENV"
test "$TRAFFIC_ROLLED_BACK" = true || exit 1
grep -q -- '--to-revisions push-test-old=100' "$TRACE" || exit 1
`)
})
test(
'success creates successor before retirement and repeated rollouts retain one consumer',
options,
() =>
exercise((h) => {
for (const attempt of ['1', '2']) {
if (attempt === '2') {
writeFileSync(join(h.dir, 'env'), 'GITHUB_RUN_ATTEMPT=2\n')
}
for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public', 'retire']) {
h.run(key)
}
const state = h.state()
assert.equal(state.serving, `push-test-a123-${attempt}`)
assert.deepEqual(Object.keys(state.revisions), [state.serving])
assert.equal(state.peak, 3)
}
})
)
test('ambiguous promotion failure also leaves rollback intent', { skip: process.platform === 'win32' }, () => {
exercise(`
gcloud() { return 1; }
( ${shift} )
test "$?" != 0 || exit 1
source "$GITHUB_ENV"
test "$TRAFFIC_SHIFT_ATTEMPTED" = true
`)
})
for (const failure of ['deploy-before', 'deploy-after', 'describe']) {
test(`validation ${failure} recovers without deleting latest`, options, () =>
exercise((h) => {
h.run('preflight')
h.change((state) => {
state.failure = failure
})
h.run('candidate', false)
recover(h)
})
)
}
for (const failure of ['deploy-before', 'deploy-after', 'delete', 'describe']) {
test(
`activation ${failure} frees validation slot before recovery and stays within three`,
options,
() =>
exercise((h) => {
h.run('preflight')
h.run('candidate')
h.change((state) => {
state.failure = failure
})
h.run('activate', false)
recover(h)
})
)
}
const activate = step('Retire inert validation and activate the verified image')
test('partial activation records the new revision before deploy and deletes its consumer', { skip: process.platform === 'win32' }, () => {
exercise(`
CANDIDATE_TAG=c123-1
sleep() { :; }
gcloud() {
echo "$*" >> "$TRACE"
case "$*" in
'run deploy '*) return 1 ;;
'run revisions list '*) echo "$CANDIDATE_REVISION" ;;
esac
}
( ${activate} )
test "$?" != 0 || exit 1
source "$GITHUB_ENV"
test "$ACTIVATION_ATTEMPTED" = true || exit 1
test "$CANDIDATE_REVISION" = push-test-a123-1 || exit 1
test "$CANDIDATE_TAG" = a123-1 || exit 1
( ${cleanup} ) || exit 1
grep -q 'run revisions delete push-test-c123-1' "$TRACE" || exit 1
grep -q 'run revisions delete push-test-a123-1' "$TRACE" || exit 1
grep -q -- '--remove-env-vars ORCA_PUSH_MODE' "$TRACE" || exit 1
test "$(grep -n 'run revisions delete push-test-c123-1' "$TRACE" | cut -d: -f1)" -lt \
"$(grep -n 'run deploy' "$TRACE" | cut -d: -f1)" || exit 1
`)
})
test(
'ambiguous traffic shift records intent before mutation, rolls back and recovers',
options,
() =>
exercise((h) => {
h.run('preflight')
h.run('candidate')
h.run('activate')
h.change((state) => {
state.failure = 'traffic-after'
})
h.run('shift', false)
assert.match(readFileSync(join(h.dir, 'env'), 'utf8'), /TRAFFIC_SHIFT_ATTEMPTED=true/)
h.change((state) => {
delete state.failure
})
h.run('rollback')
recover(h)
})
)
test('failed validation retirement never activates another consumer', { skip: process.platform === 'win32' }, () => {
exercise(`
CANDIDATE_TAG=c123-1
gcloud() {
echo "$*" >> "$TRACE"
case "$*" in
'run revisions delete '*) return 1 ;;
esac
test('failed public check rolls back and recovers', options, () =>
exercise((h) => {
for (const key of ['preflight', 'candidate', 'activate', 'shift']) {
h.run(key)
}
( ${activate} )
test "$?" != 0 || exit 1
! grep -q 'run deploy' "$TRACE" || exit 1
! grep -q ACTIVATION_ATTEMPTED "$GITHUB_ENV" 2>/dev/null || exit 1
`)
})
h.change((state) => {
state.failure = 'public'
})
h.run('public', false)
h.change((state) => {
delete state.failure
})
h.run('rollback')
recover(h)
})
)
for (const defect of [
'runtime',
'secret',
'mode',
'image',
'traffic',
'scaling',
'deploy-before',
'deploy-after'
]) {
test(`recovery rejects ${defect} and preserves partial-create state`, options, () =>
exercise((h) => {
h.run('preflight')
h.run('candidate')
h.change((state) => {
const revision = state.revisions[state.latest]
if (defect === 'runtime') {
revision.spec.serviceAccountName = 'wrong@test'
}
if (defect === 'secret') {
revision.spec.containers[0].env[0].valueFrom.secretKeyRef.key = '8'
}
if (defect === 'scaling') {
revision.metadata.annotations['autoscaling.knative.dev/maxScale'] = '3'
}
if (defect === 'traffic') {
state.serving = state.latest
}
if (defect.startsWith('deploy-')) {
state.failure = defect
}
})
// Corrupt the recovery response after the modeled deploy while keeping real jq assertions.
const extra = ['mode', 'image'].includes(defect)
? `
gcloud() {
node "$MODEL_SCRIPT" "$@" > "$RUNNER_TEMP/out" || return $?
if [[ "$*" == 'run services describe '* && "$*" == *'--format=json'* ]]; then
jq '${defect === 'mode' ? '.spec.template.spec.containers[0].env += [{name:"ORCA_PUSH_MODE",value:"validation"}]' : '.spec.template.spec.containers[0].image = "wrong"'}' "$RUNNER_TEMP/out"
else cat "$RUNNER_TEMP/out"; fi
}`
: ''
h.run('restore', false, extra)
const recorded = readFileSync(join(h.dir, 'env'), 'utf8')
assert.match(recorded, /TEMPLATE_RECOVERY_REVISION=push-test-r123-1/)
assert.doesNotMatch(recorded, /TEMPLATE_RESTORED=true/)
assert.ok(h.state().peak <= 3)
})
)
}
test('failed validation retirement blocks a fourth revision during recovery', options, () =>
exercise((h) => {
h.run('preflight')
h.run('candidate')
h.change((state) => {
state.failure = 'delete'
})
h.run('activate', false)
h.run('restore', false)
assert.equal(h.state().peak, 3)
assert.equal(h.state().revisions['push-test-r123-1'], undefined)
})
)
test(
'failed retirement after public checks leaves verified serving and blocks the next run',
options,
() =>
exercise((h) => {
for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public']) {
h.run(key)
}
h.change((state) => {
state.failure = 'delete'
})
h.run('retire', false)
assert.equal(h.state().serving, 'push-test-a123-1')
h.run('preflight', false)
assert.match(workflow, /env.ROLLOUT_VERIFIED != 'true'/)
})
)
test(
'recovery promotion failure keeps consumers for operator diagnosis and blocks new rollout',
options,
() =>
exercise((h) => {
h.run('preflight')
h.run('candidate')
h.run('restore')
h.change((state) => {
state.failure = 'public'
})
h.run('promoteRecovery', false)
assert.doesNotMatch(readFileSync(join(h.dir, 'env'), 'utf8'), /RECOVERY_VERIFIED=true/)
h.run('preflight', false)
})
)
const capability = step('Require image support for inert validation')
for (const [label, source, expected] of [
['old image', 'export function loadPushConfig() { return {}; }', 1],
['invalid mode accepted', 'export function loadPushConfig(env) { return { mode: env.ORCA_PUSH_MODE }; }', 1],
['validation supported', `export function loadPushConfig(env) {
for (const [label, source, ok] of [
['old image', 'export function loadPushConfig() { return {}; }', false],
[
'invalid mode accepted',
'export function loadPushConfig(env) { return { mode: env.ORCA_PUSH_MODE }; }',
false
],
[
'validation supported',
`export function loadPushConfig(env) {
if (env.ORCA_PUSH_MODE !== 'validation') throw new Error('invalid mode');
return { mode: 'validation' };
}`, 0]
}`,
true
]
]) {
test(`pre-production image smoke: ${label}`, { skip: process.platform === 'win32' }, () => {
exercise(`
docker() { node "\${@: -3}"; }
( ${capability} )
test "$?" = ${expected}
`, (dir) => {
const dist = join(dir, 'apps', 'push', 'dist')
test(`pre-production image smoke: ${label}`, options, () =>
exercise((h) => {
const dist = join(h.dir, 'apps', 'push', 'dist')
mkdirSync(dist, { recursive: true })
writeFileSync(join(dir, 'package.json'), '{"type":"module"}')
writeFileSync(join(h.dir, 'package.json'), '{"type":"module"}')
writeFileSync(join(dist, 'config.js'), source)
h.run(capability, ok, 'docker() { node "${@: -3}"; }')
})
})
)
}
const restore = step('Restore the known-good service template')
const priorSpec = { serviceAccountName: 'runtime@test', containerConcurrency: 40,
containers: [{ image: env.ROLLBACK_IMAGE, env: [
{ name: 'ORCA_PUSH_DATABASE_URL', valueFrom: { secretKeyRef: { name: 'database', key: '7' } } },
{ name: 'ORCA_PUSH_DATABASE_POOL_MAX', value: '2' }
] }] }
const recoveredService = { spec: { template: { spec: priorSpec, metadata: { annotations: {
'autoscaling.knative.dev/minScale': '1', 'autoscaling.knative.dev/maxScale': '2'
} } } }, status: { traffic: [{ revisionName: env.ROLLBACK_REVISION, percent: 100 }] } }
function recoveryFiles(dir, service = recoveredService) {
writeFileSync(join(dir, 'push-rollback-revision.json'), JSON.stringify({ spec: priorSpec }))
writeFileSync(join(dir, 'recovered.json'), JSON.stringify(service))
}
test('recovery requires cleanup success and restores no traffic before verified rollback', { skip: process.platform === 'win32' }, () => {
const block = workflow.slice(workflow.indexOf('- name: Restore the known-good service template'))
assert.match(block, /env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.CANDIDATE_DELETED == 'true'/)
assert.ok(workflow.indexOf('- name: Delete the rejected candidate revision') <
workflow.indexOf('- name: Restore the known-good service template'))
exercise(`
TRAFFIC_SHIFT_ATTEMPTED=true
gcloud() { echo unexpected >> "$TRACE"; }
( ${restore} )
test "$?" != 0 || exit 1
test ! -e "$TRACE"
`)
})
for (const absent of [false, true]) {
test(`failed validation restores known-good template after candidate ${absent ? 'was never created' : 'deletion'}`, { skip: process.platform === 'win32' }, () => {
exercise(`
CANDIDATE_TAG=c123-1
sleep() { echo shutdown-allowance >> "$TRACE"; }
gcloud() {
echo "$*" >> "$TRACE"
case "$*" in
'run revisions list '*) ${absent ? ':' : 'echo "$CANDIDATE_REVISION"'} ;;
'run services describe '*) cat "$RUNNER_TEMP/recovered.json" ;;
esac
}
( ${cleanup} ) || exit 1
source "$GITHUB_ENV"
test "$CANDIDATE_DELETED" = true || exit 1
( ${restore} ) || exit 1
source "$GITHUB_ENV"
test "$TEMPLATE_RESTORED" = true || exit 1
grep -q -- '--image registry/push@sha256:' "$TRACE" || exit 1
grep -q -- '--remove-env-vars ORCA_PUSH_MODE --no-traffic' "$TRACE" || exit 1
test "$(grep -n shutdown-allowance "$TRACE" | cut -d: -f1)" -lt \
"$(grep -n 'run deploy' "$TRACE" | cut -d: -f1)"
`, recoveryFiles)
})
}
test('failed candidate deletion does not authorize template recovery', { skip: process.platform === 'win32' }, () => {
exercise(`
gcloud() {
case "$*" in
'run revisions list '*) echo "$CANDIDATE_REVISION" ;;
'run revisions delete '*) return 1 ;;
esac
}
( ${cleanup} )
test "$?" != 0 || exit 1
! grep -q CANDIDATE_DELETED=true "$GITHUB_ENV" 2>/dev/null
`)
})
for (const defect of ['runtime', 'secret', 'mode', 'image', 'traffic', 'scaling', 'deploy']) {
test(`template recovery rejects ${defect} failure and records attempted revision`, { skip: process.platform === 'win32' }, () => {
const service = structuredClone(recoveredService)
if (defect === 'runtime') service.spec.template.spec.serviceAccountName = 'wrong@test'
if (defect === 'secret') service.spec.template.spec.containers[0].env[0].valueFrom.secretKeyRef.key = '8'
if (defect === 'mode') service.spec.template.spec.containers[0].env.push({ name: 'ORCA_PUSH_MODE', value: 'validation' })
if (defect === 'image') service.spec.template.spec.containers[0].image = 'rejected'
if (defect === 'traffic') service.status.traffic[0].revisionName = 'rejected'
if (defect === 'scaling') service.spec.template.metadata.annotations['autoscaling.knative.dev/maxScale'] = '3'
exercise(`
sleep() { :; }
gcloud() {
case "$*" in
'run deploy '*) ${defect === 'deploy' ? 'return 1' : ':'} ;;
'run services describe '*) cat "$RUNNER_TEMP/recovered.json" ;;
esac
}
( ${restore} )
test "$?" != 0 || exit 1
source "$GITHUB_ENV"
test "$TEMPLATE_RECOVERY_REVISION" = push-test-r123-1 || exit 1
test -z "\${TEMPLATE_RESTORED:-}"
`, (dir) => recoveryFiles(dir, service))
})
}
const retireRecovery = step('Retire the template recovery revision')
for (const absent of [false, true]) {
test(`recovery retirement handles ${absent ? 'partial creation without a revision' : 'an existing recovery consumer'}`, { skip: process.platform === 'win32' }, () => {
exercise(`
TEMPLATE_RECOVERY_REVISION=push-test-r123-1
sleep() { echo shutdown-allowance >> "$TRACE"; }
gcloud() {
echo "$*" >> "$TRACE"
case "$*" in
'run revisions list '*) ${absent ? ':' : 'echo "$TEMPLATE_RECOVERY_REVISION"'} ;;
esac
}
( ${retireRecovery} ) || exit 1
${absent ? '!' : ''} grep -q 'run revisions delete' "$TRACE" || exit 1
grep -q shutdown-allowance "$TRACE"
`)
})
}
test('recovery retirement runs after success or failure without mutating the restored template', () => {
const block = workflow.slice(workflow.indexOf('- name: Retire the template recovery revision'),
workflow.indexOf('- name: Drop the candidate traffic tag'))
assert.match(block, /always\(\) && env.TEMPLATE_RECOVERY_REVISION != ''/)
assert.doesNotMatch(retireRecovery, /run (deploy|services update|services replace)/)
assert.match(workflow, /image="\$\(jq -er '\.status.imageDigest'/)
})
cloud/dev/scripts/push-gateway-workflow.test.mjs
+10 -5
View File
@@ -240,7 +240,7 @@ test('the summary is written before anything that can fail after the shift', ()
const summary = indexOfStep('Publish the rollout summary')
assert.ok(summary > indexOfStep('Shift all traffic to the verified candidate'))
assert.ok(summary < indexOfStep('Verify the public origin after the shift'))
assert.match(workflow, /--to-revisions \$\{ROLLBACK_REVISION\}=100/)
assert.match(workflow, /Known-good image:/)
assert.match(workflow, /GITHUB_STEP_SUMMARY/)
})
@@ -262,7 +262,7 @@ test('a failure after the shift rolls production back automatically', () => {
)
assert.match(
body,
/if: \$\{\{ \(failure\(\) \|\| cancelled\(\)\) && env\.TRAFFIC_SHIFT_ATTEMPTED == 'true' \}\}/,
/if: \$\{\{ \(failure\(\) \|\| cancelled\(\)\) && env\.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env\.ROLLOUT_VERIFIED != 'true' \}\}/,
'the rollback must be conditioned on both failure and the shift marker'
)
assert.match(body, /test -n "\$\{ROLLBACK_REVISION:-\}"/)
@@ -273,15 +273,15 @@ test('a failure after the shift rolls production back automatically', () => {
// Why: a candidate that never took traffic still holds a warm instance and a Cloud SQL pool. Its
// tag comes off first, because Cloud Run refuses to delete a revision a traffic target names.
test('a failure before the shift deletes the candidate it created', () => {
test('verified recovery authorizes rejected candidate deletion', () => {
const body = workflow.slice(
workflow.indexOf('- name: Delete the rejected candidate revision'),
workflow.indexOf('- name: Drop the candidate traffic tag')
)
assert.match(
body,
/env\.TRAFFIC_SHIFT_ATTEMPTED != 'true' \|\| env\.TRAFFIC_ROLLED_BACK == 'true'/,
'the cleanup must be conditioned on both failure and the absence of the shift marker'
/env\.RECOVERY_VERIFIED == 'true'/,
'cleanup must wait for verified recovery traffic and public checks'
)
assert.match(body, /if test -z "\$\{CANDIDATE_REVISION:-\}"; then/)
assert.ok(
@@ -308,3 +308,8 @@ test('push credentials cannot assume the shared Relay deploy identity', () => {
assert.doesNotMatch(workflow, /PRODUCTION_GCP_RELAY_DEPLOY_/)
assert.doesNotMatch(terraform('push-gateway.tf'), /member\s*=\s*local\.relay_github_deploy_service_account_member/)
})
// A latest revision needs a successor even when validation is inert.
test('dedicated database admits three simultaneous revision pools', () => {
assert.match(terraform('push-gateway.tf'), /var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/)
})
cloud/dev/scripts/push-validation-workflow.test.mjs
+2 -2
View File
@@ -15,7 +15,7 @@ const activation = position('Retire inert validation and activate the verified i
const shift = position('Shift all traffic to the verified candidate')
test('the exact build digest must support validation before production boot', () => {
assert.match(workflow, /docker buildx build --push --metadata-file/)
assert.match(workflow, /docker buildx build --push --platform linux\/amd64 --provenance=false --metadata-file/)
assert.match(workflow, /containerimage\.digest/)
assert.doesNotMatch(workflow, /gcloud artifacts docker images describe/)
assert.ok(capability < deploy)
@@ -30,7 +30,7 @@ test('inert validation and credential checks precede deliberate activation of th
assert.match(workflow.slice(deploy, activation), /\.mode == "validation"/)
assert.ok(position('Prove the runtime identity can reach FCM') < activation)
const active = workflow.slice(activation, shift)
assert.ok(active.indexOf('gcloud run revisions delete') < active.indexOf('gcloud run deploy'))
assert.ok(active.indexOf('gcloud run deploy') < active.indexOf('gcloud run revisions delete'))
assert.match(active, /--image "\$\{IMAGE\}"/)
assert.match(active, /--remove-env-vars ORCA_PUSH_MODE/)
assert.match(active, /\.spec\.containers\[0\]\.image == \$image/)
cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs
+2 -2
View File
@@ -73,8 +73,8 @@ export function calculateRelayCloudSqlConnectionBudget(inputs) {
relayDirectorCandidate: retainedDirectorRollback * 2,
apiCandidate: retainedDirectorRollback + inputs.apiInstances * inputs.apiPoolMax,
authCandidate: retainedDirectorRollback + inputs.authInstances * inputs.authPoolMax,
// Serving push pools are already in configuredMaximum; the tagged candidate adds one copy.
pushCandidate: retainedDirectorRollback + pushDraw,
// Serving is already counted; validation/rejected and its successor add two pools.
pushCandidate: retainedDirectorRollback + pushDraw * 2,
relayCells: retainedDirectorRollback
}
const rolloutOverlap = Math.max(...Object.values(candidateOverlap))
cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs
+6 -6
View File
@@ -22,7 +22,7 @@ test('production plus the push gateway keeps allowance and reserve below the cei
assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30)
assert.equal(report.rolloutOverlap.apiCandidate, 65)
assert.equal(report.rolloutOverlap.authCandidate, 35)
assert.equal(report.rolloutOverlap.pushCandidate, 19)
assert.equal(report.rolloutOverlap.pushCandidate, 23)
assert.equal(report.rolloutOverlap.relayCells, 15)
assert.equal(report.rolloutOverlap.retainedDirectorRollback, 15)
// The gateway does not set the maximum; the API candidate does, as it did before it existed.
@@ -65,10 +65,10 @@ test('the same relay shape without the gateway stays inside the ceiling', () =>
assert.equal(report.withinBudget, true)
})
// Why: a tagged candidate is directly addressable and sits outside the service-wide cap, so both
// Why: a tagged candidate is directly addressable and sits outside the service-wide cap, so all three
// push revisions can reach the ceiling at once. The API and auth candidates add one copy; this
// one adds two, like the director candidate.
test('the push rollout scenario doubles the gateway draw over the retained director', () => {
test('the push rollout scenario triples the gateway draw over the retained director', () => {
const report = calculateRelayCloudSqlConnectionBudget({
cellPoolTotal: 0,
asiaCellCount: 0,
@@ -87,8 +87,8 @@ test('the push rollout scenario doubles the gateway draw over the retained direc
})
assert.equal(report.consumers.push, 4)
// 15 retained director rollback, plus the 4-connection draw counted twice.
assert.equal(report.rolloutOverlap.pushCandidate, 19)
// Serving is in the base; overlap adds 15 retained director plus two 4-connection pools.
assert.equal(report.rolloutOverlap.pushCandidate, 23)
})
test('fails closed when pool growth consumes the explicit reserve', () => {
@@ -176,7 +176,7 @@ test('a tfvars push_max_instances override wins over the variable default', () =
})
assert.equal(report.consumers.push, 6)
assert.equal(report.rolloutOverlap.pushCandidate, 9)
assert.equal(report.rolloutOverlap.pushCandidate, 15)
})
test('requires strict headroom below the physical ceiling', () => {
cloud/docs/push-database-cutover.md
+13 -8
View File
@@ -45,7 +45,7 @@ tier, backup policy, and regional availability. No Cloud Run service changes in
Do not apply a plan that would shift traffic or revert runtime configuration.
4. Dispatch `cloud-push-deploy.yml` from main with the exact reviewed source SHA. It creates
an inert, read-only candidate inheriting the dedicated attachment, probes readiness and provider
access, then deletes it and deliberately activates the same digest under the same lease.
access, then deliberately creates an active successor of the same digest before deleting validation.
Activation starts schema writes and workers before HTTP promotion. Verify the candidate's SQL
attachment and pinned secret reference as well as its image and health.
5. Register a test phone against the deployed origin and prove real APNs delivery. Check
@@ -53,11 +53,16 @@ tier, backup policy, and regional availability. No Cloud Run service changes in
connections have drained. Leave the old database intact; do not delete shared resources.
If activation fails before promotion, the existing HTTP serving revision is unchanged, but
activated workers may already have sent notifications or mutated the queue. Delete the rejected
revision to stop those workers; traffic rollback alone does not stop consumers. After cleanup,
the workflow restores the known-good image and normal mode in the service template, verifies
runtime settings and secret references, and retires the untagged recovery revision. Recovery
can run known-good schema/workers; it does not undo earlier queue or schema changes.
activated workers may already have sent notifications or mutated the queue. Cloud Run will not
delete the latest created revision, even untagged at zero traffic. Recovery creates a known-good
successor first, verifies its template/runtime/secret shape and health, promotes and verifies it,
then deletes rejected and previous consumers. The recovery successor remains serving; it can run
known-good schema/workers before promotion and does not undo earlier queue or schema changes.
A partial activation leaving three resources must retire non-latest inert validation before
recovery creates another; failed retirement stops automation. Every deploy requires a single
serving revision resource at admission, so review and retire historical/leftover revisions under
the lease before dispatch. A Terraform attachment update can itself create such a revision:
verify/promote that known-good image and attachment and retire the former revision before dispatch.
The deploy workflow can roll traffic back on failure; in this internal reset rollout,
that may discard registrations created during the probe window. After successful activation,
application rollback should retain the dedicated attachment and deploy an older compatible
@@ -67,8 +72,8 @@ not a lossless rollback. Future public migrations require a separately rehearsed
## Capacity and resizing
The initial gateway keeps its existing two-connection pool and two-instance maximum.
Dedicated database rollout pools are capped at 64 total connections across serving and
candidate revisions, leaving room for maintenance and operators; this is an admission
Dedicated database rollout pools are capped at 64 total configured pool connections across three simultaneous
revision resources (serving, validation/rejected, and active/recovery successor), leaving room for maintenance and operators; this is an admission
budget, not a throughput claim. Increase the pool only after measuring deployed contention.
Keep the shared database allocation reserved until source connections have drained.
cloud/docs/push-gateway.md
+55 -45
View File
@@ -36,7 +36,7 @@ coalescing window lives in instance memory, so the floor is what keeps a notific
ceiling is a different question, answered below.
The maximum and the pool are set by the connection budget, not by the gateway's own appetite. Two
instances times a two-connection pool is a draw of 4, and a rollout doubles it to 8, because the
instances times a two-connection pool is a draw of 4, and a rollout triples it to 12, because the
tagged candidate is directly addressable and sits outside the service-wide cap. The shared Cloud
SQL instance's 400 connections were already spoken for by the relay cells, the directors, auth,
and the API, which left five. Four is the whole of the room there was, and the gateway fits in
@@ -181,61 +181,70 @@ asserts Terraform-owned scaling. It deploys a tagged, zero-traffic validation re
prove schema compatibility, provider delivery, or active-worker readiness. Container probes
can still use `/health` without treating an inert process as unhealthy.
The build explicitly targets `linux/amd64` with provenance disabled so build metadata records a
single manifest digest, rather than an OCI index that Cloud Run resolves to a different digest.
The workflow verifies the exact image and scaling, probes readiness and mode, and checks the
runtime identity with a validate-only FCM request. It then removes the tag and deletes validation,
allowing ten seconds for shutdown before starting another revision. This orders the rollout
within the two-revision connection budget; the delay is not proof of SQL connection drain.
Verify revision termination and SQL sessions during controlled rollout acceptance.
runtime identity with a validate-only FCM request. Cloud Run rejects deletion of the latest
created revision even when it has no tag or traffic. Activation therefore creates a successor
before removing the validation tag and deleting validation. The dedicated 64-connection budget
and legacy shared budget reserve three simultaneous revision pools: serving, validation/rejected,
and active/recovery successor (12 configured pool connections at the current two-by-two shape).
Revision deletion is not proof of physical SQL session drain; verify termination and SQL sessions
in controlled rollout acceptance. There is no shutdown sleep used as a drain gate.
**The next step deliberately activates production effects.** It deploys a distinct revision of
the exact validated digest, removing the validation override so the default `active` mode applies.
Schema setup runs on its existing one-connection untimed pool, followed by workers and pruners.
These can mutate production and send notifications **before HTTP traffic moves**. The workflow
checks the active revision's digest, runtime identity, scaling, readiness and mode, then moves all
HTTP traffic and checks the public origin. The summary records activation intent and rollback.
Terraform continues to own configuration and scaling; the temporary validation environment entry
is removed on activation, so no new ignored Terraform field is needed.
**Activation deliberately starts production effects.** The distinct active revision uses the exact
validated digest with the validation override removed. Schema setup runs on its existing
one-connection untimed pool, followed by workers and pruners, before HTTP promotion. The workflow
checks digest, full runtime spec and secret-reference shape, scaling, readiness and active mode,
then moves HTTP traffic and checks the public origin. Those checks commit the new serving revision;
subsequent retirement failures do not trigger rollback to a possibly deleted previous revision.
The previous consumer is retired and all tags are cleared. Retain the previous immutable image
from the summary: later recovery redeploys that digest, because the previous revision is deleted.
Failure before activation deletes the inert candidate. Failure during activation also deletes the
partially created active revision when traffic has not moved. After any attempted traffic shift,
the workflow first restores and verifies previous traffic, then removes the candidate tag and
deletes the rejected revision. It then restores the service template with the previous serving
revision’s resolved image digest and no validation override, leaving traffic on the old revision.
This creates an untagged recovery revision: known-good schema and workers can execute before
it is retired, even with no HTTP traffic. The workflow verifies the template’s runtime settings,
secret references, scaling and normal mode, then deletes the recovery revision under the same
lease. Deletion leaves the safe service template in place for later Terraform reconciliation.
If candidate deletion fails, no recovery revision is created; failed recovery attempts still
record their revision name for retirement and operator diagnosis.
Before any candidate creation, the workflow requires exactly one revision resource, the sole HTTP
serving revision. Existing historical revisions or leftovers from interrupted runs require explicit
operator review and cleanup under the lease first; the workflow does not blindly delete them.
This gate and retirement after every successful rollout prevent repeated runs accumulating workers.
Terraform still owns configuration and scaling; removing validation mode adds no ignored field.
Traffic restoration alone does **not** stop queue consumers. If rollback, template restoration
or deletion fails, operator recovery must complete under the rollout lease; do not call
the rollout recovered merely because the old origin answers. A canceled runner can also require
manual cleanup. Successful rollout removes the active candidate tag.
On failure before public checks pass, any attempted traffic shift is first rolled back and verified.
If partial activation created a successor, recovery retires non-latest validation first; deletion
failure stops recovery before a fourth resource can be created. Recovery then deploys the captured
known-good digest as a tagged, zero-traffic successor with normal mode. It verifies template shape,
secret references and scaling, probes tagged readiness and active mode, promotes the recovery
revision, verifies traffic and public health, and only then deletes rejected and previous revisions.
The latest recovery revision remains serving. Known-good recovery schema and workers can execute
before promotion; neither recovery nor traffic rollback undoes schema changes or sent notifications.
Manual rollback must restore traffic, retire the rejected revision, and restore the service
template under the rollout lease. Use the exact names and known-good image digest from the summary:
Partial creates record deterministic names before mutation. Failed recovery or deletion requires
operator cleanup under the lease; the next automated run refuses leftover resources. A canceled
runner can require the same intervention. Traffic restoration alone does not stop queue consumers.
Manual recovery must preserve the three-resource bound and keep the successor serving:
```sh
gcloud run services update-traffic orca-cloud-push \
--project onorca-cloud --region us-central1 --to-revisions <previous-revision>=100
gcloud run services update-traffic orca-cloud-push \
--project onorca-cloud --region us-central1 --remove-tags <candidate-tag>
gcloud run revisions delete <rejected-active-revision> \
--project onorca-cloud --region us-central1
# Verify termination/drain before creating another revision.
# Hold the rollout lease; inspect latest, traffic, tags and existing revisions first.
# If three resources remain after partial activation, retire non-latest inert validation first.
# Restore previous traffic if its revision still exists and a failed candidate took traffic.
gcloud run deploy orca-cloud-push \
--project onorca-cloud --region us-central1 --image <known-good-image-at-digest> \
--remove-env-vars ORCA_PUSH_MODE --no-traffic --revision-suffix <unique-recovery-suffix>
# Verify template image/mode/runtime/secret references/scaling and unchanged traffic, then retire it.
gcloud run revisions delete <recovery-revision> \
--remove-env-vars ORCA_PUSH_MODE --no-traffic \
--tag <unique-recovery-tag> --revision-suffix <unique-recovery-suffix>
# Verify exact digest, template spec/secret references/scaling, tagged /ready and active /health.
gcloud run services update-traffic orca-cloud-push \
--project onorca-cloud --region us-central1 --to-revisions <recovery-revision>=100
# Verify traffic and public /ready and /health before retiring old consumers.
gcloud run services update-traffic orca-cloud-push \
--project onorca-cloud --region us-central1 --clear-tags
gcloud run revisions delete <rejected-or-previous-revision> \
--project onorca-cloud --region us-central1
# Repeat only for reviewed obsolete revisions; retain the latest serving recovery revision.
```
Never merely remove validation mode while the template still holds a rejected image. Terraform
owns environment configuration but ignores the image, so that would activate rejected code.
Remove a tag only if it remains present. Verify the old revision is serving, the template is safe,
and both rejected/recovery revision deletion and connection drain completed;
Remove a tag only if it remains present. Verify the recovery revision is serving, the template is safe,
and obsolete revision deletion and connection drain completed;
already accepted provider sends cannot be undone. Activation-time schema changes must be additive
and compatible with the rollback image: rollback does not reverse migrations or queue mutations.
The inert phase intentionally cannot validate a new schema by applying it to production. Review
@@ -351,9 +360,10 @@ issuance and breaks Cloud Run host routing.
Candidate tags and deterministic revision names are recorded before deployment. Promotion intent is
recorded before changing traffic, so a failed verification or ambiguous mutation result still triggers
rollback. Failed candidates are deleted only before attempted promotion or after verified rollback.
A known-good template is restored after successful cleanup, and its untagged recovery revision
is retired so it cannot remain an extra consumer. The summary runs even if candidate discovery or traffic verification fails.
rollback. A known-good successor must exist before the rejected latest revision can be deleted.
After verified recovery promotion and public checks, rejected and previous consumers are retired;
the recovery revision remains serving. Failed cleanup blocks subsequent rollout admission.
The summary runs even if candidate discovery or traffic verification fails.
Push uses the relay's schema-startup retry implementation through `@orca-cloud/postgres-schema`.
Session replacement is serialized per host and a unique host index upgrades older databases by
cloud/infra/terraform/push-gateway.tf
+2 -2
View File
@@ -310,8 +310,8 @@ resource "google_cloud_run_v2_service" "push" {
}
precondition {
condition = !var.push_dedicated_database_active || var.push_max_instances * var.push_database_pool_max * 2 <= 64
error_message = "Dedicated push serving and candidate pools must fit the 64-connection rollout budget."
condition = !var.push_dedicated_database_active || var.push_max_instances * var.push_database_pool_max * 3 <= 64
error_message = "Dedicated push serving, validation/rejected and successor pools must fit the 64-connection rollout budget."
}
ignore_changes = [