Restore independent push gateway deployment (#19225)

* Restore isolated push gateway deployment workflow

* Register push deployment in the shared SQL lease census

* Restore push workflow inventory and identity contracts

.github/workflows/cloud-push-deploy.yml

@@ -0,0 +1,363 @@
+name: Deploy Push Gateway Production
+
+on:
+  workflow_dispatch:
+    inputs:
+      source_sha:
+        description: Full reviewed commit SHA to build (feature may remain unmerged)
+        required: true
+        type: string
+      confirmation:
+        description: Enter DEPLOY_PUSH_GATEWAY to shift production traffic
+        required: true
+        type: string
+
+permissions:
+  contents: read
+  id-token: write
+
+# The gateway applies its own schema at startup against the shared Cloud SQL instance, so a
+# deploy is a connection-budget rollout and belongs in the same serialized group as the relay.
+concurrency:
+  group: production-cloud-sql-rollout
+  cancel-in-progress: false
+
+defaults:
+  run:
+    working-directory: cloud
+
+jobs:
+  deploy:
+    if: >-
+      ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
+      github.ref == 'refs/heads/main' }}
+    runs-on: blacksmith-2vcpu-ubuntu-2204
+    environment: production
+    env:
+      GCP_PROJECT_ID: onorca-cloud
+      GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
+      SERVICE_NAME: orca-cloud-push
+      REPOSITORY_ID: orca-cloud
+      IMAGE_NAME: push
+      PUSH_ORIGIN: https://push.onorca.dev
+      PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud.iam.gserviceaccount.com
+      # Scaling the serving revision must already hold, matching push_min_instances and
+      # push_max_instances. Terraform owns both, and the candidate inherits them from the
+      # service, so this deploy never passes a scaling flag: doing so would write a
+      # Terraform-owned field that `lifecycle.ignore_changes` does not cover, and a later
+      # `push_max_instances` raise would then be reverted by every deploy. These two values
+      # are the expected shape, asserted before the candidate is created and again on the
+      # candidate itself, so a deploy that would change the gateway's Cloud SQL draw fails.
+      PUSH_MIN_INSTANCES: 1
+      PUSH_MAX_INSTANCES: 2
+      CONFIRMATION: ${{ inputs.confirmation }}
+      SOURCE_SHA: ${{ inputs.source_sha }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Require the explicit deploy confirmation
+        shell: bash
+        run: |
+          set -euo pipefail
+          test "${CONFIRMATION}" = DEPLOY_PUSH_GATEWAY
+          [[ "${SOURCE_SHA}" =~ ^[a-f0-9]{40}$ ]]
+
+      # Keep the workflow and rollout lease on main; only the Docker build uses candidate code.
+      - name: Fetch the immutable gateway source
+        shell: bash
+        run: |
+          set -euo pipefail
+          git fetch --no-tags origin "${SOURCE_SHA}"
+          test "$(git rev-parse FETCH_HEAD)" = "${SOURCE_SHA}"
+          mkdir -p "${RUNNER_TEMP}/push-source"
+          git archive "${SOURCE_SHA}" cloud | tar -x -C "${RUNNER_TEMP}/push-source"
+
+      - uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+
+      - uses: google-github-actions/setup-gcloud@v2
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Configure Docker auth
+        run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
+
+      # Why: the build runs before the lease. Artifact Registry is not the Cloud SQL instance,
+      # and a multi-minute image build inside the lease blocks every relay deploy and rehome for
+      # its duration. The lease below covers exactly the connection-budget window: deploy, probe,
+      # shift.
+      - name: Build and publish the immutable gateway image
+        shell: bash
+        run: |
+          set -euo pipefail
+          image_tag="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${SOURCE_SHA}"
+          docker build -f "${RUNNER_TEMP}/push-source/cloud/apps/push/Dockerfile" \
+            -t "${image_tag}" "${RUNNER_TEMP}/push-source/cloud"
+          docker push "${image_tag}"
+          digest="$(gcloud artifacts docker images describe "${image_tag}" \
+            --format='value(image_summary.digest)')"
+          [[ "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
+          echo "IMAGE=${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${digest}" \
+            >> "${GITHUB_ENV}"
+          echo "IMAGE_DIGEST=${digest}" >> "${GITHUB_ENV}"
+
+      # Held across the deploy, not just a separate schema step: the gateway opens its pool and
+      # applies its schema while the new revision starts, so the revision is the schema step.
+      - uses: ./.github/actions/cloud-sql-rollout-lease
+        with:
+          bucket: onorca-cloud-terraform-state
+          object: terraform/state/cloud-sql-rollout/production.lock
+
+      # Why: the candidate inherits the serving revision's scaling. A serving revision that has
+      # drifted below the floor would hand the candidate a cold start on every notification, and
+      # one that has drifted above the ceiling would hand it a larger Cloud SQL draw than the
+      # rollout lease was taken for. Refuse to inherit either rather than latch it.
+      - name: Record the serving revision and require its Terraform-owned scaling
+        shell: bash
+        run: |
+          set -euo pipefail
+          serving="$(gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            | jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
+                     | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
+          test -n "${serving}"
+          floor="$(gcloud run revisions describe "${serving}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
+          if [[ "${floor:-0}" -lt "${PUSH_MIN_INSTANCES}" ]]; then
+            echo "serving revision ${serving} holds ${floor:-0} minimum instances," \
+              "below ${PUSH_MIN_INSTANCES}; deploying would inherit and latch it." >&2
+            echo "Restore the floor first: gcloud run services update ${SERVICE_NAME}" \
+              "--region ${GCP_REGION} --min-instances=${PUSH_MIN_INSTANCES}" >&2
+            exit 1
+          fi
+          ceiling="$(gcloud run revisions describe "${serving}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
+          test "${ceiling}" = "${PUSH_MAX_INSTANCES}"
+          echo "serving revision ${serving} holds ${floor} minimum and ${ceiling} maximum instances"
+          echo "ROLLBACK_REVISION=${serving}" >> "${GITHUB_ENV}"
+
+      # No traffic and a per-revision tag: the candidate boots, applies schema, and is probed on
+      # its own URL while every phone and desktop still reaches the previous revision.
+      - name: Deploy the candidate revision with no traffic
+        shell: bash
+        run: |
+          set -euo pipefail
+          tag="c${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          echo "CANDIDATE_TAG=${tag}" >> "${GITHUB_ENV}"
+          echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
+          gcloud run deploy "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" \
+            --region "${GCP_REGION}" \
+            --image "${IMAGE}" \
+            --tag "${tag}" \
+            --revision-suffix "${tag}" \
+            --no-traffic \
+            --quiet
+          candidate="$(gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            | jq -er --arg tag "${tag}" \
+              '[.status.traffic[] | select(.tag == $tag)]
+               | if length == 1 then .[0] else error("tagged candidate is not unique") end')"
+          test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
+          echo "CANDIDATE_URL=$(jq -r '.url' <<< "${candidate}")" >> "${GITHUB_ENV}"
+
+      # A tagged revision is directly addressable and sits outside the service-wide cap, so the
+      # candidate and the serving revision each draw up to the ceiling during the probe window.
+      # The lease is taken for exactly that doubling; a candidate that inherited a wider ceiling
+      # would exceed it, so the inherited scaling is asserted here too.
+      - name: Require the candidate to serve the exact image and inherited scaling
+        shell: bash
+        run: |
+          set -euo pipefail
+          served="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --format='value(spec.containers[0].image)')"
+          test "${served}" = "${IMAGE}"
+          test "${CANDIDATE_REVISION}" != "${ROLLBACK_REVISION}"
+          candidate_ceiling="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
+          test "${candidate_ceiling}" = "${PUSH_MAX_INSTANCES}"
+
+      - name: Probe the candidate readiness endpoint
+        shell: bash
+        run: |
+          set -euo pipefail
+          [[ "${CANDIDATE_URL}" =~ ^https://[^/]+$ ]]
+          for attempt in $(seq 1 30); do
+            code="$(curl -sS -o "${RUNNER_TEMP}/push-ready.json" -w '%{http_code}' \
+              --max-time 10 "${CANDIDATE_URL}/ready" || true)"
+            if test "${code}" = 200; then
+              jq -e . < "${RUNNER_TEMP}/push-ready.json" > /dev/null
+              curl --fail --silent --show-error --max-time 10 "${CANDIDATE_URL}/health" \
+                | jq -e '.ok == true and .deliveryProtocol == 2' > /dev/null
+              echo "candidate ${CANDIDATE_REVISION} is ready after ${attempt} attempt(s)"
+              exit 0
+            fi
+            echo "attempt ${attempt}: /ready returned ${code}"
+            sleep 5
+          done
+          echo "candidate ${CANDIDATE_REVISION} never reported ready" >&2
+          exit 1
+
+      # Why: a gateway that boots and answers /ready can still be unable to send. This proves the
+      # runtime account's FCM grant end to end without delivering anything: validate_only stops
+      # Google before any push, and the deliberately invalid token means a healthy credential
+      # answers INVALID_ARGUMENT. PERMISSION_DENIED is the failure this step exists to catch.
+      #
+      # Only the four verdicts below are conclusive. A 429, a 5xx, or a transport failure says
+      # nothing about the credential, so it is retried rather than treated as either answer; a
+      # denied credential still fails on the first attempt, without burning the retries.
+      - name: Prove the runtime identity can reach FCM
+        shell: bash
+        run: |
+          set -euo pipefail
+          token="$(gcloud auth print-access-token \
+            --impersonate-service-account "${PUSH_RUNTIME_SERVICE_ACCOUNT}")"
+          test -n "${token}"
+          echo "::add-mask::${token}"
+          body='{"validate_only":true,"message":{"token":"orca-push-deploy-probe-invalid-token","notification":{"title":"Orca","body":"deploy probe"}}}'
+          for attempt in $(seq 1 5); do
+            code="$(curl -sS -o "${RUNNER_TEMP}/push-fcm.json" -w '%{http_code}' --max-time 20 \
+              -X POST "https://fcm.googleapis.com/v1/projects/${GCP_PROJECT_ID}/messages:send" \
+              -H "Authorization: Bearer ${token}" \
+              -H 'Content-Type: application/json' \
+              --data "${body}" || true)"
+            status="$(jq -r '.error.status // empty' < "${RUNNER_TEMP}/push-fcm.json" || true)"
+            echo "attempt ${attempt}: FCM validate-only send returned HTTP ${code} status ${status:-OK}"
+            if test "${status}" = PERMISSION_DENIED || test "${status}" = INVALID_ARGUMENT ||
+              test "${code}" = 401 || test "${code}" = 403; then
+              break
+            fi
+            sleep 5
+          done
+          if test "${status}" = PERMISSION_DENIED || test "${code}" = 401 || test "${code}" = 403; then
+            echo "the push runtime identity cannot send through FCM" >&2
+            exit 1
+          fi
+          test "${status}" = INVALID_ARGUMENT
+
+      - name: Shift all traffic to the verified candidate
+        shell: bash
+        run: |
+          set -euo pipefail
+          echo "TRAFFIC_SHIFT_ATTEMPTED=true" >> "${GITHUB_ENV}"
+          gcloud run services update-traffic "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" \
+            --region "${GCP_REGION}" \
+            --to-revisions "${CANDIDATE_REVISION}=100" \
+            --quiet
+          serving="$(gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            | jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
+                     | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
+          test "${serving}" = "${CANDIDATE_REVISION}"
+          echo "TRAFFIC_SHIFTED=true" >> "${GITHUB_ENV}"
+
+      # Why: the summary is written before the origin check, not after it. Once traffic has
+      # moved, the rollback target is the single thing an operator needs, and a summary that only
+      # appeared on success would be missing in exactly the run that needs it.
+      - name: Publish the rollout summary
+        if: ${{ always() && env.CANDIDATE_REVISION != '' && env.ROLLBACK_REVISION != '' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          {
+            echo '### Push gateway rollout'
+            echo
+            echo "Source: ${SOURCE_SHA}"
+            echo
+            echo "Revision: \`${CANDIDATE_REVISION}\`"
+            echo
+            echo "Image: \`${IMAGE_DIGEST}\`"
+            echo
+            echo "Rollback: \`gcloud run services update-traffic ${SERVICE_NAME}" \
+              "--region ${GCP_REGION} --to-revisions ${ROLLBACK_REVISION}=100\`"
+          } >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Verify the public origin after the shift
+        shell: bash
+        run: |
+          set -euo pipefail
+          for attempt in $(seq 1 30); do
+            code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \
+              "${PUSH_ORIGIN}/ready" || true)"
+            if test "${code}" = 200; then
+              curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
+                | jq -e '.ok == true and .deliveryProtocol == 2' > /dev/null
+              echo "${PUSH_ORIGIN} is ready after ${attempt} attempt(s)"
+              exit 0
+            fi
+            echo "attempt ${attempt}: ${PUSH_ORIGIN}/ready returned ${code}"
+            sleep 5
+          done
+          echo "${PUSH_ORIGIN} never reported ready after the shift" >&2
+          exit 1
+
+      # Why: everything after the shift runs with production on the candidate. A failure there
+      # is not a failure to deploy, it is a live gateway that has to go back, so the traffic move
+      # is undone here rather than left to whoever reads the run.
+      - name: Roll traffic back to the previous revision
+        if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          test -n "${ROLLBACK_REVISION:-}"
+          gcloud run services update-traffic "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" \
+            --region "${GCP_REGION}" \
+            --to-revisions "${ROLLBACK_REVISION}=100" \
+            --quiet
+          serving="$(gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            | jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
+                     | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
+          test "${serving}" = "${ROLLBACK_REVISION}"
+          echo "TRAFFIC_ROLLED_BACK=true" >> "${GITHUB_ENV}"
+          {
+            echo
+            echo '### Push gateway rolled back'
+            echo
+            echo "Traffic returned to \`${ROLLBACK_REVISION}\`; the candidate" \
+              "\`${CANDIDATE_REVISION}\` no longer serves."
+          } >> "${GITHUB_STEP_SUMMARY}"
+
+      # Why: a candidate that never took traffic is a revision holding a warm floor and a Cloud
+      # SQL pool for nothing. Its tag comes off first, because Cloud Run refuses to delete a
+      # revision a traffic target still names, and clearing CANDIDATE_TAG makes the always() tag
+      # step below a no-op rather than a second failure.
+      - name: Delete the rejected candidate revision
+        if: ${{ (failure() || cancelled()) && (env.TRAFFIC_SHIFT_ATTEMPTED != 'true' || env.TRAFFIC_ROLLED_BACK == 'true') }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          test -n "${CANDIDATE_REVISION:-}" || exit 0
+          if test -n "${CANDIDATE_TAG:-}"; then
+            gcloud run services update-traffic "${SERVICE_NAME}" \
+              --project "${GCP_PROJECT_ID}" \
+              --region "${GCP_REGION}" \
+              --remove-tags "${CANDIDATE_TAG}" \
+              --quiet
+            echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
+          fi
+          gcloud run revisions delete "${CANDIDATE_REVISION}" \
+            --project "${GCP_PROJECT_ID}" \
+            --region "${GCP_REGION}" \
+            --quiet
+          echo "deleted the candidate revision ${CANDIDATE_REVISION}"
+
+      - name: Drop the candidate traffic tag
+        if: always()
+        shell: bash
+        run: |
+          set -euo pipefail
+          test -n "${CANDIDATE_TAG:-}" || exit 0
+          gcloud run services update-traffic "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" \
+            --region "${GCP_REGION}" \
+            --remove-tags "${CANDIDATE_TAG}" \
+            --quiet

cloud/dev/scripts/cloud-sql-rollout-lock-census.mjs

@@ -283,6 +283,8 @@ export const LEASED_WORKFLOWS = named([
     'operate-relay-production-rehome.yml',
     production({ leaseFiles: ['operate-relay-production-rehome-job.yml'] })
   ],
+  // The gateway applies its schema at startup, so its deploy revision is the schema step.
+  ['push-deploy.yml', production()],
   ['deploy-relay-asia-topology.yml', eitherEnvironment()],
   ['operate-relay-asia-admission.yml', eitherEnvironment()],
   ['deploy-relay-staging.yml', staging()],

cloud/dev/scripts/relay-production-identity-boundaries.test.mjs

@@ -32,7 +32,8 @@ test('no workflow names the retired generic production deploy identity', async (
     'deploy-relay-production.yml',
     'operate-relay-asia-admission.yml',
     'operate-relay-production-rehome-job.yml',
-    'publish-relay-production.yml'
+    'publish-relay-production.yml',
+    'push-deploy.yml'
   ].map((name) => relayWorkflowFile(name)).sort())
 })
 

cloud/dev/scripts/relay-public-workflow-contract.test.mjs

@@ -20,7 +20,7 @@ const UNGATED = relayWorkflowFile('verify.yml')
 const relayWorkflows = () => workflowFiles().filter((file) => file !== UNGATED)
 
 test('the copy carries every relay workflow', () => {
-  assert.equal(relayWorkflows().length, 24)
+  assert.equal(relayWorkflows().length, 25)
 })
 
 // Why: workflow_run chains match by display name, not filename. Renaming a file is safe; renaming