From 307f0cdaeedc25ce0b3bae1cd96d66df914fa458 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:01:25 -0700 Subject: [PATCH] Move Relay deployment configuration into private cloud operations (#31792) * refactor(cloud): move deployment configuration into private operations repo * docs(cloud): clarify private operator ownership * fix(ci): install pnpm through npm-compatible bootstrap * fix(ci): avoid native downloader metadata timeout * test(push): move deployment preflight contract into private repo * fix(ci): bootstrap Node before installing pnpm in minimal containers * fix(ci): pass Windows boundary test paths through the Node runner * fix(ci): run Windows boundary Vitest with installed Node --- ...cloud-bootstrap-relay-staging-capacity.yml | 678 ---- .../cloud-deploy-relay-asia-topology.yml | 288 -- .../cloud-deploy-relay-fence-broker.yml | 92 - ...d-deploy-relay-production-capacity-job.yml | 825 ---- ...cloud-deploy-relay-production-capacity.yml | 356 -- ...cloud-deploy-relay-production-director.yml | 311 -- ...d-deploy-relay-production-multi-target.yml | 500 --- ...d-deploy-relay-production-same-cap-job.yml | 930 ----- ...cloud-deploy-relay-production-same-cap.yml | 433 --- .../cloud-deploy-relay-production.yml | 220 -- ...oud-deploy-relay-staging-gce-candidate.yml | 112 - .../workflows/cloud-deploy-relay-staging.yml | 242 -- .../cloud-monitor-relay-clock-skew.yml | 90 - .../cloud-monitor-relay-production-job.yml | 216 -- .../cloud-monitor-relay-production.yml | 75 - .../cloud-operate-relay-asia-admission.yml | 558 --- ...ud-operate-relay-production-cell-flags.yml | 127 - ...ud-operate-relay-production-rehome-job.yml | 342 -- .../cloud-operate-relay-production-rehome.yml | 112 - ...operate-relay-production-reserve-watch.yml | 341 -- ...cloud-operate-relay-staging-cell-flags.yml | 129 - .../workflows/cloud-power-relay-staging.yml | 104 - .../cloud-prove-relay-asia-staging.yml | 500 --- .../cloud-prove-relay-staging-capacity.yml | 928 ----- .../cloud-publish-relay-production.yml | 133 - .github/workflows/cloud-push-deploy.yml | 568 --- .../cloud-recover-relay-staging-c4-image.yml | 414 -- ...loud-requeue-relay-staging-c4-recovery.yml | 60 - .github/workflows/cloud-verify.yml | 76 +- .github/workflows/pr.yml | 2 +- cloud/README.md | 56 +- .../push/src/push-deploy-preflight.test.ts | 17 - cloud/apps/relay-fence-broker/Dockerfile | 51 - cloud/apps/relay-ops/README.md | 66 - cloud/apps/relay-ops/package.json | 30 - cloud/apps/relay-ops/public/app.js | 273 -- cloud/apps/relay-ops/public/index.html | 115 - cloud/apps/relay-ops/public/styles.css | 165 - cloud/apps/relay-ops/src/cost-model.test.ts | 85 - cloud/apps/relay-ops/src/cost-model.ts | 115 - .../relay-ops/src/dashboard-snapshot.test.ts | 62 - .../apps/relay-ops/src/dashboard-snapshot.ts | 167 - .../relay-ops/src/environment-config.test.ts | 156 - .../apps/relay-ops/src/environment-config.ts | 130 - .../apps/relay-ops/src/gcloud-client.test.ts | 43 - cloud/apps/relay-ops/src/gcloud-client.ts | 77 - cloud/apps/relay-ops/src/github-runs.ts | 77 - .../src/incident-live-preflight-cli.test.ts | 874 ----- .../src/incident-live-preflight-cli.ts | 360 -- .../src/incident-monitor-cli.test.ts | 455 --- .../relay-ops/src/incident-monitor-cli.ts | 510 --- .../src/incident-monitor-sources.test.ts | 655 ---- .../relay-ops/src/incident-monitor-sources.ts | 783 ---- .../relay-ops/src/incident-monitor.test.ts | 1728 --------- cloud/apps/relay-ops/src/incident-monitor.ts | 1111 ------ cloud/apps/relay-ops/src/incident-selector.ts | 77 - cloud/apps/relay-ops/src/index.ts | 108 - .../relay-ops/src/monitoring-snapshot.test.ts | 85 - .../apps/relay-ops/src/monitoring-snapshot.ts | 379 -- .../src/pre-drain-hard-rule-sources.test.ts | 210 -- .../src/pre-drain-hard-rule-sources.ts | 217 -- .../src/pre-drain-sample-cli.test.ts | 101 - .../relay-ops/src/pre-drain-sample-cli.ts | 158 - .../relay-ops/src/pre-drain-sample.test.ts | 358 -- cloud/apps/relay-ops/src/pre-drain-sample.ts | 259 -- .../relay-ops/src/relay-repository.test.ts | 42 - cloud/apps/relay-ops/src/relay-repository.ts | 14 - .../relay-ops/src/resource-inventory.test.ts | 370 -- .../apps/relay-ops/src/resource-inventory.ts | 437 --- .../relay-ops/src/staging-workflow.test.ts | 19 - cloud/apps/relay-ops/src/staging-workflow.ts | 36 - cloud/apps/relay-ops/tsconfig.build.json | 4 - cloud/apps/relay-ops/tsconfig.json | 11 - ...ll-connection-hard-cap-consistency.test.ts | 75 - .../src/cell-startup-environment.test.ts | 55 - ...ional-rehome-inventory-line-census.test.ts | 163 - .../production-cloud-sql-app-consumers.json | 15 - .../2026-10-21-director-stall.json | 151 - .../terraform-root-partition/families.json | 339 -- .../capture-terraform-plan-baseline.mjs | 83 - .../capture-terraform-plan-baseline.test.mjs | 15 - .../scripts/check-relay-same-cap-headroom.mjs | 167 - .../check-relay-same-cap-headroom.test.mjs | 230 -- ...ify-relay-production-capacity-director.mjs | 109 - ...elay-production-capacity-director.test.mjs | 93 - .../classify-relay-staging-bootstrap.mjs | 47 - .../classify-relay-staging-bootstrap.test.mjs | 60 - .../scripts/cloud-sql-rollout-lock-census.mjs | 314 -- cloud/dev/scripts/deploy-relay-blue-green.mjs | 1538 -------- .../scripts/deploy-relay-blue-green.test.mjs | 1343 ------- .../scripts/deploy-relay-gce-candidate.mjs | 876 ----- .../deploy-relay-gce-candidate.test.mjs | 922 ----- .../scripts/deploy-relay-gce-multi-target.mjs | 3135 ---------------- .../deploy-relay-gce-multi-target.test.mjs | 3320 ----------------- .../scripts/drive-relay-director-deploy.mjs | 1012 ----- .../drive-relay-director-deploy.test.mjs | 1090 ------ cloud/dev/scripts/github-smoke-token.mjs | 136 - cloud/dev/scripts/github-smoke-token.test.mjs | 111 - cloud/dev/scripts/infra.mjs | 146 - cloud/dev/scripts/infra.test.mjs | 77 - cloud/dev/scripts/load-relay-controls.mjs | 695 ---- .../measure-relay-drain-disconnect-gap.mjs | 161 - ...easure-relay-drain-disconnect-gap.test.mjs | 158 - .../scripts/operate-relay-asia-admission.mjs | 509 --- .../operate-relay-asia-admission.test.mjs | 967 ----- .../dev/scripts/operate-relay-cell-flags.mjs | 619 --- .../scripts/operate-relay-cell-flags.test.mjs | 611 --- .../scripts/operate-relay-regional-rehome.mjs | 344 -- .../operate-relay-regional-rehome.test.mjs | 446 --- cloud/dev/scripts/power-staging-relay.mjs | 537 --- .../dev/scripts/power-staging-relay.test.mjs | 427 --- .../prepare-relay-asia-director-cells.mjs | 86 - ...prepare-relay-asia-director-cells.test.mjs | 153 - .../prepare-relay-asia-topology-input.mjs | 140 - ...prepare-relay-asia-topology-input.test.mjs | 211 -- .../scripts/prepare-relay-capacity-canary.mjs | 183 - .../prepare-relay-capacity-canary.test.mjs | 300 -- ...epare-relay-production-capacity-canary.mjs | 196 - ...-relay-production-capacity-canary.test.mjs | 391 -- .../scripts/probe-relay-legacy-admission.mjs | 73 - .../probe-relay-legacy-admission.test.mjs | 89 - .../dev/scripts/probe-relay-rehome-trust.mjs | 112 - .../scripts/probe-relay-rehome-trust.test.mjs | 217 -- ...ion-cell-image-digest-consistency.test.mjs | 85 - ...production-cloud-sql-rollout-lock.test.mjs | 182 - cloud/dev/scripts/push-cloud-run-model.mjs | 140 - .../scripts/push-gateway-recovery.test.mjs | 373 -- .../scripts/push-gateway-workflow.test.mjs | 329 -- ...ead-relay-production-capacity-identity.mjs | 31 - ...elay-production-capacity-identity.test.mjs | 44 - ...lay-serving-regional-placement-version.mjs | 140 - ...erving-regional-placement-version.test.mjs | 209 -- ...lay-admin-endpoint-retry-workflow.test.mjs | 137 - .../scripts/relay-admin-transient-retry.mjs | 29 - .../relay-admin-transient-retry.test.mjs | 130 - .../dev/scripts/relay-admission-selector.mjs | 280 -- .../scripts/relay-admission-selector.test.mjs | 232 -- .../scripts/relay-asia-cloud-sql-metrics.mjs | 33 - .../scripts/relay-asia-rollout-evidence.mjs | 639 ---- .../relay-asia-rollout-evidence.test.mjs | 592 --- .../relay-asia-topology-workflow.test.mjs | 244 -- ...relay-clock-skew-monitor-workflow.test.mjs | 73 - .../relay-cloud-sql-connection-budget.mjs | 148 - ...relay-cloud-sql-connection-budget.test.mjs | 173 - .../relay-control-flag-alerts.test.mjs | 37 - .../scripts/relay-db-fault-alerts.test.mjs | 76 - .../relay-deploy-workflow-argv.test.mjs | 106 - .../scripts/relay-director-deploy-plan.mjs | 335 -- .../relay-evidence-code-provenance.mjs | 101 - .../dev/scripts/relay-gce-terraform-fence.mjs | 1387 ------- .../relay-gce-terraform-fence.test.mjs | 1522 -------- .../scripts/relay-live-cell-image-overlay.mjs | 84 - .../relay-live-cell-image-overlay.test.mjs | 119 - .../scripts/relay-load-connection-failure.mjs | 45 - .../relay-load-connection-failure.test.mjs | 57 - cloud/dev/scripts/relay-load-control-peer.mjs | 918 ----- .../scripts/relay-load-control-peer.test.mjs | 1074 ------ .../relay-load-director-capacity-gate.mjs | 147 - ...relay-load-director-capacity-gate.test.mjs | 260 -- .../scripts/relay-load-duplicate-assign.mjs | 66 - .../relay-load-duplicate-assign.test.mjs | 127 - cloud/dev/scripts/relay-load-model.mjs | 76 - cloud/dev/scripts/relay-load-model.test.mjs | 25 - .../dev/scripts/relay-load-phase-barrier.mjs | 37 - .../scripts/relay-load-phase-barrier.test.mjs | 65 - .../scripts/relay-load-placement-boundary.mjs | 29 - .../relay-load-placement-boundary.test.mjs | 53 - cloud/dev/scripts/relay-load-profile.mjs | 440 --- cloud/dev/scripts/relay-load-profile.test.mjs | 427 --- .../scripts/relay-load-reader-evidence.mjs | 51 - .../relay-load-reader-evidence.test.mjs | 76 - .../scripts/relay-load-rebind-boundary.mjs | 59 - .../relay-load-rebind-boundary.test.mjs | 164 - .../scripts/relay-load-region-behavior.mjs | 35 - .../relay-load-region-behavior.test.mjs | 47 - .../relay-load-request-unit-boundary.mjs | 43 - .../relay-load-request-unit-boundary.test.mjs | 52 - cloud/dev/scripts/relay-load-resume-phone.mjs | 220 -- .../scripts/relay-load-resume-phone.test.mjs | 24 - .../dev/scripts/relay-load-run-lifecycle.mjs | 25 - .../scripts/relay-load-run-lifecycle.test.mjs | 68 - cloud/dev/scripts/relay-load-splice-retry.mjs | 40 - .../scripts/relay-load-splice-retry.test.mjs | 81 - .../relay-lock-contention-alerts.test.mjs | 83 - cloud/dev/scripts/relay-monitor-evidence.mjs | 359 -- .../scripts/relay-monitor-evidence.test.mjs | 640 ---- ...ay-phone-watch-recorded-quiet-minutes.json | 1 - .../relay-production-capacity-wave.mjs | 172 - .../relay-production-capacity-wave.test.mjs | 129 - ...elay-production-capacity-workflow.test.mjs | 440 --- ...ay-production-identity-boundaries.test.mjs | 180 - .../relay-production-same-cap-wave.mjs | 322 -- .../relay-production-same-cap-wave.test.mjs | 946 ----- .../relay-public-workflow-contract.test.mjs | 82 - .../relay-reconnect-herd-alerts.test.mjs | 45 - .../dev/scripts/relay-recovery-wave-gate.mjs | 517 --- .../scripts/relay-recovery-wave-gate.test.mjs | 214 -- .../relay-region-hint-metrics.test.mjs | 84 - .../relay-region-observation-evidence.mjs | 152 - ...relay-region-observation-evidence.test.mjs | 53 - .../relay-rehome-aggregate-evidence.mjs | 106 - .../relay-rehome-aggregate-evidence.test.mjs | 135 - cloud/dev/scripts/relay-repository.mjs | 51 - cloud/dev/scripts/relay-repository.test.mjs | 39 - .../scripts/relay-reserve-cell-refusal.mjs | 16 - .../relay-reserve-dead-man-alert.test.mjs | 109 - .../dev/scripts/relay-reserve-watch-rules.mjs | 494 --- ...elay-same-cap-job-mode-conditions.test.mjs | 163 - .../relay-same-cap-script-census.test.mjs | 692 ---- .../relay-same-cap-shadow-gate-verdict.mjs | 470 --- .../scripts/relay-same-cap-shadow-gate.mjs | 494 --- .../relay-same-cap-shadow-gate.test.mjs | 842 ----- ...relay-staging-c4-refresh-workflow.test.mjs | 172 - .../relay-staging-capacity-identity.test.mjs | 269 -- .../relay-staging-deploy-identity.test.mjs | 238 -- .../render-workload-identity-conditions.mjs | 536 --- cloud/dev/scripts/run-relay-load-model.mjs | 8 - .../scripts/run-relay-recovery-wave-gate.mjs | 19 - .../sanitize-relay-asia-admission-result.mjs | 98 - ...itize-relay-asia-admission-result.test.mjs | 120 - cloud/dev/scripts/smoke-relay.mjs | 211 -- .../dev/scripts/staging-relay-apply-guard.mjs | 51 - .../staging-relay-apply-guard.test.mjs | 30 - .../dev/scripts/terraform-root-partition.mjs | 131 - .../scripts/terraform-root-partition.test.mjs | 117 - .../validate-relay-asia-topology-plan.mjs | 345 -- ...validate-relay-asia-topology-plan.test.mjs | 535 --- .../scripts/validate-relay-capacity-plan.mjs | 667 ---- .../validate-relay-capacity-plan.test.mjs | 1430 ------- .../verify-relay-capacity-transition.mjs | 537 --- .../verify-relay-capacity-transition.test.mjs | 1511 -------- .../scripts/verify-relay-legacy-bootstrap.mjs | 292 -- .../verify-relay-legacy-bootstrap.test.mjs | 395 -- .../dev/scripts/watch-relay-reserve-cell.mjs | 875 ----- .../scripts/watch-relay-reserve-cell.test.mjs | 1680 --------- ...oad-identity-attribute-conditions.test.mjs | 155 - cloud/docs/orca-relay-capacity-testing.md | 259 -- cloud/docs/orca-relay-operations.md | 692 ---- cloud/docs/push-database-cutover.md | 112 - cloud/docs/push-gateway.md | 405 -- .../relay-database-failure-diagnostics.md | 43 - .../relay-improvement-checklist-2026-09.md | 192 - .../docs/relay-improvement-roadmap-2026-09.md | 67 - cloud/docs/relay-incident-monitor.md | 396 -- .../docs/relay-reconnect-2026-09-findings.md | 1032 ----- cloud/docs/relay-roll2-plan-2026-09.md | 159 - cloud/docs/relay-terraform-fencing-plan.md | 149 - cloud/docs/relay-workflows.md | 927 ----- .../staging-relay-deploy-identity-rollout.md | 177 - cloud/infra/terraform/.terraform.lock.hcl | 76 - cloud/infra/terraform/README.md | 409 -- cloud/infra/terraform/backend/production.hcl | 3 - cloud/infra/terraform/backend/staging.hcl | 3 - .../terraform/environments/production.tfvars | 533 --- .../terraform/environments/staging.tfvars | 139 - cloud/infra/terraform/outputs.tf | 216 -- .../terraform/push-dedicated-database.tf | 111 - cloud/infra/terraform/push-deploy-identity.tf | 82 - cloud/infra/terraform/push-gateway.tf | 305 -- cloud/infra/terraform/relay-asia-proof-iam.tf | 79 - .../terraform/relay-asia-topology-iam.tf | 207 - cloud/infra/terraform/relay-control-bucket.tf | 77 - cloud/infra/terraform/relay-database.tf | 88 - cloud/infra/terraform/relay-dns.tf | 47 - cloud/infra/terraform/relay-fence-broker.tf | 243 -- cloud/infra/terraform/relay-gce-cells.tf | 401 -- cloud/infra/terraform/relay-gce-foundation.tf | 212 -- .../terraform/relay-gce-startup.sh.tftpl | 139 - cloud/infra/terraform/relay-github-actions.tf | 682 ---- .../terraform/relay-github-workflow-trust.tf | 39 - cloud/infra/terraform/relay-observability.tf | 2082 ----------- cloud/infra/terraform/relay-shared.tf | 90 - .../terraform/relay-staging-deploy-iam.tf | 182 - cloud/infra/terraform/relay.tf | 684 ---- cloud/infra/terraform/variables.tf | 602 --- cloud/infra/terraform/versions.tf | 27 - cloud/package.json | 14 +- cloud/pnpm-lock.yaml | 25 - .../win32-test-lane-registration.test.mjs | 2 +- 279 files changed, 20 insertions(+), 87737 deletions(-) delete mode 100644 .github/workflows/cloud-bootstrap-relay-staging-capacity.yml delete mode 100644 .github/workflows/cloud-deploy-relay-asia-topology.yml delete mode 100644 .github/workflows/cloud-deploy-relay-fence-broker.yml delete mode 100644 .github/workflows/cloud-deploy-relay-production-capacity-job.yml delete mode 100644 .github/workflows/cloud-deploy-relay-production-capacity.yml delete mode 100644 .github/workflows/cloud-deploy-relay-production-director.yml delete mode 100644 .github/workflows/cloud-deploy-relay-production-multi-target.yml delete mode 100644 .github/workflows/cloud-deploy-relay-production-same-cap-job.yml delete mode 100644 .github/workflows/cloud-deploy-relay-production-same-cap.yml delete mode 100644 .github/workflows/cloud-deploy-relay-production.yml delete mode 100644 .github/workflows/cloud-deploy-relay-staging-gce-candidate.yml delete mode 100644 .github/workflows/cloud-deploy-relay-staging.yml delete mode 100644 .github/workflows/cloud-monitor-relay-clock-skew.yml delete mode 100644 .github/workflows/cloud-monitor-relay-production-job.yml delete mode 100644 .github/workflows/cloud-monitor-relay-production.yml delete mode 100644 .github/workflows/cloud-operate-relay-asia-admission.yml delete mode 100644 .github/workflows/cloud-operate-relay-production-cell-flags.yml delete mode 100644 .github/workflows/cloud-operate-relay-production-rehome-job.yml delete mode 100644 .github/workflows/cloud-operate-relay-production-rehome.yml delete mode 100644 .github/workflows/cloud-operate-relay-production-reserve-watch.yml delete mode 100644 .github/workflows/cloud-operate-relay-staging-cell-flags.yml delete mode 100644 .github/workflows/cloud-power-relay-staging.yml delete mode 100644 .github/workflows/cloud-prove-relay-asia-staging.yml delete mode 100644 .github/workflows/cloud-prove-relay-staging-capacity.yml delete mode 100644 .github/workflows/cloud-publish-relay-production.yml delete mode 100644 .github/workflows/cloud-push-deploy.yml delete mode 100644 .github/workflows/cloud-recover-relay-staging-c4-image.yml delete mode 100644 .github/workflows/cloud-requeue-relay-staging-c4-recovery.yml delete mode 100644 cloud/apps/push/src/push-deploy-preflight.test.ts delete mode 100644 cloud/apps/relay-fence-broker/Dockerfile delete mode 100644 cloud/apps/relay-ops/README.md delete mode 100644 cloud/apps/relay-ops/package.json delete mode 100644 cloud/apps/relay-ops/public/app.js delete mode 100644 cloud/apps/relay-ops/public/index.html delete mode 100644 cloud/apps/relay-ops/public/styles.css delete mode 100644 cloud/apps/relay-ops/src/cost-model.test.ts delete mode 100644 cloud/apps/relay-ops/src/cost-model.ts delete mode 100644 cloud/apps/relay-ops/src/dashboard-snapshot.test.ts delete mode 100644 cloud/apps/relay-ops/src/dashboard-snapshot.ts delete mode 100644 cloud/apps/relay-ops/src/environment-config.test.ts delete mode 100644 cloud/apps/relay-ops/src/environment-config.ts delete mode 100644 cloud/apps/relay-ops/src/gcloud-client.test.ts delete mode 100644 cloud/apps/relay-ops/src/gcloud-client.ts delete mode 100644 cloud/apps/relay-ops/src/github-runs.ts delete mode 100644 cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts delete mode 100644 cloud/apps/relay-ops/src/incident-live-preflight-cli.ts delete mode 100644 cloud/apps/relay-ops/src/incident-monitor-cli.test.ts delete mode 100644 cloud/apps/relay-ops/src/incident-monitor-cli.ts delete mode 100644 cloud/apps/relay-ops/src/incident-monitor-sources.test.ts delete mode 100644 cloud/apps/relay-ops/src/incident-monitor-sources.ts delete mode 100644 cloud/apps/relay-ops/src/incident-monitor.test.ts delete mode 100644 cloud/apps/relay-ops/src/incident-monitor.ts delete mode 100644 cloud/apps/relay-ops/src/incident-selector.ts delete mode 100644 cloud/apps/relay-ops/src/index.ts delete mode 100644 cloud/apps/relay-ops/src/monitoring-snapshot.test.ts delete mode 100644 cloud/apps/relay-ops/src/monitoring-snapshot.ts delete mode 100644 cloud/apps/relay-ops/src/pre-drain-hard-rule-sources.test.ts delete mode 100644 cloud/apps/relay-ops/src/pre-drain-hard-rule-sources.ts delete mode 100644 cloud/apps/relay-ops/src/pre-drain-sample-cli.test.ts delete mode 100644 cloud/apps/relay-ops/src/pre-drain-sample-cli.ts delete mode 100644 cloud/apps/relay-ops/src/pre-drain-sample.test.ts delete mode 100644 cloud/apps/relay-ops/src/pre-drain-sample.ts delete mode 100644 cloud/apps/relay-ops/src/relay-repository.test.ts delete mode 100644 cloud/apps/relay-ops/src/relay-repository.ts delete mode 100644 cloud/apps/relay-ops/src/resource-inventory.test.ts delete mode 100644 cloud/apps/relay-ops/src/resource-inventory.ts delete mode 100644 cloud/apps/relay-ops/src/staging-workflow.test.ts delete mode 100644 cloud/apps/relay-ops/src/staging-workflow.ts delete mode 100644 cloud/apps/relay-ops/tsconfig.build.json delete mode 100644 cloud/apps/relay-ops/tsconfig.json delete mode 100644 cloud/apps/relay/src/cell-connection-hard-cap-consistency.test.ts delete mode 100644 cloud/apps/relay/src/cell-startup-environment.test.ts delete mode 100644 cloud/apps/relay/src/regional-rehome-inventory-line-census.test.ts delete mode 100644 cloud/dev/contracts/production-cloud-sql-app-consumers.json delete mode 100644 cloud/dev/fixtures/relay-reserve-watch/2026-10-21-director-stall.json delete mode 100644 cloud/dev/fixtures/terraform-root-partition/families.json delete mode 100644 cloud/dev/scripts/capture-terraform-plan-baseline.mjs delete mode 100644 cloud/dev/scripts/capture-terraform-plan-baseline.test.mjs delete mode 100644 cloud/dev/scripts/check-relay-same-cap-headroom.mjs delete mode 100644 cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs delete mode 100644 cloud/dev/scripts/classify-relay-production-capacity-director.mjs delete mode 100644 cloud/dev/scripts/classify-relay-production-capacity-director.test.mjs delete mode 100644 cloud/dev/scripts/classify-relay-staging-bootstrap.mjs delete mode 100644 cloud/dev/scripts/classify-relay-staging-bootstrap.test.mjs delete mode 100644 cloud/dev/scripts/cloud-sql-rollout-lock-census.mjs delete mode 100644 cloud/dev/scripts/deploy-relay-blue-green.mjs delete mode 100644 cloud/dev/scripts/deploy-relay-blue-green.test.mjs delete mode 100644 cloud/dev/scripts/deploy-relay-gce-candidate.mjs delete mode 100644 cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs delete mode 100644 cloud/dev/scripts/deploy-relay-gce-multi-target.mjs delete mode 100644 cloud/dev/scripts/deploy-relay-gce-multi-target.test.mjs delete mode 100644 cloud/dev/scripts/drive-relay-director-deploy.mjs delete mode 100644 cloud/dev/scripts/drive-relay-director-deploy.test.mjs delete mode 100644 cloud/dev/scripts/github-smoke-token.mjs delete mode 100644 cloud/dev/scripts/github-smoke-token.test.mjs delete mode 100644 cloud/dev/scripts/infra.mjs delete mode 100644 cloud/dev/scripts/infra.test.mjs delete mode 100644 cloud/dev/scripts/load-relay-controls.mjs delete mode 100644 cloud/dev/scripts/measure-relay-drain-disconnect-gap.mjs delete mode 100644 cloud/dev/scripts/measure-relay-drain-disconnect-gap.test.mjs delete mode 100644 cloud/dev/scripts/operate-relay-asia-admission.mjs delete mode 100644 cloud/dev/scripts/operate-relay-asia-admission.test.mjs delete mode 100644 cloud/dev/scripts/operate-relay-cell-flags.mjs delete mode 100644 cloud/dev/scripts/operate-relay-cell-flags.test.mjs delete mode 100644 cloud/dev/scripts/operate-relay-regional-rehome.mjs delete mode 100644 cloud/dev/scripts/operate-relay-regional-rehome.test.mjs delete mode 100644 cloud/dev/scripts/power-staging-relay.mjs delete mode 100644 cloud/dev/scripts/power-staging-relay.test.mjs delete mode 100644 cloud/dev/scripts/prepare-relay-asia-director-cells.mjs delete mode 100644 cloud/dev/scripts/prepare-relay-asia-director-cells.test.mjs delete mode 100644 cloud/dev/scripts/prepare-relay-asia-topology-input.mjs delete mode 100644 cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs delete mode 100644 cloud/dev/scripts/prepare-relay-capacity-canary.mjs delete mode 100644 cloud/dev/scripts/prepare-relay-capacity-canary.test.mjs delete mode 100644 cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs delete mode 100644 cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs delete mode 100644 cloud/dev/scripts/probe-relay-legacy-admission.mjs delete mode 100644 cloud/dev/scripts/probe-relay-legacy-admission.test.mjs delete mode 100644 cloud/dev/scripts/probe-relay-rehome-trust.mjs delete mode 100644 cloud/dev/scripts/probe-relay-rehome-trust.test.mjs delete mode 100644 cloud/dev/scripts/production-cell-image-digest-consistency.test.mjs delete mode 100644 cloud/dev/scripts/production-cloud-sql-rollout-lock.test.mjs delete mode 100644 cloud/dev/scripts/push-cloud-run-model.mjs delete mode 100644 cloud/dev/scripts/push-gateway-recovery.test.mjs delete mode 100644 cloud/dev/scripts/push-gateway-workflow.test.mjs delete mode 100644 cloud/dev/scripts/read-relay-production-capacity-identity.mjs delete mode 100644 cloud/dev/scripts/read-relay-production-capacity-identity.test.mjs delete mode 100644 cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs delete mode 100644 cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs delete mode 100644 cloud/dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs delete mode 100644 cloud/dev/scripts/relay-admin-transient-retry.mjs delete mode 100644 cloud/dev/scripts/relay-admin-transient-retry.test.mjs delete mode 100644 cloud/dev/scripts/relay-admission-selector.mjs delete mode 100644 cloud/dev/scripts/relay-admission-selector.test.mjs delete mode 100644 cloud/dev/scripts/relay-asia-cloud-sql-metrics.mjs delete mode 100644 cloud/dev/scripts/relay-asia-rollout-evidence.mjs delete mode 100644 cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs delete mode 100644 cloud/dev/scripts/relay-asia-topology-workflow.test.mjs delete mode 100644 cloud/dev/scripts/relay-clock-skew-monitor-workflow.test.mjs delete mode 100644 cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs delete mode 100644 cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs delete mode 100644 cloud/dev/scripts/relay-control-flag-alerts.test.mjs delete mode 100644 cloud/dev/scripts/relay-db-fault-alerts.test.mjs delete mode 100644 cloud/dev/scripts/relay-deploy-workflow-argv.test.mjs delete mode 100644 cloud/dev/scripts/relay-director-deploy-plan.mjs delete mode 100644 cloud/dev/scripts/relay-evidence-code-provenance.mjs delete mode 100644 cloud/dev/scripts/relay-gce-terraform-fence.mjs delete mode 100644 cloud/dev/scripts/relay-gce-terraform-fence.test.mjs delete mode 100644 cloud/dev/scripts/relay-live-cell-image-overlay.mjs delete mode 100644 cloud/dev/scripts/relay-live-cell-image-overlay.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-connection-failure.mjs delete mode 100644 cloud/dev/scripts/relay-load-connection-failure.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-control-peer.mjs delete mode 100644 cloud/dev/scripts/relay-load-control-peer.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-director-capacity-gate.mjs delete mode 100644 cloud/dev/scripts/relay-load-director-capacity-gate.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-duplicate-assign.mjs delete mode 100644 cloud/dev/scripts/relay-load-duplicate-assign.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-model.mjs delete mode 100644 cloud/dev/scripts/relay-load-model.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-phase-barrier.mjs delete mode 100644 cloud/dev/scripts/relay-load-phase-barrier.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-placement-boundary.mjs delete mode 100644 cloud/dev/scripts/relay-load-placement-boundary.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-profile.mjs delete mode 100644 cloud/dev/scripts/relay-load-profile.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-reader-evidence.mjs delete mode 100644 cloud/dev/scripts/relay-load-reader-evidence.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-rebind-boundary.mjs delete mode 100644 cloud/dev/scripts/relay-load-rebind-boundary.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-region-behavior.mjs delete mode 100644 cloud/dev/scripts/relay-load-region-behavior.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-request-unit-boundary.mjs delete mode 100644 cloud/dev/scripts/relay-load-request-unit-boundary.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-resume-phone.mjs delete mode 100644 cloud/dev/scripts/relay-load-resume-phone.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-run-lifecycle.mjs delete mode 100644 cloud/dev/scripts/relay-load-run-lifecycle.test.mjs delete mode 100644 cloud/dev/scripts/relay-load-splice-retry.mjs delete mode 100644 cloud/dev/scripts/relay-load-splice-retry.test.mjs delete mode 100644 cloud/dev/scripts/relay-lock-contention-alerts.test.mjs delete mode 100644 cloud/dev/scripts/relay-monitor-evidence.mjs delete mode 100644 cloud/dev/scripts/relay-monitor-evidence.test.mjs delete mode 100644 cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json delete mode 100644 cloud/dev/scripts/relay-production-capacity-wave.mjs delete mode 100644 cloud/dev/scripts/relay-production-capacity-wave.test.mjs delete mode 100644 cloud/dev/scripts/relay-production-capacity-workflow.test.mjs delete mode 100644 cloud/dev/scripts/relay-production-identity-boundaries.test.mjs delete mode 100644 cloud/dev/scripts/relay-production-same-cap-wave.mjs delete mode 100644 cloud/dev/scripts/relay-production-same-cap-wave.test.mjs delete mode 100644 cloud/dev/scripts/relay-public-workflow-contract.test.mjs delete mode 100644 cloud/dev/scripts/relay-reconnect-herd-alerts.test.mjs delete mode 100644 cloud/dev/scripts/relay-recovery-wave-gate.mjs delete mode 100644 cloud/dev/scripts/relay-recovery-wave-gate.test.mjs delete mode 100644 cloud/dev/scripts/relay-region-hint-metrics.test.mjs delete mode 100644 cloud/dev/scripts/relay-region-observation-evidence.mjs delete mode 100644 cloud/dev/scripts/relay-region-observation-evidence.test.mjs delete mode 100644 cloud/dev/scripts/relay-rehome-aggregate-evidence.mjs delete mode 100644 cloud/dev/scripts/relay-rehome-aggregate-evidence.test.mjs delete mode 100644 cloud/dev/scripts/relay-repository.mjs delete mode 100644 cloud/dev/scripts/relay-repository.test.mjs delete mode 100644 cloud/dev/scripts/relay-reserve-cell-refusal.mjs delete mode 100644 cloud/dev/scripts/relay-reserve-dead-man-alert.test.mjs delete mode 100644 cloud/dev/scripts/relay-reserve-watch-rules.mjs delete mode 100644 cloud/dev/scripts/relay-same-cap-job-mode-conditions.test.mjs delete mode 100644 cloud/dev/scripts/relay-same-cap-script-census.test.mjs delete mode 100644 cloud/dev/scripts/relay-same-cap-shadow-gate-verdict.mjs delete mode 100644 cloud/dev/scripts/relay-same-cap-shadow-gate.mjs delete mode 100644 cloud/dev/scripts/relay-same-cap-shadow-gate.test.mjs delete mode 100644 cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs delete mode 100644 cloud/dev/scripts/relay-staging-capacity-identity.test.mjs delete mode 100644 cloud/dev/scripts/relay-staging-deploy-identity.test.mjs delete mode 100644 cloud/dev/scripts/render-workload-identity-conditions.mjs delete mode 100644 cloud/dev/scripts/run-relay-load-model.mjs delete mode 100644 cloud/dev/scripts/run-relay-recovery-wave-gate.mjs delete mode 100644 cloud/dev/scripts/sanitize-relay-asia-admission-result.mjs delete mode 100644 cloud/dev/scripts/sanitize-relay-asia-admission-result.test.mjs delete mode 100644 cloud/dev/scripts/smoke-relay.mjs delete mode 100644 cloud/dev/scripts/staging-relay-apply-guard.mjs delete mode 100644 cloud/dev/scripts/staging-relay-apply-guard.test.mjs delete mode 100644 cloud/dev/scripts/terraform-root-partition.mjs delete mode 100644 cloud/dev/scripts/terraform-root-partition.test.mjs delete mode 100644 cloud/dev/scripts/validate-relay-asia-topology-plan.mjs delete mode 100644 cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs delete mode 100644 cloud/dev/scripts/validate-relay-capacity-plan.mjs delete mode 100644 cloud/dev/scripts/validate-relay-capacity-plan.test.mjs delete mode 100644 cloud/dev/scripts/verify-relay-capacity-transition.mjs delete mode 100644 cloud/dev/scripts/verify-relay-capacity-transition.test.mjs delete mode 100644 cloud/dev/scripts/verify-relay-legacy-bootstrap.mjs delete mode 100644 cloud/dev/scripts/verify-relay-legacy-bootstrap.test.mjs delete mode 100644 cloud/dev/scripts/watch-relay-reserve-cell.mjs delete mode 100644 cloud/dev/scripts/watch-relay-reserve-cell.test.mjs delete mode 100644 cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs delete mode 100644 cloud/docs/orca-relay-capacity-testing.md delete mode 100644 cloud/docs/orca-relay-operations.md delete mode 100644 cloud/docs/push-database-cutover.md delete mode 100644 cloud/docs/push-gateway.md delete mode 100644 cloud/docs/relay-database-failure-diagnostics.md delete mode 100644 cloud/docs/relay-improvement-checklist-2026-09.md delete mode 100644 cloud/docs/relay-improvement-roadmap-2026-09.md delete mode 100644 cloud/docs/relay-incident-monitor.md delete mode 100644 cloud/docs/relay-reconnect-2026-09-findings.md delete mode 100644 cloud/docs/relay-roll2-plan-2026-09.md delete mode 100644 cloud/docs/relay-terraform-fencing-plan.md delete mode 100644 cloud/docs/relay-workflows.md delete mode 100644 cloud/docs/staging-relay-deploy-identity-rollout.md delete mode 100644 cloud/infra/terraform/.terraform.lock.hcl delete mode 100644 cloud/infra/terraform/README.md delete mode 100644 cloud/infra/terraform/backend/production.hcl delete mode 100644 cloud/infra/terraform/backend/staging.hcl delete mode 100644 cloud/infra/terraform/environments/production.tfvars delete mode 100644 cloud/infra/terraform/environments/staging.tfvars delete mode 100644 cloud/infra/terraform/outputs.tf delete mode 100644 cloud/infra/terraform/push-dedicated-database.tf delete mode 100644 cloud/infra/terraform/push-deploy-identity.tf delete mode 100644 cloud/infra/terraform/push-gateway.tf delete mode 100644 cloud/infra/terraform/relay-asia-proof-iam.tf delete mode 100644 cloud/infra/terraform/relay-asia-topology-iam.tf delete mode 100644 cloud/infra/terraform/relay-control-bucket.tf delete mode 100644 cloud/infra/terraform/relay-database.tf delete mode 100644 cloud/infra/terraform/relay-dns.tf delete mode 100644 cloud/infra/terraform/relay-fence-broker.tf delete mode 100644 cloud/infra/terraform/relay-gce-cells.tf delete mode 100644 cloud/infra/terraform/relay-gce-foundation.tf delete mode 100644 cloud/infra/terraform/relay-gce-startup.sh.tftpl delete mode 100644 cloud/infra/terraform/relay-github-actions.tf delete mode 100644 cloud/infra/terraform/relay-github-workflow-trust.tf delete mode 100644 cloud/infra/terraform/relay-observability.tf delete mode 100644 cloud/infra/terraform/relay-shared.tf delete mode 100644 cloud/infra/terraform/relay-staging-deploy-iam.tf delete mode 100644 cloud/infra/terraform/relay.tf delete mode 100644 cloud/infra/terraform/variables.tf delete mode 100644 cloud/infra/terraform/versions.tf diff --git a/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml b/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml deleted file mode 100644 index 38e0efb9acc..00000000000 --- a/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml +++ /dev/null @@ -1,678 +0,0 @@ -name: Bootstrap Relay Staging Capacity - -on: - workflow_dispatch: - inputs: - confirmation: - description: Enter BOOTSTRAP_STAGING_CAPACITY - required: true - type: string - -permissions: - contents: read - id-token: write - -concurrency: - group: relay-staging-mutation - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - bootstrap: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - environment: staging - env: - GCP_PROJECT_ID: onorca-cloud-staging - GCP_REGION: us-central1 - DIRECTOR_ORIGIN: https://relay-staging.onorca.dev - CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - LEGACY_C3_IMAGE_DIGEST: sha256:2d0f6e6db2b0eb9d6aba188698de8330f8c30b4e76badfcf0fac3f3eb9508a87 - steps: - - uses: actions/checkout@v4 - - - id: deploy-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - id: capacity-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - token_format: access_token - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-staging-terraform-state - object: terraform/state/cloud-sql-rollout/staging.lock - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_wrapper: false - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - name: Require explicit bootstrap confirmation - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: test "${CONFIRMATION}" = "BOOTSTRAP_STAGING_CAPACITY" - - - name: Read and verify reviewed 600/60 topology - shell: bash - run: | - node dev/scripts/infra.mjs init --env staging - terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \ - > "${RUNNER_TEMP}/relay-gce-state.json" - DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars \ - <<< 'local.relay_director_cells_json' | jq -r '.')" - DESIRED_IMAGES_JSON="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars \ - <<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.image })' \ - | jq -r '.')" - DESIRED_ZONES_JSON="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars \ - <<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.zone })' \ - | jq -r '.')" - DESIRED_TARGET_SIZES_JSON="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars \ - <<< 'jsonencode(local.relay_gce_cell_target_sizes)' | jq -r '.')" - jq -e \ - --argjson images "${DESIRED_IMAGES_JSON}" \ - --argjson target_sizes "${DESIRED_TARGET_SIZES_JSON}" \ - '([.[] | select(.id == "staging-gce-c2")] | length == 1) and - ([.[] | select(.id == "staging-gce-c3")] | length == 1) and - (any(.[]; .id == "staging-gce-c2" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and - (any(.[]; .id == "staging-gce-c3" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and - ($images["staging-gce-c2"] == $images["staging-gce-c3"]) and - ($target_sizes["staging-gce-c2"] == 1) and - ($target_sizes["staging-gce-c3"] == 1)' \ - <<< "${DESIRED_CELLS_JSON}" >/dev/null - jq \ - --argjson desired "${DESIRED_CELLS_JSON}" \ - --argjson images "${DESIRED_IMAGES_JSON}" \ - --argjson zones "${DESIRED_ZONES_JSON}" \ - '($desired | map({key: .id, value: .}) | from_entries) as $cells | - to_entries | map(. as $entry | { - key: $entry.key, - value: ($entry.value + { - origin: $cells[$entry.key].url, - image: $images[$entry.key], - zone: $zones[$entry.key], - connection_hard_cap: $cells[$entry.key].connectionHardCap, - connection_unobserved_bound: $cells[$entry.key].connectionUnobservedBound - }) - }) | from_entries' \ - "${RUNNER_TEMP}/relay-gce-state.json" \ - > "${RUNNER_TEMP}/relay-gce-topology.json" - ACTIVE_REVISION="$(gcloud run services describe orca-cloud-relay-staging \ - --project "${GCP_PROJECT_ID}" \ - --region us-central1 \ - --format=json \ - | jq -r ' - [.status.traffic[] | select((.percent // 0) > 0)] | - if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" - test -n "${ACTIVE_REVISION}" - DESIRED_IMAGE="$(jq -r '.["staging-gce-c2"]' <<< "${DESIRED_IMAGES_JSON}")" - gcloud run revisions describe "${ACTIVE_REVISION}" \ - --project "${GCP_PROJECT_ID}" \ - --region us-central1 \ - --format=json \ - | jq -e \ - --arg image "${DESIRED_IMAGE}" \ - --arg capacity_service_account "${CAPACITY_SERVICE_ACCOUNT}" \ - '(.spec.containers[0].image == $image) and - any(.spec.containers[0].env[]?; - .name == "ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT" and - .value == $capacity_service_account)' >/dev/null - CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --format=json \ - | jq -cer ' - [.spec.containers[0].env[]? | - select(.name == "ORCA_RELAY_CELLS_JSON") | .value] | - if length == 1 then .[0] | fromjson else error("missing director topology") end')" - jq -e --argjson desired "${DESIRED_CELLS_JSON}" ' - def without_bootstrap_capacity: - map(if .id == "staging-gce-c2" or .id == "staging-gce-c3" - then del(.connectionHardCap, .connectionUnobservedBound) - else . end); - without_bootstrap_capacity == ($desired | without_bootstrap_capacity) - ' <<< "${CURRENT_CELLS_JSON}" >/dev/null - - - name: Bootstrap C2 then C3 - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - - topology="${RUNNER_TEMP}/relay-gce-topology.json" - - fixed_one_instance_name() { - local cell_id="$1" - local mig_name zone - mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")" - zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")" - gcloud compute instance-groups managed list-instances \ - "${mig_name}" \ - --project "${GCP_PROJECT_ID}" \ - --zone "${zone}" \ - --format=json \ - | jq -er ' - if length == 1 and .[0].instanceStatus == "RUNNING" and - .[0].currentAction == "NONE" - then .[0].instance | split("/") | last - else error("legacy cell does not have one stable running instance") end' - } - - fixed_one_instance_id() { - local cell_id="$1" - local instance_name="$2" - local zone - zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")" - gcloud compute instances describe "${instance_name}" \ - --project "${GCP_PROJECT_ID}" \ - --zone "${zone}" \ - --format='value(id)' - } - - write_legacy_metrics() { - local cell_id="$1" - local instance_id="$2" - local after="$3" - local output="$4" - gcloud logging read \ - "resource.type=\"gce_instance\" AND - resource.labels.instance_id=\"${instance_id}\" AND - jsonPayload.event=\"orca_relay_runtime_metrics\" AND - jsonPayload.cellId=\"${cell_id}\" AND - timestamp>=\"${after}\"" \ - --project "${GCP_PROJECT_ID}" \ - --limit 10 \ - --order desc \ - --format json \ - | jq '[.[] | { - timestamp, - cellId: .jsonPayload.cellId, - metricVersion: .jsonPayload.metricVersion, - totalConnections: .jsonPayload.totalConnections, - preAuthConnections: .jsonPayload.preAuthConnections, - controls: .jsonPayload.controls, - splices: .jsonPayload.splices, - pendingSplices: .jsonPayload.pendingSplices, - queuedBytes: .jsonPayload.queuedBytes - }]' > "${output}" - } - - verify_legacy_cell() { - local cell_id="$1" - local admission="$2" - local after="$3" - local expected_instance_id="$4" - local runtime_started_after="${5:-}" - local previous_incarnation_digest="${6:-}" - local hard_cap="${7:-}" - local unobserved_bound="${8:-}" - local capacity_state="${9:-}" - local current_instance current_instance_id metrics origin result - local runtime_start_args=() incarnation_args=() capacity_args=() - current_instance="$(fixed_one_instance_name "${cell_id}")" - current_instance_id="$(fixed_one_instance_id "${cell_id}" "${current_instance}")" - test "${current_instance_id}" = "${expected_instance_id}" - metrics="${RUNNER_TEMP}/${cell_id}-legacy-runtime-metrics.json" - origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")" - if test -n "${runtime_started_after}"; then - runtime_start_args=(--runtime-started-after "${runtime_started_after}") - fi - if test -n "${previous_incarnation_digest}"; then - incarnation_args=(--previous-incarnation-digest "${previous_incarnation_digest}") - fi - if test -n "${hard_cap}"; then - capacity_args=(--hard-cap "${hard_cap}" --unobserved-bound "${unobserved_bound}") - fi - if test -n "${capacity_state}"; then - capacity_args+=(--capacity-state "${capacity_state}") - fi - for _attempt in $(seq 1 18); do - write_legacy_metrics \ - "${cell_id}" "${current_instance_id}" "${after}" "${metrics}" - if result="$(node dev/scripts/verify-relay-legacy-bootstrap.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${origin}" \ - --cell-id "${cell_id}" \ - --admission "${admission}" \ - --expected-image-digest "${LEGACY_C3_IMAGE_DIGEST}" \ - --metrics-after "${after}" \ - --metrics-file "${metrics}" \ - "${runtime_start_args[@]}" \ - "${incarnation_args[@]}" \ - "${capacity_args[@]}")"; then - echo "${result}" - return 0 - fi - sleep 10 - done - return 1 - } - - post_admin() { - local origin="$1" - local path="$2" - local body="$3" - curl --fail --silent --show-error \ - --request POST \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "${body}" \ - "${origin}${path}" - } - - runtime_kind() { - local cell_id="$1" - local origin desired_digest digest - origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")" - desired_digest="$(jq -r --arg cell "${cell_id}" \ - '.[$cell].image | split("@") | last' "${topology}")" - digest="$(post_admin "${origin}" /v1/admin/runtime-status '{"v":1}' \ - | jq -er '.imageDigest')" - if test "${digest}" = "${LEGACY_C3_IMAGE_DIGEST}"; then - echo legacy - elif test "${digest}" = "${desired_digest}"; then - echo modern - else - echo 'bootstrap cell image is neither legacy nor reviewed' >&2 - return 1 - fi - } - - cell_admission() { - local cell_id="$1" - post_admin "${DIRECTOR_ORIGIN}" /v1/admin/cell-status \ - "$(jq -cn --arg cell "${cell_id}" '{v:1, cellId:$cell}')" \ - | jq -er '.status.admissionState | - if . == "general" or . == "migration-only" then . - else error("bootstrap admission is not recoverable") end' - } - - verify_modern_cell() { - local cell_id="$1" - local admission="$2" - local origin - origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")" - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${origin}" \ - --cell-id "${cell_id}" \ - --hard-cap 600 \ - --unobserved-bound 60 \ - --heartbeat fresh \ - --admission "${admission}" \ - --draining forbidden \ - --activity allowed - } - - ensure_modern_general() { - local cell_id="$1" - local admission="$2" - verify_modern_cell "${cell_id}" "${admission}" - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-id "${cell_id}" \ - --mode restore \ - --general-cell-ids "${cell_id}" - verify_modern_cell "${cell_id}" general - } - - prepare_legacy_c3_fallback() { - legacy_c3_metrics_boundary="$(node -e \ - 'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')" - legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)" - legacy_c3_instance_id="$(fixed_one_instance_id \ - staging-gce-c3 "${legacy_c3_instance}")" - verify_legacy_cell \ - staging-gce-c3 general \ - "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" - node dev/scripts/probe-relay-legacy-admission.mjs \ - --cell-origin https://c3.relay-staging.onorca.dev - legacy_c3_restart_started_after= - legacy_c3_old_incarnation= - } - - normalize_legacy_c3() { - legacy_pre_boundary="$(node -e \ - 'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')" - legacy_c2_instance="$(fixed_one_instance_name staging-gce-c2)" - legacy_c2_instance_id="$(fixed_one_instance_id \ - staging-gce-c2 "${legacy_c2_instance}")" - verify_legacy_cell \ - staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}" - node dev/scripts/probe-relay-legacy-admission.mjs \ - --cell-origin https://c2.relay-staging.onorca.dev - - legacy_c3_isolated=false - restore_legacy_c3_fallback() { - if test "${legacy_c3_isolated}" = true; then - verify_legacy_cell \ - staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}" - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-id staging-gce-c3 \ - --mode restore-fallback \ - --general-cell-ids staging-gce-c2 - fi - } - - trap restore_legacy_c3_fallback EXIT - legacy_c3_isolated=true - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin https://c3.relay-staging.onorca.dev \ - --cell-id staging-gce-c3 \ - --mode isolate - legacy_c3_drain_boundary="$(node -e \ - 'process.stdout.write(new Date().toISOString())')" - legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)" - legacy_c3_instance_id="$(fixed_one_instance_id \ - staging-gce-c3 "${legacy_c3_instance}")" - legacy_c3_drained="$(verify_legacy_cell \ - staging-gce-c3 migration-only \ - "${legacy_c3_drain_boundary}" "${legacy_c3_instance_id}")" - echo "${legacy_c3_drained}" - legacy_c3_old_incarnation="$(jq -er '.incarnationDigest' \ - <<< "${legacy_c3_drained}")" - legacy_c3_restart_started_after="$(node -e \ - 'process.stdout.write(new Date().toISOString())')" - gcloud compute instance-groups managed recreate-instances \ - orca-cloud-staging-relay-gce-c3 \ - --instances "${legacy_c3_instance}" \ - --project "${GCP_PROJECT_ID}" \ - --zone us-central1-a \ - --quiet - gcloud compute instance-groups managed wait-until \ - orca-cloud-staging-relay-gce-c3 \ - --stable \ - --project "${GCP_PROJECT_ID}" \ - --zone us-central1-a \ - --timeout 900 - legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)" - legacy_c3_instance_id="$(fixed_one_instance_id \ - staging-gce-c3 "${legacy_c3_instance}")" - legacy_c3_metrics_boundary="$(node -e \ - 'process.stdout.write(new Date().toISOString())')" - verify_legacy_cell \ - staging-gce-c3 migration-only \ - "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \ - "${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}" - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-id staging-gce-c3 \ - --mode restore \ - --general-cell-ids staging-gce-c2,staging-gce-c3 - verify_legacy_cell \ - staging-gce-c3 general \ - "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \ - "${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}" - legacy_c3_isolated=false - trap - EXIT - } - - roll_cell() ( - local cell_id="$1" - local fallback_cell_id="$2" - local target_kind="$3" - local fallback_kind="$4" - local active_revision cell_origin current_cells_json desired_bound desired_cap - local desired_cells_json director_result image mig_name plan - local fallback_origin plan_changes plan_result restored target_drain_boundary - local target_instance target_instance_id zone - cell_origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")" - fallback_origin="$(jq -r --arg cell "${fallback_cell_id}" \ - '.[$cell].origin' "${topology}")" - zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")" - mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")" - image="$(jq -r --arg cell "${cell_id}" '.[$cell].image' "${topology}")" - desired_cap="$(jq -r --arg cell "${cell_id}" \ - '.[$cell].connection_hard_cap' "${topology}")" - desired_bound="$(jq -r --arg cell "${cell_id}" \ - '.[$cell].connection_unobserved_bound' "${topology}")" - plan="${RUNNER_TEMP}/${cell_id}-capacity-bootstrap.tfplan" - restored=false - - restore_fallback() { - if test "${restored}" = false; then - verify_fallback - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-id "${cell_id}" \ - --mode restore-fallback \ - --general-cell-ids "${fallback_cell_id}" - fi - } - - verify_fallback() { - if test "${fallback_kind}" = legacy; then - verify_legacy_cell \ - "${fallback_cell_id}" general \ - "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \ - "${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}" - return - fi - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${fallback_origin}" \ - --cell-id "${fallback_cell_id}" \ - --hard-cap 600 \ - --unobserved-bound 60 \ - --heartbeat fresh \ - --admission general \ - --draining forbidden \ - --activity allowed - } - - verify_fallback - trap restore_fallback EXIT - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${cell_origin}" \ - --cell-id "${cell_id}" \ - --mode isolate - target_drain_boundary="$(node -e \ - 'process.stdout.write(new Date().toISOString())')" - if test "${target_kind}" = legacy; then - target_instance="$(fixed_one_instance_name "${cell_id}")" - target_instance_id="$(fixed_one_instance_id "${cell_id}" "${target_instance}")" - verify_legacy_cell \ - "${cell_id}" migration-only \ - "${target_drain_boundary}" "${target_instance_id}" \ - '' '' "${desired_cap}" "${desired_bound}" absent-or-stale - else - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${cell_origin}" \ - --cell-id "${cell_id}" \ - --heartbeat either \ - --admission migration-only \ - --draining required \ - --activity quiescent - fi - - active_revision="$(gcloud run services describe orca-cloud-relay-staging \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --format=json \ - | jq -r ' - [.status.traffic[] | select((.percent // 0) > 0)] | - if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" - test -n "${active_revision}" - current_cells_json="$(gcloud run revisions describe "${active_revision}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --format=json \ - | jq -cer ' - [.spec.containers[0].env[]? | - select(.name == "ORCA_RELAY_CELLS_JSON") | .value] | - if length == 1 then .[0] | fromjson else error("missing director topology") end')" - desired_cells_json="$(jq -ce \ - --arg cell "${cell_id}" \ - --argjson cap "${desired_cap}" \ - --argjson bound "${desired_bound}" \ - 'map(if .id == $cell then . + { - connectionHardCap: $cap, - connectionUnobservedBound: $bound - } else . end)' <<< "${current_cells_json}")" - director_result="$(node dev/scripts/deploy-relay-blue-green.mjs \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --service orca-cloud-relay-staging \ - --image "${image}" \ - --role director \ - --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \ - --capacity-cell-id "${cell_id}" \ - --director-cells-json "${desired_cells_json}" \ - --prune-revisions true \ - --release-id "bootstrap-${cell_id}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}")" - echo "${director_result}" - if test "${target_kind}" = legacy; then - verify_legacy_cell \ - "${cell_id}" migration-only \ - "${target_drain_boundary}" "${target_instance_id}" \ - '' '' "${desired_cap}" "${desired_bound}" - else - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${cell_origin}" \ - --cell-id "${cell_id}" \ - --hard-cap "${desired_cap}" \ - --unobserved-bound "${desired_bound}" \ - --heartbeat either \ - --admission migration-only \ - --draining required \ - --activity quiescent - fi - - terraform -chdir=infra/terraform plan \ - -var-file=environments/staging.tfvars \ - "-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \ - "-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \ - -out="${plan}" - plan_result="$(terraform -chdir=infra/terraform show -json "${plan}" \ - | node dev/scripts/validate-relay-capacity-plan.mjs \ - --mode bootstrap-cell \ - --cell-id "${cell_id}" \ - --hard-cap 600 \ - --unobserved-bound 60 \ - --image "${image}" \ - --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")" - echo "${plan_result}" - plan_changes="$(jq -r '.changes' <<< "${plan_result}")" - [[ "${plan_changes}" =~ ^(0|2)$ ]] - if test "${plan_changes}" = 2; then - terraform -chdir=infra/terraform apply -auto-approve "${plan}" - else - target_instance="$(fixed_one_instance_name "${cell_id}")" - gcloud compute instance-groups managed recreate-instances "${mig_name}" \ - --instances "${target_instance}" \ - --project "${GCP_PROJECT_ID}" \ - --zone "${zone}" \ - --quiet - fi - gcloud compute instance-groups managed wait-until "${mig_name}" \ - --stable \ - --project "${GCP_PROJECT_ID}" \ - --zone "${zone}" \ - --timeout 900 - - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${cell_origin}" \ - --cell-id "${cell_id}" \ - --hard-cap 600 \ - --unobserved-bound 60 \ - --heartbeat fresh \ - --admission migration-only \ - --draining forbidden \ - --activity allowed - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-id "${cell_id}" \ - --mode restore \ - --general-cell-ids staging-gce-c2,staging-gce-c3 - restored=true - trap - EXIT - ) - - c2_kind="$(runtime_kind staging-gce-c2)" - c3_kind="$(runtime_kind staging-gce-c3)" - c2_admission="$(cell_admission staging-gce-c2)" - c3_admission="$(cell_admission staging-gce-c3)" - bootstrap_phase="$(node dev/scripts/classify-relay-staging-bootstrap.mjs \ - --c2-kind "${c2_kind}" \ - --c2-admission "${c2_admission}" \ - --c3-kind "${c3_kind}" \ - --c3-admission "${c3_admission}")" - jq -cn --arg phase "${bootstrap_phase}" \ - '{event:"relay_staging_bootstrap_phase", phase:$phase}' - - case "${bootstrap_phase}" in - normalize-and-roll-both) - normalize_legacy_c3 - roll_cell staging-gce-c2 staging-gce-c3 legacy legacy - roll_cell staging-gce-c3 staging-gce-c2 legacy modern - ;; - resume-c2-then-c3) - prepare_legacy_c3_fallback - roll_cell staging-gce-c2 staging-gce-c3 legacy legacy - roll_cell staging-gce-c3 staging-gce-c2 legacy modern - ;; - roll-c2) - ensure_modern_general staging-gce-c3 "${c3_admission}" - roll_cell staging-gce-c2 staging-gce-c3 legacy modern - ;; - roll-c3) - ensure_modern_general staging-gce-c2 "${c2_admission}" - roll_cell staging-gce-c3 staging-gce-c2 legacy modern - ;; - complete) - ensure_modern_general staging-gce-c2 "${c2_admission}" - ensure_modern_general staging-gce-c3 "${c3_admission}" - ;; - *) - echo 'unsupported staging bootstrap phase' >&2 - exit 1 - ;; - esac - - - name: Verify both bootstrapped cells - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - for number in 2 3; do - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "https://c${number}.relay-staging.onorca.dev" \ - --cell-id "staging-gce-c${number}" \ - --hard-cap 600 \ - --unobserved-bound 60 \ - --heartbeat fresh \ - --admission general \ - --draining forbidden \ - --activity allowed - done diff --git a/.github/workflows/cloud-deploy-relay-asia-topology.yml b/.github/workflows/cloud-deploy-relay-asia-topology.yml deleted file mode 100644 index e7fde8f8514..00000000000 --- a/.github/workflows/cloud-deploy-relay-asia-topology.yml +++ /dev/null @@ -1,288 +0,0 @@ -name: Deploy Relay Asia Topology - -on: - workflow_dispatch: - inputs: - environment: - description: Target Relay environment - required: true - type: choice - options: [staging, production] - mode: - description: Validate a saved plan or apply that exact plan - required: true - default: plan - type: choice - options: [plan, apply] - cell-ids: - description: Exact reviewed comma-separated cell set (Asia, or the US cells at the Asia shape) - required: true - type: string - image: - description: Full environment Relay image pinned by sha256 digest - required: true - type: string - confirmation: - description: Enter APPLY_RELAY_ASIA_TOPOLOGY for apply mode - required: false - type: string - -permissions: - contents: read - id-token: write - -concurrency: - group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }} - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - topology: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 30 - environment: ${{ inputs.environment }} - env: - DEPLOY_MODE: ${{ inputs.mode }} - TARGET_ENVIRONMENT: ${{ inputs.environment }} - TARGET_CELL_IDS: ${{ inputs.cell-ids }} - TARGET_IMAGE: ${{ inputs.image }} - GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }} - CLOUD_SQL_INSTANCE: ${{ inputs.environment == 'production' && 'orca-cloud-auth-db' || 'orca-cloud-staging-auth-db' }} - VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360 - VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17 - # SHOW max_connections on the live instance, 2026-09-16; no flag is set. - VERIFIED_DEFAULT_MAX_CONNECTIONS: '500' - TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }} - TF_VARS: ${{ inputs.environment == 'production' && 'environments/production.tfvars' || 'environments/staging.tfvars' }} - TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }} - TOPOLOGY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT }} - steps: - - uses: actions/checkout@v4 - - - name: Validate the reviewed request before authentication - shell: bash - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - set -euo pipefail - test -n "${TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER}" - test -n "${TOPOLOGY_SERVICE_ACCOUNT}" - target_region=asia-east2 - case "${TARGET_ENVIRONMENT}:${TARGET_CELL_IDS}" in - staging:staging-gce-c4) ;; - production:production-gce-c27,production-gce-c28,production-gce-c29) ;; - production:production-gce-c30) ;; - production:production-gce-c31) ;; - production:production-gce-c32,production-gce-c33) target_region=us-central1 ;; - production:production-gce-c34) ;; - production:production-gce-c35) ;; - *) echo "cell-ids do not match the reviewed environment topology" >&2; exit 1 ;; - esac - echo "TARGET_REGION=${target_region}" >> "${GITHUB_ENV}" - [[ "${TARGET_IMAGE}" =~ ^us-central1-docker\.pkg\.dev/${GCP_PROJECT_ID}/orca-cloud/relay@sha256:[0-9a-f]{64}$ ]] - if test "${DEPLOY_MODE}" = apply; then - test "${CONFIRMATION}" = APPLY_RELAY_ASIA_TOPOLOGY - else - test "${DEPLOY_MODE}" = plan - test -z "${CONFIRMATION}" - fi - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.17.0 - terraform_wrapper: false - - - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ env.TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ env.TOPOLOGY_SERVICE_ACCOUNT }} - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }} - object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }} - - - name: Require the checked Cloud SQL connection budget - shell: bash - run: | - set -euo pipefail - budget="$(node dev/scripts/relay-cloud-sql-connection-budget.mjs)" - checked_max="$(jq -er '.maxConnections' <<< "${budget}")" - jq -e '.withinBudget == true' <<< "${budget}" >/dev/null - if test "${TARGET_ENVIRONMENT}" = production; then - instance="$(gcloud sql instances describe "${CLOUD_SQL_INSTANCE}" \ - --project "${GCP_PROJECT_ID}" --format=json)" - live_flag="$(jq -er '[.settings.databaseFlags[]? | - select(.name == "max_connections") | .value] | - if length <= 1 then (.[0] // "") else error("duplicate max_connections flags") end' \ - <<< "${instance}")" - if test -n "${live_flag}"; then - live_max="${live_flag}" - live_source=explicit-flag - else - # No flag: the ceiling is the tier default measured for this exact - # shape; fail closed if the shape changes. - test "$(jq -er '.settings.tier' <<< "${instance}")" = \ - "${VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER}" - test "$(jq -er '.databaseVersion' <<< "${instance}")" = \ - "${VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION}" - live_max="${VERIFIED_DEFAULT_MAX_CONNECTIONS}" - live_source=verified-shape-default - fi - test "${live_max}" = "${checked_max}" - else - live_max="not-read-for-staging" - live_source=not-read-for-staging - fi - { - echo "### Relay Cloud SQL connection budget" - echo "- Checked maximum: ${checked_max}" - echo "- Configured maximum: $(jq -er '.configuredMaximum' <<< "${budget}")" - echo "- Rollout operating maximum: $(jq -er '.operatingMaximum' <<< "${budget}")" - echo "- Explicit reserve: $(jq -er '.explicitReserve' <<< "${budget}")" - echo "- Production live max_connections: ${live_max}" - echo "- Production live maximum source: ${live_source}" - } >> "${GITHUB_STEP_SUMMARY}" - - - name: Initialize the exact environment state - run: terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}" - - - id: targets - name: Build the exact additive target set - shell: bash - run: | - set -euo pipefail - file="${RUNNER_TEMP}/relay-asia-targets" - : > "${file}" - # The root region's network predates every cell; only asia-east2's is additive. - if test "${TARGET_REGION}" = asia-east2; then - printf '%s\n' \ - '-target=google_compute_subnetwork.relay_gce_additional["asia-east2"]' \ - '-target=google_compute_router.relay_gce_additional["asia-east2"]' \ - '-target=google_compute_router_nat.relay_gce_additional["asia-east2"]' >> "${file}" - fi - printf '%s\n' '-target=google_compute_url_map.relay_gce[0]' >> "${file}" - IFS=, read -ra cells <<< "${TARGET_CELL_IDS}" - for cell_id in "${cells[@]}"; do - printf '%s\n' \ - "-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \ - "-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \ - "-target=google_compute_backend_service.relay_gce_cell[\"${cell_id}\"]" >> "${file}" - done - echo "file=${file}" >> "${GITHUB_OUTPUT}" - - - id: live-images - name: Plan every non-target cell at the image it serves - shell: bash - run: | - set -euo pipefail - # The URL map target pulls every cell's template into the plan, and same-cap rolls - # leave committed images behind the served ones; read only templates out of state. - cells="${RUNNER_TEMP}/relay-committed-cells.json" - live="${RUNNER_TEMP}/relay-live-cell-templates.json" - overlay="${RUNNER_TEMP}/relay-asia-live-images.tfvars.json" - committed_plan="${RUNNER_TEMP}/relay-committed-cells.tfplan" - mapfile -t targets < "${{ steps.targets.outputs.file }}" - # Not console: it evaluates every output against state, where a new cell has no MIG yet. - terraform -chdir=infra/terraform plan -input=false -refresh=false -lock=false \ - -var-file="${TF_VARS}" "${targets[@]}" -out="${committed_plan}" > /dev/null - terraform -chdir=infra/terraform show -json "${committed_plan}" \ - | jq -ce '.variables.relay_gce_cells.value | objects' > "${cells}" - terraform -chdir=infra/terraform show -json | jq -ce '[ - .values.root_module.resources[]? - | select(.mode == "managed" and - .type == "google_compute_instance_template" and .name == "relay_gce_cell") - | { index, metadata_startup_script: .values.metadata_startup_script } - ]' > "${live}" - summary="$(node dev/scripts/relay-live-cell-image-overlay.mjs \ - --cells-json "${cells}" --live-templates-json "${live}" \ - --cell-ids "${TARGET_CELL_IDS}" --output "${overlay}")" - echo "file=${overlay}" >> "${GITHUB_OUTPUT}" - { - echo "### Live images held for non-target cells" - echo "- Cells whose committed image differs from the served one: $(jq -r '.drifted | join(", ")' <<< "${summary}")" - } >> "${GITHUB_STEP_SUMMARY}" - - - name: Create and validate the saved topology plan - id: plan - shell: bash - run: | - set -euo pipefail - plan="${RUNNER_TEMP}/relay-asia-topology.tfplan" - plan_json="${RUNNER_TEMP}/relay-asia-topology.json" - mapfile -t targets < "${{ steps.targets.outputs.file }}" - terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \ - -var-file="${TF_VARS}" -var-file="${{ steps.live-images.outputs.file }}" \ - "${targets[@]}" -out="${plan}" - terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}" - committed="${RUNNER_TEMP}/relay-committed-asia-topology.json" - jq -e '{ - relay_gce_cells: .variables.relay_gce_cells.value, - relay_gce_additional_region_subnetwork_cidrs: - .variables.relay_gce_additional_region_subnetwork_cidrs.value - }' "${plan_json}" > "${committed}" - node dev/scripts/prepare-relay-asia-topology-input.mjs \ - --existing-json "${committed}" \ - --environment "${TARGET_ENVIRONMENT}" \ - --cell-ids "${TARGET_CELL_IDS}" \ - --image "${TARGET_IMAGE}" - result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \ - --plan-json "${plan_json}" \ - --environment "${TARGET_ENVIRONMENT}" \ - --cell-ids "${TARGET_CELL_IDS}" \ - --region "${TARGET_REGION}" \ - --image "${TARGET_IMAGE}")" - changes="$(jq -er '.changes' <<< "${result}")" - digest="$(sha256sum "${plan}" | awk '{print $1}')" - echo "plan=${plan}" >> "${GITHUB_OUTPUT}" - echo "changes=${changes}" >> "${GITHUB_OUTPUT}" - { - echo "### Relay Asia topology saved plan" - echo "- Environment: ${TARGET_ENVIRONMENT}" - echo "- Cells: ${TARGET_CELL_IDS}" - echo "- Region: ${TARGET_REGION}" - echo "- Mutating resources: ${changes}" - echo "- Saved-plan SHA-256: ${digest}" - } >> "${GITHUB_STEP_SUMMARY}" - - - name: Apply only the validated saved plan - if: ${{ inputs.mode == 'apply' }} - run: terraform -chdir=infra/terraform apply -input=false -auto-approve "${{ steps.plan.outputs.plan }}" - - - name: Prove the exact topology targets converged - if: ${{ inputs.mode == 'apply' }} - shell: bash - run: | - set -euo pipefail - mapfile -t targets < "${{ steps.targets.outputs.file }}" - plan="${RUNNER_TEMP}/relay-asia-topology-readback.tfplan" - plan_json="${RUNNER_TEMP}/relay-asia-topology-readback.json" - terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \ - -var-file="${TF_VARS}" -var-file="${{ steps.live-images.outputs.file }}" \ - "${targets[@]}" -out="${plan}" - terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}" - result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \ - --plan-json "${plan_json}" \ - --environment "${TARGET_ENVIRONMENT}" \ - --cell-ids "${TARGET_CELL_IDS}" \ - --region "${TARGET_REGION}" \ - --image "${TARGET_IMAGE}")" - test "$(jq -er '.changes' <<< "${result}")" = 0 - - - name: Record the required selector-safe next step - if: ${{ inputs.mode == 'apply' }} - run: | - { - echo "### Required next step" - echo "The VMs are not eligible for ordinary placement yet." - echo "Register the exact new cells atomically as migration-only before any director configuration lists them." - echo "Rollback is migration-only admission; do not destroy the Asia network on rollout day." - } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/cloud-deploy-relay-fence-broker.yml b/.github/workflows/cloud-deploy-relay-fence-broker.yml deleted file mode 100644 index 5c120ecb8c9..00000000000 --- a/.github/workflows/cloud-deploy-relay-fence-broker.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Deploy Relay Fence Broker - -on: - workflow_dispatch: - inputs: - image-digest: - description: Immutable broker image digest built from this main commit - required: true - type: string - -permissions: - contents: read - id-token: write - -concurrency: - group: production-cloud-sql-rollout - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - deploy: - if: >- - ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && - github.ref == 'refs/heads/main' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - environment: production - env: - GCP_PROJECT_ID: onorca-cloud - GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} - SERVICE_NAME: orca-cloud-relay-fence - IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay-fence-broker - IMAGE_DIGEST: ${{ inputs.image-digest }} - steps: - - uses: actions/checkout@v4 - - - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-terraform-state - object: terraform/state/cloud-sql-rollout/production.lock - - - name: Resolve exact-commit broker image - run: | - [[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] - IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}" - SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \ - --format='value(image_summary.digest)')" - test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}" - TAGS="$(gcloud artifacts docker tags list "${IMAGE_REPOSITORY}" \ - --filter="version:${IMAGE_DIGEST}" \ - --format=json)" - jq -e --arg tag "/tags/sha-${GITHUB_SHA}" \ - 'any(.[]; .tag | endswith($tag))' <<< "${TAGS}" - echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}" - - - name: Deploy broker image only - run: | - gcloud run services update "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --image "${IMAGE}" \ - --quiet - - - name: Verify ready singleton revision - run: | - SERVICE="$(gcloud run services describe "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --format=json)" - jq -e '.status.conditions[] | select(.type == "Ready" and .status == "True")' \ - <<< "${SERVICE}" - REVISION="$(jq -r \ - '[.status.traffic[] | select((.percent // 0) == 100)] | - if length == 1 then .[0].revisionName // empty else empty end' \ - <<< "${SERVICE}")" - test -n "${REVISION}" - SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --format='value(spec.containers[0].image)')" - test "${SERVED_IMAGE}" = "${IMAGE}" diff --git a/.github/workflows/cloud-deploy-relay-production-capacity-job.yml b/.github/workflows/cloud-deploy-relay-production-capacity-job.yml deleted file mode 100644 index ec5ef0177fc..00000000000 --- a/.github/workflows/cloud-deploy-relay-production-capacity-job.yml +++ /dev/null @@ -1,825 +0,0 @@ -name: Deploy Relay Production Capacity Job - -on: - workflow_call: - inputs: - mode: - required: true - type: string - target-cell-id: - required: true - type: string - confirmation: - required: true - type: string - monitor-run-id: - required: true - type: string - monitor-run-attempt: - required: true - type: string - evidence-mode: - required: true - type: string - wave-cell-ids: - required: true - type: string - wave-index: - required: true - type: string - source-wave-run-id: - required: true - type: string - -permissions: - actions: read - contents: read - id-token: write - -defaults: - run: - working-directory: cloud - -jobs: - capacity: - if: ${{ github.ref == 'refs/heads/main' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 75 - environment: production - env: - # These drains are unpaced, but restart-safe still waits out the production pace window - # from the drain, the same one the same-cap drain uses. - RESTART_SAFE_PACE_WINDOW_MS: '300000' - GCP_PROJECT_ID: onorca-cloud - GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} - DIRECTOR_SERVICE_NAME: orca-cloud-relay - DIRECTOR_ORIGIN: https://relay.onorca.dev - TARGET_CELL_ID: ${{ inputs.target-cell-id }} - CAPACITY_CELL_IDS: production-gce-c7,production-gce-c8,production-gce-c9,production-gce-c10,production-gce-c13,production-gce-c14,production-gce-c15,production-gce-c16,production-gce-c19,production-gce-c20,production-gce-c21,production-gce-c22,production-gce-c23,production-gce-c24,production-gce-c25,production-gce-c26 - PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d - COMPATIBLE_DIRECTOR_IMAGE_DIGEST: sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73 - COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f - CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - DEPLOY_MODE: ${{ inputs.mode }} - EVIDENCE_MODE: ${{ inputs.evidence-mode }} - MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} - MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} - WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }} - WAVE_INDEX: ${{ inputs.wave-index }} - SOURCE_WAVE_RUN_ID: ${{ inputs.source-wave-run-id }} - steps: - - name: Require exact reusable-workflow invocation - working-directory: . - run: | - [[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]] - if test "${EVIDENCE_MODE}" = continuation; then - test "${DEPLOY_MODE}" = apply - [[ "${WAVE_INDEX}" =~ ^[0-3]$ ]] - test "${WAVE_CELL_IDS}" != none - test "${SOURCE_WAVE_RUN_ID}" = none - elif test "${EVIDENCE_MODE}" = resume; then - test "${DEPLOY_MODE}" = apply - test "${WAVE_INDEX}" = resume - test "${WAVE_CELL_IDS}" != none - [[ "${SOURCE_WAVE_RUN_ID}" =~ ^[0-9]+$ ]] - else - test "${EVIDENCE_MODE}" = single - test "${WAVE_CELL_IDS}" = none - test "${WAVE_INDEX}" = 0 - test "${SOURCE_WAVE_RUN_ID}" = none - fi - - - name: Require production workflow configuration - working-directory: . - env: - DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - CAPACITY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - run: | - test -n "${GCP_REGION}" - test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}" - test -n "${DEPLOY_SERVICE_ACCOUNT}" - test -n "${CAPACITY_WORKLOAD_IDENTITY_PROVIDER}" - test -n "${CAPACITY_SERVICE_ACCOUNT}" - - - uses: actions/checkout@v4 - - - id: resume-provenance - if: ${{ inputs.evidence-mode == 'resume' }} - env: - GH_TOKEN: ${{ github.token }} - run: | - test "${MONITOR_RUN_ATTEMPT}" = 1 - case "${MONITOR_RUN_ID}:${SOURCE_WAVE_RUN_ID}:${WAVE_CELL_IDS}:${TARGET_CELL_ID}" in - 31554591366:31555510376:production-gce-c16,production-gce-c15,production-gce-c14,production-gce-c13:production-gce-c13) - EXPECTED_SHA=a917e8e1fc1a2654e8cb81ba39b57733ec56be9c - EXPECTED_SOURCE_ATTEMPT=1 - ;; - 31562760783:31563664692:production-gce-c10,production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c10) - EXPECTED_SHA=6082e9ca89a918ca51f0c87db003f5e8805b64b7 - EXPECTED_SOURCE_ATTEMPT=1 - ;; - 31571019947:31572080665:production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c8) - EXPECTED_SHA=e59958130c9d9b7a6cd805df2678d08997842c7c - EXPECTED_SOURCE_ATTEMPT=2 - ;; - *) exit 1 ;; - esac - MONITOR_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${MONITOR_RUN_ID}" \ - --jq 'select(.name == "Monitor Relay Production" and - .path == ".github/workflows/cloud-monitor-relay-production.yml" and - .head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and - .event == "workflow_dispatch" and .conclusion == "success" and .run_attempt == 1) | - .head_sha')" - SOURCE_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \ - --jq 'select(.name == "Deploy Relay Production Capacity" and - .path == ".github/workflows/cloud-deploy-relay-production-capacity.yml" and - .head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and - .event == "workflow_dispatch" and .conclusion == "failure") | - .head_sha')" - SOURCE_ATTEMPT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \ - --jq '.run_attempt')" - [[ "${MONITOR_SHA}" =~ ^[0-9a-f]{40}$ ]] - test "${MONITOR_SHA}" = "${EXPECTED_SHA}" - test "${SOURCE_SHA}" = "${MONITOR_SHA}" - test "${SOURCE_ATTEMPT}" = "${EXPECTED_SOURCE_ATTEMPT}" - echo "commit-sha=${MONITOR_SHA}" >> "${GITHUB_OUTPUT}" - - - name: Require fresh dry-run evidence reference - if: ${{ inputs.mode == 'apply' }} - run: | - [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]] - [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] - - - name: Download private dry-run evidence - if: ${{ inputs.mode == 'apply' }} - uses: actions/download-artifact@v4 - with: - name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ runner.temp }}/relay-monitor-evidence - github-token: ${{ github.token }} - run-id: ${{ inputs.monitor-run-id }} - - - uses: ./.github/actions/setup-pnpm - with: - package-json-file: cloud/package.json - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - run: pnpm install --frozen-lockfile - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_wrapper: false - - - name: Verify dry-run artifact before cloud authentication - if: ${{ inputs.mode == 'apply' }} - run: | - EVIDENCE_COMMIT_SHA="${GITHUB_SHA}" - if test "${EVIDENCE_MODE:-single}" = resume; then - EVIDENCE_COMMIT_SHA="${{ steps.resume-provenance.outputs.commit-sha }}" - fi - node dev/scripts/relay-monitor-evidence.mjs verify-restore \ - --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ - --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ - --run-id "${MONITOR_RUN_ID}" \ - --run-attempt "${MONITOR_RUN_ATTEMPT}" \ - --commit-sha "${EVIDENCE_COMMIT_SHA}" \ - --mode dry-run - - - name: Reject previously consumed dry-run evidence - if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }} - env: - GH_TOKEN: ${{ github.token }} - run: | - MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - COUNT="$(gh api \ - "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \ - --jq '.total_count')" - test "${COUNT}" = "0" - - - name: Download this workflow's wave authority - if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }} - uses: actions/download-artifact@v4 - with: - name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ runner.temp }}/relay-wave-authority - github-token: ${{ github.token }} - run-id: ${{ github.run_id }} - - - name: Download the failed wave authority for resume - if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }} - uses: actions/download-artifact@v4 - with: - name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ runner.temp }}/relay-wave-authority - github-token: ${{ github.token }} - run-id: ${{ inputs.source-wave-run-id }} - - - name: Require wave evidence consumed by this workflow - if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }} - run: | - MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${GITHUB_RUN_ID}" - - - name: Require wave evidence consumed by the failed source workflow - if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }} - run: | - MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${SOURCE_WAVE_RUN_ID}" - - - id: deploy-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-terraform-state - object: terraform/state/cloud-sql-rollout/production.lock - release: 'false' - - - name: Require exact mutation confirmation - if: ${{ inputs.mode != 'verify' }} - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - if test "${EVIDENCE_MODE:-single}" = resume; then - test "${CONFIRMATION}" = "RESUME_SELECTED_CELL_TO_1000 ${TARGET_CELL_ID}" - elif test "${DEPLOY_MODE}" = apply; then - test "${CONFIRMATION}" = "RAISE_SELECTED_CELL_TO_1000" - else - test "${CONFIRMATION}" = "ROLL_BACK_SELECTED_CELL_TO_600 ${TARGET_CELL_ID}" - fi - - - name: Initialize the exact production backend - run: node dev/scripts/infra.mjs init --env production - - - name: Build the exact selected-cell configuration - shell: bash - run: | - if test "${DEPLOY_MODE}" = rollback; then - TARGET_HARD_CAP=600 - else - TARGET_HARD_CAP=1000 - fi - TARGET_UNOBSERVED_BOUND=60 - TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}" - [[ "${TARGET_HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]] - CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev" - CELLS_JSON="$(terraform -chdir=infra/terraform console \ - -var-file=environments/production.tfvars \ - <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')" - OVERRIDE_CELLS_JSON="$(jq -ce \ - --arg cell "${TARGET_CELL_ID}" \ - --argjson cap "${TARGET_HARD_CAP}" \ - --argjson bound "${TARGET_UNOBSERVED_BOUND}" \ - '.[$cell].connection_hard_cap = $cap | - .[$cell].connection_unobserved_bound = $bound' \ - <<< "${CELLS_JSON}")" - jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \ - '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-capacity.tfvars.json" - BASE_CELLS_JSON="$(terraform -chdir=infra/terraform console \ - -var-file=environments/production.tfvars \ - <<< 'local.relay_director_cells_json' | jq -er '.')" - IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image" - ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone" - DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \ - -var-file=environments/production.tfvars \ - -var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \ - <<< "${IMAGE_EXPRESSION}" | jq -r '.')" - TARGET_ZONE="$(terraform -chdir=infra/terraform console \ - -var-file=environments/production.tfvars \ - -var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \ - <<< "${ZONE_EXPRESSION}" | jq -r '.')" - MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \ - | jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')" - jq -e --arg cell "${TARGET_CELL_ID}" \ - 'any(.[]; .id == $cell and .connectionHardCap == 1000 and - .connectionUnobservedBound == 60)' \ - <<< "${BASE_CELLS_JSON}" >/dev/null - [[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]] - DESIRED_IMAGE_DIGEST="${DESIRED_IMAGE##*@}" - [[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]] - test "${MIG_NAME}" = "orca-cloud-relay-gce-${TARGET_HOSTNAME}" - { - echo "CELL_ORIGIN=${CELL_ORIGIN}" - echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}" - echo "TARGET_HARD_CAP=${TARGET_HARD_CAP}" - echo "TARGET_UNOBSERVED_BOUND=${TARGET_UNOBSERVED_BOUND}" - echo "DESIRED_IMAGE=${DESIRED_IMAGE}" - echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}" - echo "TARGET_ZONE=${TARGET_ZONE}" - echo "MIG_NAME=${MIG_NAME}" - echo "BASE_CELLS_JSON=${BASE_CELLS_JSON}" - } >> "${GITHUB_ENV}" - - - name: Require the exact compatible production image and topology - shell: bash - run: | - SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" - ACTIVE_REVISION="$(jq -r \ - '[.status.traffic[] | select((.percent // 0) > 0)] | - if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \ - <<< "${SERVICE_JSON}")" - test -n "${ACTIVE_REVISION}" - ACTIVE_REVISION_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" - ACTIVE_IMAGE="$(jq -er '.spec.containers[0].image' <<< "${ACTIVE_REVISION_JSON}")" - ACTIVE_IMAGE_DIGEST="${ACTIVE_IMAGE##*@}" - if test "${ACTIVE_IMAGE}" != "${DESIRED_IMAGE}"; then - test "${ACTIVE_IMAGE_DIGEST}" = "${COMPATIBLE_DIRECTOR_IMAGE_DIGEST}" - test "${DESIRED_IMAGE_DIGEST}" = "${COMPATIBLE_CELL_IMAGE_DIGEST}" - fi - CURRENT_CELLS_JSON="$(jq -cer '[.spec.containers[0].env[]? | - select(.name == "ORCA_RELAY_CELLS_JSON") | .value] | - if length == 1 then .[0] | fromjson else error("missing director topology") end' \ - <<< "${ACTIVE_REVISION_JSON}")" - CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON="$( - node dev/scripts/read-relay-production-capacity-identity.mjs \ - <<< "${ACTIVE_REVISION_JSON}" - )" - CLASSIFICATION="$(jq -nc \ - --argjson baseCells "${BASE_CELLS_JSON}" \ - --argjson currentCells "${CURRENT_CELLS_JSON}" \ - --arg capacityCellIds "${CAPACITY_CELL_IDS}" \ - --arg targetCellId "${TARGET_CELL_ID}" \ - --argjson targetHardCap "${TARGET_HARD_CAP}" \ - --argjson currentCapacityServiceAccount \ - "${CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON}" \ - '{baseCells:$baseCells, currentCells:$currentCells, - capacityCellIds:($capacityCellIds | split(",")), - targetCellId:$targetCellId, targetHardCap:$targetHardCap, - currentCapacityServiceAccount:$currentCapacityServiceAccount}' \ - | node dev/scripts/classify-relay-production-capacity-director.mjs \ - --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")" - TOPOLOGY_PHASE="$(jq -er '.topologyPhase' <<< "${CLASSIFICATION}")" - DESIRED_CELLS_JSON="$(jq -cer '.desiredCells' <<< "${CLASSIFICATION}")" - DIRECTOR_READY="$(jq -er \ - 'if (.directorReady | type) == "boolean" then - (.directorReady | tostring) - else error("invalid directorReady classification") end' \ - <<< "${CLASSIFICATION}")" - { - echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}" - echo "TOPOLOGY_PHASE=${TOPOLOGY_PHASE}" - echo "DIRECTOR_READY=${DIRECTOR_READY}" - echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" - } >> "${GITHUB_ENV}" - - - name: Require the exact wave predecessor topology - if: ${{ inputs.mode == 'apply' && (inputs.evidence-mode == 'continuation' || inputs.evidence-mode == 'resume') }} - run: test "${TOPOLOGY_PHASE}" = predecessor - - - name: Verify fresh dry-run evidence against the live selector - if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - node dev/scripts/relay-monitor-evidence.mjs verify-mutation \ - --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ - --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ - --run-id "${MONITOR_RUN_ID}" \ - --run-attempt "${MONITOR_RUN_ATTEMPT}" \ - --commit-sha "${GITHUB_SHA}" \ - --mode dry-run \ - --mutation-mode capacity-transition \ - --source-cell-id "${TARGET_CELL_ID}" \ - --director-origin "${DIRECTOR_ORIGIN}" - - - name: Recheck every live safety signal - if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - pnpm incident:relay-preflight -- \ - --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" - - - name: Recheck exact wave state and every live safety signal - if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - node dev/scripts/relay-production-capacity-wave.mjs build-preflight \ - --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \ - --wave-cell-ids "${WAVE_CELL_IDS}" \ - --wave-index "${WAVE_INDEX}" \ - --target-cell-id "${TARGET_CELL_ID}" \ - --output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" - RETRY_ARGS=() - if test "${WAVE_INDEX}" != 0; then RETRY_ARGS=(--retry-freshness); fi - pnpm incident:relay-preflight -- \ - --state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \ - "${RETRY_ARGS[@]}" - - - name: Recheck exact isolated resume state and every live safety signal - if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - node dev/scripts/relay-production-capacity-wave.mjs build-resume-preflight \ - --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \ - --wave-cell-ids "${WAVE_CELL_IDS}" \ - --target-cell-id "${TARGET_CELL_ID}" \ - --output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" - pnpm incident:relay-preflight -- \ - --state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \ - --retry-freshness - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap 600 \ - --unobserved-bound 60 \ - --heartbeat fresh \ - --admission migration-only \ - --draining required \ - --activity allowed \ - --runtime required \ - --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" - - - name: Consume the single-use dry-run evidence - if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }} - run: | - MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption" - printf '%s\n' "${GITHUB_RUN_ID}" \ - > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}" - - - name: Publish the consumed-evidence marker - if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }} - uses: actions/upload-artifact@v4 - with: - name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - retention-days: 90 - if-no-files-found: error - - - name: Verify current selected-cell capacity - if: ${{ inputs.mode == 'verify' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - CURRENT_CAP="${TARGET_HARD_CAP}" - CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}" - if test "${TOPOLOGY_PHASE}" = predecessor; then - CURRENT_CAP=600 - CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" - fi - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${CURRENT_CAP}" \ - --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \ - --heartbeat fresh \ - --admission general \ - --draining forbidden \ - --activity allowed \ - --expected-image-digests "${CURRENT_IMAGE_DIGEST}" - - - name: Arm fail-closed mutation cleanup - if: ${{ inputs.mode != 'verify' }} - run: echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" - - - name: Reversibly isolate only the selected cell - if: ${{ inputs.mode != 'verify' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - test "${MUTATION_STARTED:-false}" = true || exit 0 - node dev/scripts/prepare-relay-production-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --mode isolate - - - name: Drain the selected cell or prove an offline rollback - if: ${{ inputs.mode != 'verify' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - if node dev/scripts/prepare-relay-production-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --mode drain; then - echo "OFFLINE_ROLLBACK=false" >> "${GITHUB_ENV}" - elif test "${DEPLOY_MODE}" = rollback; then - echo "OFFLINE_ROLLBACK=true" >> "${GITHUB_ENV}" - else - exit 1 - fi - - - id: restart-auth-one - if: ${{ inputs.mode != 'verify' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - id: restart-gate-one - name: Require restart-safe selected-cell activity - if: ${{ inputs.mode != 'verify' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-one.outputs.id_token }} - run: | - if test "${OFFLINE_ROLLBACK:-false}" = true; then - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --heartbeat stale \ - --admission migration-only \ - --draining either \ - --activity restart-safe \ - --runtime unavailable - echo "settled=true" >> "${GITHUB_OUTPUT}" - exit 0 - fi - CURRENT_CAP=600 - if test "${TOPOLOGY_PHASE}" = desired; then - CURRENT_CAP="${TARGET_HARD_CAP}" - elif test "${TARGET_HARD_CAP}" = 600; then - CURRENT_CAP=1000 - fi - GATE_LOG="${RUNNER_TEMP}/relay-capacity-restart-gate-one.log" - set +e - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${CURRENT_CAP}" \ - --unobserved-bound 60 \ - --heartbeat either \ - --admission migration-only \ - --draining required \ - --activity restart-safe \ - --runtime required \ - --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \ - --timeout-ms 450000 \ - --expected-image-digests \ - "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" \ - 2> "${GATE_LOG}" - GATE_EXIT=$? - set -e - cat "${GATE_LOG}" >&2 - if test "${GATE_EXIT}" = 0; then - echo "settled=true" >> "${GITHUB_OUTPUT}" - exit 0 - fi - if test "$(wc -l < "${GATE_LOG}" | tr -d ' ')" = 1 && - grep -Eq '^capacity transition verification timed out: \{.*\}$' "${GATE_LOG}"; then - echo "settled=false" >> "${GITHUB_OUTPUT}" - exit 0 - fi - exit "${GATE_EXIT}" - - - id: restart-auth-two - if: ${{ steps.restart-gate-one.outputs.settled == 'false' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Require extended restart-safe selected-cell activity - if: ${{ steps.restart-gate-one.outputs.settled == 'false' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-two.outputs.id_token }} - run: | - CURRENT_CAP=600 - if test "${TOPOLOGY_PHASE}" = desired; then - CURRENT_CAP="${TARGET_HARD_CAP}" - elif test "${TARGET_HARD_CAP}" = 600; then - CURRENT_CAP=1000 - fi - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${CURRENT_CAP}" \ - --unobserved-bound 60 \ - --heartbeat either \ - --admission migration-only \ - --draining required \ - --activity restart-safe \ - --runtime required \ - --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \ - --timeout-ms 450000 \ - --expected-image-digests \ - "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" - - - name: Deploy only the reviewed director topology - if: ${{ inputs.mode != 'verify' }} - run: | - if test "${DIRECTOR_READY}" = true; then exit 0; fi - RELEASE_ID="capacity-${TARGET_HOSTNAME}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" - node dev/scripts/deploy-relay-blue-green.mjs \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --service "${DIRECTOR_SERVICE_NAME}" \ - --image "${ACTIVE_IMAGE}" \ - --role director \ - --max-instances 5 \ - --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \ - --capacity-cell-id "${TARGET_CELL_ID}" \ - --director-cells-json "${DESIRED_CELLS_JSON}" \ - --min-instances 5 \ - --prune-revisions false \ - --release-id "${RELEASE_ID}" - - - id: director-transition-auth - if: ${{ inputs.mode != 'verify' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Require fail-closed director transition - if: ${{ inputs.mode != 'verify' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.director-transition-auth.outputs.id_token }} - run: | - if test "${OFFLINE_ROLLBACK:-false}" = true; then - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --heartbeat stale \ - --admission migration-only \ - --draining either \ - --activity restart-safe \ - --runtime unavailable - exit 0 - fi - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${TARGET_HARD_CAP}" \ - --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \ - --heartbeat either \ - --admission migration-only \ - --draining required \ - --activity restart-safe \ - --runtime required \ - --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \ - --timeout-ms 480000 \ - --expected-image-digests \ - "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" - - - id: capacity-auth - if: ${{ inputs.mode != 'verify' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Plan and apply only the empty selected cell - if: ${{ inputs.mode != 'verify' }} - shell: bash - run: | - terraform -chdir=infra/terraform plan \ - -var-file=environments/production.tfvars \ - -var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \ - "-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ - "-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ - -out="${RUNNER_TEMP}/relay-capacity-cell.tfplan" - PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \ - "${RUNNER_TEMP}/relay-capacity-cell.tfplan" \ - | node dev/scripts/validate-relay-capacity-plan.mjs \ - --mode bootstrap-cell \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${TARGET_HARD_CAP}" \ - --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \ - --image "${DESIRED_IMAGE}" \ - --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")" - echo "${PLAN_RESULT}" - PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")" - [[ "${PLAN_CHANGES}" =~ ^(0|1|2)$ ]] - if test "${PLAN_CHANGES}" != 0; then - terraform -chdir=infra/terraform apply \ - -auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan" - else - INSTANCE="$(gcloud compute instance-groups managed list-instances \ - "${MIG_NAME}" --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \ - --format=json | jq -er 'if length == 1 and - .[0].instanceStatus == "RUNNING" and .[0].currentAction == "NONE" - then .[0].instance | split("/") | last - else error("selected cell is not one stable running instance") end')" - gcloud compute instance-groups managed recreate-instances \ - "${MIG_NAME}" --instances "${INSTANCE}" \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --quiet - fi - gcloud compute instance-groups managed wait-until \ - "${MIG_NAME}" --stable --project "${GCP_PROJECT_ID}" \ - --zone "${TARGET_ZONE}" --timeout 900 - - - id: capacity-transition-auth - if: ${{ inputs.mode != 'verify' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Verify fresh exact selected-cell heartbeat before admission - if: ${{ inputs.mode != 'verify' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }} - run: | - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${TARGET_HARD_CAP}" \ - --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \ - --heartbeat fresh \ - --admission migration-only \ - --draining forbidden \ - --activity allowed \ - --expected-image-digests "${DESIRED_IMAGE_DIGEST}" - - - name: Restore only the selected cell to general admission - if: ${{ inputs.mode != 'verify' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }} - run: | - node dev/scripts/prepare-relay-production-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --mode activate - - - name: Verify the live general selected cell - if: ${{ inputs.mode != 'verify' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }} - run: | - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${TARGET_HARD_CAP}" \ - --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \ - --heartbeat fresh \ - --admission general \ - --draining forbidden \ - --activity allowed \ - --expected-image-digests "${DESIRED_IMAGE_DIGEST}" - - - id: cleanup-auth - if: ${{ failure() && inputs.mode != 'verify' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Keep the selected cell isolated after a failed mutation - if: ${{ failure() && inputs.mode != 'verify' }} - continue-on-error: true - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }} - run: | - test "${MUTATION_STARTED:-false}" = true || exit 0 - CLEANUP_STATUS=0 - node dev/scripts/prepare-relay-production-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --mode isolate || CLEANUP_STATUS=$? - node dev/scripts/prepare-relay-production-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --mode drain || CLEANUP_STATUS=$? - exit "${CLEANUP_STATUS}" diff --git a/.github/workflows/cloud-deploy-relay-production-capacity.yml b/.github/workflows/cloud-deploy-relay-production-capacity.yml deleted file mode 100644 index 4ec6dc65040..00000000000 --- a/.github/workflows/cloud-deploy-relay-production-capacity.yml +++ /dev/null @@ -1,356 +0,0 @@ -name: Deploy Relay Production Capacity - -on: - workflow_dispatch: - inputs: - mode: - description: Verify, change one cell, or raise a sequential wave - required: true - default: verify - type: choice - options: - - verify - - apply - - rollback - - wave-apply - - wave-resume - target-cell-id: - description: Exact serving cell for verify, apply, or rollback - required: true - default: production-gce-c26 - type: choice - options: - - production-gce-c7 - - production-gce-c8 - - production-gce-c9 - - production-gce-c10 - - production-gce-c13 - - production-gce-c14 - - production-gce-c15 - - production-gce-c16 - - production-gce-c19 - - production-gce-c20 - - production-gce-c21 - - production-gce-c22 - - production-gce-c23 - - production-gce-c24 - - production-gce-c25 - - production-gce-c26 - wave-cell-ids: - description: Ordered comma-separated wave of two to four serving cells - required: false - default: none - type: string - confirmation: - description: Enter the exact single-cell or wave confirmation - required: false - type: string - monitor-run-id: - description: Successful fresh dry-run monitor workflow run ID for apply - required: false - type: string - monitor-run-attempt: - description: Exact dry-run monitor workflow attempt for apply - required: false - type: string - source-wave-run-id: - description: Failed wave run that isolated the resume target - required: false - type: string - -permissions: - actions: read - contents: read - id-token: write - -concurrency: - group: production-cloud-sql-rollout - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - single_cell: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode != 'wave-apply' && inputs.mode != 'wave-resume') }} - uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml - with: - mode: ${{ inputs.mode }} - target-cell-id: ${{ inputs.target-cell-id }} - confirmation: ${{ inputs.confirmation }} - monitor-run-id: ${{ inputs.monitor-run-id }} - monitor-run-attempt: ${{ inputs.monitor-run-attempt }} - evidence-mode: single - wave-cell-ids: none - wave-index: '0' - source-wave-run-id: none - secrets: inherit - - resume_cell: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-resume' && github.ref == 'refs/heads/main') }} - uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml - with: - mode: apply - target-cell-id: ${{ inputs.target-cell-id }} - confirmation: ${{ inputs.confirmation }} - monitor-run-id: ${{ inputs.monitor-run-id }} - monitor-run-attempt: ${{ inputs.monitor-run-attempt }} - evidence-mode: resume - wave-cell-ids: ${{ inputs.wave-cell-ids }} - wave-index: resume - source-wave-run-id: ${{ inputs.source-wave-run-id }} - secrets: inherit - - wave_gate: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-apply' && github.ref == 'refs/heads/main') }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 30 - environment: production - outputs: - cells: ${{ steps.wave.outputs.cells }} - env: - DIRECTOR_ORIGIN: https://relay.onorca.dev - MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} - MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} - OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence - PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d - COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f - WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }} - steps: - - name: Require production workflow configuration - working-directory: . - env: - DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - run: | - test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}" - test -n "${DEPLOY_SERVICE_ACCOUNT}" - - - uses: actions/checkout@v4 - - - uses: ./.github/actions/setup-pnpm - with: - package-json-file: cloud/package.json - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - run: pnpm install --frozen-lockfile - - - id: wave - name: Validate the exact wave request - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - CELLS="$(node dev/scripts/relay-production-capacity-wave.mjs validate \ - --wave-cell-ids "${WAVE_CELL_IDS}" \ - --confirmation "${CONFIRMATION}")" - echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}" - - - name: Require fresh dry-run evidence reference - run: | - [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]] - [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] - - - name: Download private dry-run evidence - uses: actions/download-artifact@v4 - with: - name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ github.workspace }}/relay-monitor-evidence - github-token: ${{ github.token }} - run-id: ${{ inputs.monitor-run-id }} - - - name: Verify dry-run artifact before cloud authentication - run: | - node dev/scripts/relay-monitor-evidence.mjs verify-restore \ - --directory "${OUTPUT_DIRECTORY}" \ - --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ - --run-id "${MONITOR_RUN_ID}" \ - --run-attempt "${MONITOR_RUN_ATTEMPT}" \ - --commit-sha "${GITHUB_SHA}" \ - --mode dry-run - - - name: Reject previously consumed dry-run evidence - env: - GH_TOKEN: ${{ github.token }} - run: | - MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - COUNT="$(gh api \ - "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \ - --jq '.total_count')" - test "${COUNT}" = "0" - - - id: deploy-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-terraform-state - object: terraform/state/cloud-sql-rollout/production.lock - release: 'false' - - - name: Verify wave evidence against the live selector - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - FIRST_CELL="$(jq -er '.[0]' <<< '${{ steps.wave.outputs.cells }}')" - node dev/scripts/relay-monitor-evidence.mjs verify-mutation \ - --directory "${OUTPUT_DIRECTORY}" \ - --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ - --run-id "${MONITOR_RUN_ID}" \ - --run-attempt "${MONITOR_RUN_ATTEMPT}" \ - --commit-sha "${GITHUB_SHA}" \ - --mode dry-run \ - --mutation-mode capacity-transition \ - --source-cell-id "${FIRST_CELL}" \ - --director-origin "${DIRECTOR_ORIGIN}" - - - name: Recheck every live safety signal - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - pnpm incident:relay-preflight -- \ - --state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" - - - name: Require exact 600/60 predecessor wave cells - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - while read -r CELL_ID; do - HOSTNAME="${CELL_ID#production-gce-}" - [[ "${HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]] - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "https://${HOSTNAME}.relay.onorca.dev" \ - --cell-id "${CELL_ID}" \ - --hard-cap 600 \ - --unobserved-bound 60 \ - --heartbeat fresh \ - --admission general \ - --draining forbidden \ - --activity allowed \ - --expected-image-digests \ - "${PREDECESSOR_IMAGE_DIGEST},${COMPATIBLE_CELL_IMAGE_DIGEST}" - done < <(jq -r '.[]' <<< '${{ steps.wave.outputs.cells }}') - - - name: Consume the single-use dry-run evidence - run: | - MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption" - printf '%s\n' "${GITHUB_RUN_ID}" \ - > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}" - - - name: Publish the consumed-evidence marker - uses: actions/upload-artifact@v4 - with: - name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - retention-days: 90 - if-no-files-found: error - - wave_cell_1: - needs: wave_gate - uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml - with: - mode: apply - target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[0] }} - confirmation: RAISE_SELECTED_CELL_TO_1000 - monitor-run-id: ${{ inputs.monitor-run-id }} - monitor-run-attempt: ${{ inputs.monitor-run-attempt }} - evidence-mode: continuation - wave-cell-ids: ${{ inputs.wave-cell-ids }} - wave-index: '0' - source-wave-run-id: none - secrets: inherit - - wave_cell_2: - needs: [wave_gate, wave_cell_1] - uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml - with: - mode: apply - target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[1] }} - confirmation: RAISE_SELECTED_CELL_TO_1000 - monitor-run-id: ${{ inputs.monitor-run-id }} - monitor-run-attempt: ${{ inputs.monitor-run-attempt }} - evidence-mode: continuation - wave-cell-ids: ${{ inputs.wave-cell-ids }} - wave-index: '1' - source-wave-run-id: none - secrets: inherit - - wave_cell_3: - if: ${{ needs.wave_cell_2.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[2] != null }} - needs: [wave_gate, wave_cell_2] - uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml - with: - mode: apply - target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[2] }} - confirmation: RAISE_SELECTED_CELL_TO_1000 - monitor-run-id: ${{ inputs.monitor-run-id }} - monitor-run-attempt: ${{ inputs.monitor-run-attempt }} - evidence-mode: continuation - wave-cell-ids: ${{ inputs.wave-cell-ids }} - wave-index: '2' - source-wave-run-id: none - secrets: inherit - - wave_cell_4: - if: ${{ needs.wave_cell_3.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[3] != null }} - needs: [wave_gate, wave_cell_3] - uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml - with: - mode: apply - target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[3] }} - confirmation: RAISE_SELECTED_CELL_TO_1000 - monitor-run-id: ${{ inputs.monitor-run-id }} - monitor-run-attempt: ${{ inputs.monitor-run-attempt }} - evidence-mode: continuation - wave-cell-ids: ${{ inputs.wave-cell-ids }} - wave-index: '3' - source-wave-run-id: none - secrets: inherit - - # Every wave job re-enters the run's lease with release: 'false'; only this job frees it. - release_lease: - if: always() - needs: - - single_cell - - resume_cell - - wave_gate - - wave_cell_1 - - wave_cell_2 - - wave_cell_3 - - wave_cell_4 - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 10 - environment: production - steps: - - uses: actions/checkout@v4 - - - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-terraform-state - object: terraform/state/cloud-sql-rollout/production.lock - release: 'true' diff --git a/.github/workflows/cloud-deploy-relay-production-director.yml b/.github/workflows/cloud-deploy-relay-production-director.yml deleted file mode 100644 index 6ec0029b450..00000000000 --- a/.github/workflows/cloud-deploy-relay-production-director.yml +++ /dev/null @@ -1,311 +0,0 @@ -name: Deploy Relay Production Director - -on: - workflow_dispatch: - inputs: - image-digest: - description: 'Immutable relay image digest (sha256: plus 64 lowercase hex characters)' - required: true - type: string - regional-placement-mode: - description: Preserve the live switch, explicitly enable Asia preference, or force US-first - required: true - default: preserve - type: choice - options: [preserve, enable, disable] - region-correction-cohort-percent: - description: 'Preserve the measured-correction cohort, or set an integer 0–100; durable rehome stays disabled' - required: true - default: preserve - type: string - reserve-placement: - description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve - required: true - default: preserve - type: choice - options: [preserve, 'off', dry-run, 'on'] - shadow-seat-feed-cells: - description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused - required: true - default: preserve - type: string - prune-incompatible-revisions: - description: Retain only the newly verified serving and rollback revisions - required: true - default: false - type: boolean - confirmation: - description: Enter the exact confirmation required by a destructive option - required: false - type: string - expected-rehome-generation: - description: Exact durable regional-rehome generation; it must remain disabled - required: true - type: string - bootstrap-runtime-identity: - description: One-time move from the stamped-cell identity to the director identity - required: true - default: false - type: boolean - predecessor-image-digest: - description: Exact immutable serving predecessor digest for the one-time identity bootstrap - required: true - type: string - shape-baseline-revision: - description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts - required: false - type: string - -permissions: - contents: read - id-token: write - -# Director updates and candidate operations both mutate production relay control state. -concurrency: - group: production-cloud-sql-rollout - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - deploy: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - environment: production - env: - GCP_PROJECT_ID: onorca-cloud - GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} - DIRECTOR_SERVICE_NAME: orca-cloud-relay - IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay - REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled - IMAGE_DIGEST: ${{ inputs.image-digest }} - REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }} - REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }} - RESERVE_PLACEMENT: ${{ inputs.reserve-placement }} - SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }} - PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }} - # Floor the served revision must keep, matching relay_min_instances in - # environments/production.tfvars. This gate only fails a bad deploy; Terraform - # still owns the value, and the candidate inherits it from the serving revision. - DIRECTOR_MIN_INSTANCES: 5 - DIRECTOR_MAX_INSTANCES: 5 - DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} - PREDECESSOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_RUNTIME_SERVICE_ACCOUNT }} - REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain - EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }} - BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }} - PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }} - SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }} - steps: - - uses: actions/checkout@v4 - - - id: google-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-terraform-state - object: terraform/state/cloud-sql-rollout/production.lock - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - name: Resolve immutable production image - shell: bash - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - if [[ ! "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "image-digest must be an immutable lowercase sha256 digest" >&2 - exit 1 - fi - if test "${REGION_CORRECTION_COHORT_PERCENT}" != preserve; then - [[ "${REGION_CORRECTION_COHORT_PERCENT}" =~ ^([0-9]|[1-9][0-9]|100)$ ]] - fi - IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}" - SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" - test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}" - [[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]] - [[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] - [[ "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]] - [[ "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]] - [[ "${BOOTSTRAP_RUNTIME_IDENTITY}" =~ ^(true|false)$ ]] - if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then - [[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] - test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false - test "${REGIONAL_PLACEMENT_MODE}" = preserve - test -z "${SHAPE_BASELINE_REVISION}" - test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY - elif test -n "${SHAPE_BASELINE_REVISION}"; then - [[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-[0-9]{5}-[a-z0-9]{3}$ ]] - test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false - test "${REGIONAL_PLACEMENT_MODE}" = preserve - test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE - elif test "${PRUNE_INCOMPATIBLE_REVISIONS}" = true; then - test "${REGIONAL_PLACEMENT_MODE}" = preserve - test "${CONFIRMATION}" = PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS - elif test "${REGIONAL_PLACEMENT_MODE}" = disable; then - test "${CONFIRMATION}" = FORCE_RELAY_US_FIRST - else - test -z "${CONFIRMATION}" - fi - echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}" - - # Why: the deploy INHERITS the serving revision's floor, so when that revision has - # already lost it the candidate inherits zero, the in-script gate compares zero against - # zero and passes, and the post-deploy check below only notices after traffic moved. - # The documented rollback target is created at minimum instances zero, so promoting it - # arms exactly that. Refuse to inherit a degraded floor rather than latch it. - - name: Require a healthy serving floor before deploying - shell: bash - run: | - SERVING="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -r '[.status.traffic[] | select((.percent // 0) > 0)] - | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" - test -n "${SERVING}" - FLOOR="$(gcloud run revisions describe "${SERVING}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")" - if [[ "${FLOOR:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then - echo "serving revision ${SERVING} holds ${FLOOR:-0} minimum instances," \ - "below ${DIRECTOR_MIN_INSTANCES}; deploying would inherit and latch it." >&2 - echo "Restore the floor first: gcloud run services update ${DIRECTOR_SERVICE_NAME}" \ - "--min-instances=${DIRECTOR_MIN_INSTANCES}" >&2 - exit 1 - fi - CEILING="$(gcloud run revisions describe "${SERVING}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")" - test "${CEILING}" = "${DIRECTOR_MAX_INSTANCES}" - echo "serving revision ${SERVING} holds ${FLOOR} minimum instances" - echo "SERVING_REVISION=${SERVING}" >> "${GITHUB_ENV}" - - # Why: no --min-instances here. The candidate inherits the Terraform-owned - # scaling, and this step ends with 100% traffic on it. Pinning 1 rebuilt the - # per-instance admission shortage that took placement failures to ~70%. - - name: Deploy director blue/green - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - served_version="$(gcloud run revisions describe "${SERVING_REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -r '[.spec.containers[0].env[]? | - select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") | - (.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) | - (.version // .key // empty)] | - if length == 1 then .[0] else empty end')" - if [[ "${served_version}" =~ ^[1-9][0-9]*$ ]]; then - current_version="${served_version}" - else - test "${REGIONAL_PLACEMENT_MODE}" = preserve - current_version="$(gcloud secrets versions describe latest \ - --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \ - --format='value(name)' | awk -F/ '{print $NF}')" - [[ "${current_version}" =~ ^[1-9][0-9]*$ ]] - fi - current="$(gcloud secrets versions access "${current_version}" \ - --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")" - [[ "${current}" =~ ^(true|false)$ ]] - case "${REGIONAL_PLACEMENT_MODE}" in - preserve) desired="${current}" ;; - enable) desired=true ;; - disable) desired=false ;; - *) echo "regional-placement-mode is invalid" >&2; exit 1 ;; - esac - if test "${current}" != "${desired}"; then - target_version="$(printf '%s' "${desired}" | gcloud secrets versions add \ - "${REGIONAL_PLACEMENT_SECRET}" --project "${GCP_PROJECT_ID}" --data-file=- \ - --format='value(name)' --quiet | awk -F/ '{print $NF}')" - else - target_version="${current_version}" - fi - [[ "${target_version}" =~ ^[1-9][0-9]*$ ]] - RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" - shape_args=() - if test -n "${SHAPE_BASELINE_REVISION}"; then - shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}") - fi - node dev/scripts/deploy-relay-blue-green.mjs \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --service "${DIRECTOR_SERVICE_NAME}" \ - --image "${IMAGE}" \ - --role director \ - "${shape_args[@]}" \ - --runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ - --predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" \ - --bootstrap-runtime-identity "${BOOTSTRAP_RUNTIME_IDENTITY}" \ - --predecessor-image-digest "${PREDECESSOR_IMAGE_DIGEST}" \ - --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ - --rehome-audience "${REHOME_AUDIENCE}" \ - --rehome-control-origin https://relay.onorca.dev \ - --admin-audience https://relay.onorca.dev/v1/admin/drain \ - --expected-rehome-generation "${EXPECTED_REHOME_GENERATION}" \ - --max-instances "${DIRECTOR_MAX_INSTANCES}" \ - --prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \ - --release-id "${RELEASE_ID}" \ - --regional-placement-secret-version "${target_version}" \ - --region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}" \ - --reserve-placement "${RESERVE_PLACEMENT}" \ - --shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}" - echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}" - echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}" - - - name: Verify served revision and native health - shell: bash - run: | - SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --format=json)" - REVISION="$(jq -r '[.status.traffic[] | select((.percent // 0) > 0)] | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' <<< "${SERVICE_JSON}")" - test -n "${REVISION}" - SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --format='value(spec.containers[0].image)')" - test "${SERVED_IMAGE}" = "${IMAGE}" - SERVED_REGIONAL_PLACEMENT_SECRET="$(gcloud run revisions describe "${REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -cer '[.spec.containers[0].env[] | - select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") | - (.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) | - {secret: (.secret // .name), version: (.version // .key)}] | - if length == 1 then .[0] else error("regional placement secret missing") end')" - test "$(jq -r '.secret' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \ - "${REGIONAL_PLACEMENT_SECRET}" - test "$(jq -r '.version' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \ - "${REGIONAL_PLACEMENT_VERSION}" - test "$(gcloud secrets versions access "${REGIONAL_PLACEMENT_VERSION}" --project "${GCP_PROJECT_ID}" \ - --secret "${REGIONAL_PLACEMENT_SECRET}")" = "${REGIONAL_PLACEMENT_ENABLED}" - # Why: a served revision with no warm-instance floor still passes health and digest - # checks while quietly shrinking per-instance admission capacity. - SERVED_MIN_INSTANCES="$(gcloud run revisions describe "${REVISION}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")" - if [[ "${SERVED_MIN_INSTANCES:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then - echo "served revision ${REVISION} holds ${SERVED_MIN_INSTANCES:-0} minimum instances, expected at least ${DIRECTOR_MIN_INSTANCES}" >&2 - exit 1 - fi - SERVED_MAX_INSTANCES="$(gcloud run revisions describe "${REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")" - test "${SERVED_MAX_INSTANCES}" = "${DIRECTOR_MAX_INSTANCES}" - SERVICE_URL="$(jq -r '.status.url' <<< "${SERVICE_JSON}")" - node dev/scripts/smoke-relay.mjs "${SERVICE_URL}" diff --git a/.github/workflows/cloud-deploy-relay-production-multi-target.yml b/.github/workflows/cloud-deploy-relay-production-multi-target.yml deleted file mode 100644 index cb6e3bf7d35..00000000000 --- a/.github/workflows/cloud-deploy-relay-production-multi-target.yml +++ /dev/null @@ -1,500 +0,0 @@ -name: Deploy Relay Production Multi-Target - -on: - workflow_dispatch: - inputs: - source-cell-id: - description: Existing Terraform source cell ID - required: true - type: string - target-cell-ids: - description: Comma-separated distinct Terraform target cell IDs - required: true - type: string - general-cell-ids: - description: Comma-separated proven cells that remain eligible for ordinary placement - required: false - type: string - unobserved-connection-bound: - description: Exact worst-case unobserved connection bound proven by the passing load gate - required: false - type: string - failed-target-cell-id: - description: Registered failed target to fence and supersede - required: false - type: string - replacement-target-cell-id: - description: Healthy replacement for registered failed target - required: false - type: string - mode: - description: Preflight/audit are read-only; other modes mutate production - required: true - default: preflight - type: choice - options: - - audit - - preflight - - cutover-admission - - add-migration-cells - - promote-general-cell - - retire-migration-cell - - execute - - recover-forward - - fence-source - - supersede-target - confirmation: - description: Enter CUTOVER_SELECTOR, ADD_MIGRATION_CELLS, PROMOTE_GENERAL_CELL, RETIRE_MIGRATION_CELL, EVACUATE_MULTI, RECOVER_FORWARD, or FENCE_SOURCE - required: false - type: string - selector-attempt-id: - description: Exact durable selector attempt ID for admission mutations - required: false - type: string - monitor-run-id: - description: Successful fresh dry-run monitor workflow run ID - required: false - type: string - monitor-run-attempt: - description: Exact dry-run monitor workflow attempt - required: false - type: string - broker-operation-id: - description: Stable durable broker operation ID for target supersession - required: false - type: string - completed-fence-attempt-id: - description: Exact older completed fence attempt to recover without replay - required: false - type: string - completed-fence-commit: - description: Exact older fence commit bound to the completed attempt - required: false - type: string - completed-fence-operation: - description: Exact DONE Compute resize operation to adopt - required: false - type: string - completed-fence-state-serial: - description: Exact Terraform serial before the completed fence - required: false - type: string - completed-fence-plan-generation: - description: Exact saved-plan object generation - required: false - type: string - completed-fence-state-generation: - description: Exact current Terraform state object generation - required: false - type: string - completed-fence-state-sha256: - description: Exact current Terraform state object SHA-256 - required: false - type: string - expected-lease-generation: - description: Exact live lease generation authorized for conditional takeover - required: false - type: string - expected-lease-operation-id: - description: Exact live lease operation ID authorized for takeover - required: false - type: string - expected-lease-request-digest: - description: Exact live lease request digest authorized for takeover - required: false - type: string - -permissions: - actions: read - contents: read - id-token: write - -concurrency: - group: production-cloud-sql-rollout - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - deploy: - if: >- - ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && - github.ref == 'refs/heads/main' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - environment: production - env: - GCP_PROJECT_ID: onorca-cloud - DIRECTOR_ORIGIN: https://relay.onorca.dev - ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain - SOURCE_CELL_ID: ${{ inputs.source-cell-id }} - TARGET_CELL_IDS: ${{ inputs.target-cell-ids }} - GENERAL_CELL_IDS: ${{ inputs.general-cell-ids }} - UNOBSERVED_CONNECTION_BOUND: ${{ inputs.unobserved-connection-bound }} - FAILED_TARGET_CELL_ID: ${{ inputs.failed-target-cell-id }} - REPLACEMENT_TARGET_CELL_ID: ${{ inputs.replacement-target-cell-id }} - DEPLOY_MODE: ${{ inputs.mode }} - MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} - MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} - SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }} - BROKER_OPERATION_ID: ${{ inputs.broker-operation-id }} - COMPLETED_FENCE_ATTEMPT_ID: ${{ inputs.completed-fence-attempt-id }} - COMPLETED_FENCE_COMMIT: ${{ inputs.completed-fence-commit }} - COMPLETED_FENCE_OPERATION: ${{ inputs.completed-fence-operation }} - COMPLETED_FENCE_STATE_SERIAL: ${{ inputs.completed-fence-state-serial }} - COMPLETED_FENCE_PLAN_GENERATION: ${{ inputs.completed-fence-plan-generation }} - COMPLETED_FENCE_STATE_GENERATION: ${{ inputs.completed-fence-state-generation }} - COMPLETED_FENCE_STATE_SHA256: ${{ inputs.completed-fence-state-sha256 }} - EXPECTED_LEASE_GENERATION: ${{ inputs.expected-lease-generation }} - EXPECTED_LEASE_OPERATION_ID: ${{ inputs.expected-lease-operation-id }} - EXPECTED_LEASE_REQUEST_DIGEST: ${{ inputs.expected-lease-request-digest }} - steps: - - uses: actions/checkout@v4 - - - name: Require private fence-broker environment - if: >- - ${{ inputs.mode == 'fence-source' || - inputs.mode == 'supersede-target' }} - env: - FENCE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }} - FENCE_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }} - FENCE_BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }} - run: | - test -n "${FENCE_WORKLOAD_IDENTITY_PROVIDER}" - test -n "${FENCE_SERVICE_ACCOUNT}" - test -n "${FENCE_BROKER_URI}" - - - name: Reject direct-runner Terraform fence aborts - if: ${{ inputs.mode == 'abort-fence-source' }} - run: | - echo "Terraform fence aborts require a reviewed private-broker recovery path." >&2 - exit 1 - - - name: Require fresh dry-run evidence reference - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} - run: | - [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]] - [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] - - - name: Download private dry-run evidence - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} - uses: actions/download-artifact@v4 - with: - name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ runner.temp }}/relay-monitor-evidence - github-token: ${{ github.token }} - run-id: ${{ inputs.monitor-run-id }} - - - uses: ./.github/actions/setup-pnpm - with: - package-json-file: cloud/package.json - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - run: pnpm install --frozen-lockfile - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_wrapper: false - - - name: Verify dry-run artifact before cloud authentication - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} - run: | - node dev/scripts/relay-monitor-evidence.mjs verify-restore \ - --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ - --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ - --run-id "${MONITOR_RUN_ID}" \ - --run-attempt "${MONITOR_RUN_ATTEMPT}" \ - --commit-sha "${GITHUB_SHA}" \ - --mode dry-run - - - name: Reject previously consumed dry-run evidence - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} - env: - GH_TOKEN: ${{ github.token }} - run: | - MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - COUNT="$(gh api \ - "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \ - --jq '.total_count')" - test "${COUNT}" = "0" - - - id: google-auth - if: ${{ inputs.mode != 'supersede-target' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-terraform-state - object: terraform/state/cloud-sql-rollout/production.lock - - - name: Require explicit mutation confirmation - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - if [[ "${DEPLOY_MODE}" = "cutover-admission" ]]; then - test "${CONFIRMATION}" = "CUTOVER_SELECTOR" - elif [[ "${DEPLOY_MODE}" = "add-migration-cells" ]]; then - test "${CONFIRMATION}" = "ADD_MIGRATION_CELLS" - elif [[ "${DEPLOY_MODE}" = "promote-general-cell" ]]; then - test "${CONFIRMATION}" = "PROMOTE_GENERAL_CELL" - elif [[ "${DEPLOY_MODE}" = "retire-migration-cell" ]]; then - test "${CONFIRMATION}" = "RETIRE_MIGRATION_CELL" - elif [[ "${DEPLOY_MODE}" = "execute" ]]; then - test "${CONFIRMATION}" = "EVACUATE_MULTI" - elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then - test "${CONFIRMATION}" = "RECOVER_FORWARD" - elif [[ "${DEPLOY_MODE}" = "fence-source" ]]; then - test "${CONFIRMATION}" = "FENCE_SOURCE" - elif [[ "${DEPLOY_MODE}" = "supersede-target" ]]; then - test "${CONFIRMATION}" = "SUPERSEDE_TARGET" - elif [[ "${DEPLOY_MODE}" = "abort-fence-source" ]]; then - test "${CONFIRMATION}" = "ABORT_FENCE" - else - test "${CONFIRMATION}" = "FENCE_SOURCE" - fi - - - name: Require exact source-fence broker contract - if: ${{ inputs.mode == 'fence-source' }} - run: | - test "${SOURCE_CELL_ID}" = "production-gce-c3" - test "${TARGET_CELL_IDS}" = "production-gce-c7,production-gce-c8,production-gce-c10,production-gce-c13,production-gce-c17,production-gce-c18" - [[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] - if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then - [[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]] - test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}" - [[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]] - else - test -z "${EXPECTED_LEASE_OPERATION_ID}" - test -z "${EXPECTED_LEASE_REQUEST_DIGEST}" - fi - - - name: Require exact broker cell contract - if: ${{ inputs.mode == 'supersede-target' }} - run: | - test "${SOURCE_CELL_ID}" = "production-gce-c3" - test "${FAILED_TARGET_CELL_ID}" = "production-gce-c12" - test "${REPLACEMENT_TARGET_CELL_ID}" = "production-gce-c13" - test "${TARGET_CELL_IDS}" = "production-gce-c12,production-gce-c13" - if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then - [[ "${COMPLETED_FENCE_ATTEMPT_ID}" =~ ^[0-9a-f-]{36}$ ]] - [[ "${COMPLETED_FENCE_COMMIT}" =~ ^[0-9a-f]{40}$ ]] - [[ "${COMPLETED_FENCE_OPERATION}" =~ ^[A-Za-z0-9._-]{1,256}$ ]] - [[ "${COMPLETED_FENCE_STATE_SERIAL}" =~ ^[0-9]+$ ]] - [[ "${COMPLETED_FENCE_PLAN_GENERATION}" =~ ^[1-9][0-9]*$ ]] - [[ "${COMPLETED_FENCE_STATE_GENERATION}" =~ ^[1-9][0-9]*$ ]] - [[ "${COMPLETED_FENCE_STATE_SHA256}" =~ ^[0-9a-f]{64}$ ]] - test -n "${EXPECTED_LEASE_GENERATION}" - fi - if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then - [[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]] - test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}" - [[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]] - else - test -z "${EXPECTED_LEASE_OPERATION_ID}" - test -z "${EXPECTED_LEASE_REQUEST_DIGEST}" - fi - - - name: Read reviewed Terraform topology - if: ${{ inputs.mode != 'supersede-target' }} - run: | - node dev/scripts/infra.mjs init --env production - terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json" - RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)" - DIRECTOR_MIN_INSTANCES="$(terraform -chdir=infra/terraform console \ - -var-file=environments/production.tfvars <<< 'var.relay_min_instances')" - [[ "${DIRECTOR_MIN_INSTANCES}" =~ ^[1-9][0-9]*$ ]] - echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}" - echo "DIRECTOR_MIN_INSTANCES=${DIRECTOR_MIN_INSTANCES}" >> "${GITHUB_ENV}" - - - name: Verify fresh dry-run evidence against live selector - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - SCOPED_RECOVERY_ARGS=() - if [[ ("${DEPLOY_MODE}" = "execute" || - "${DEPLOY_MODE}" = "recover-forward") && - "${SOURCE_CELL_ID}" = "production-gce-c12" ]]; then - SCOPED_RECOVERY_ARGS=( - --scoped-recovery-source-cell-id - production-gce-c3 - ) - fi - node dev/scripts/relay-monitor-evidence.mjs verify-mutation \ - --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ - --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ - --run-id "${MONITOR_RUN_ID}" \ - --run-attempt "${MONITOR_RUN_ATTEMPT}" \ - --commit-sha "${GITHUB_SHA}" \ - --mode dry-run \ - --mutation-mode "${DEPLOY_MODE}" \ - --source-cell-id "${SOURCE_CELL_ID}" \ - "${SCOPED_RECOVERY_ARGS[@]}" \ - --director-origin "${DIRECTOR_ORIGIN}" - - - name: Recheck all live safety signals - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - pnpm incident:relay-preflight -- \ - --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" - - - name: Create single-use dry-run marker - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} - run: | - MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption" - printf '%s\n' "${GITHUB_RUN_ID}" \ - > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}" - - - name: Consume dry-run evidence - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }} - uses: actions/upload-artifact@v4 - with: - name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - retention-days: 90 - if-no-files-found: error - - - id: google-fence-broker-auth - if: >- - ${{ inputs.mode == 'fence-source' || - inputs.mode == 'supersede-target' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }} - id_token_include_email: true - - - name: Invoke private target-supersession broker - if: ${{ inputs.mode == 'supersede-target' }} - env: - BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }} - BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }} - run: | - [[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] - if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then - REQUEST="$(jq -cn \ - --arg operationId "${BROKER_OPERATION_ID}" \ - --arg fenceCommit "${GITHUB_SHA}" \ - --arg attemptId "${COMPLETED_FENCE_ATTEMPT_ID}" \ - --arg completedCommit "${COMPLETED_FENCE_COMMIT}" \ - --arg gceOperation "${COMPLETED_FENCE_OPERATION}" \ - --arg stateSerial "${COMPLETED_FENCE_STATE_SERIAL}" \ - --arg planGeneration "${COMPLETED_FENCE_PLAN_GENERATION}" \ - --arg stateGeneration "${COMPLETED_FENCE_STATE_GENERATION}" \ - --arg stateSha256 "${COMPLETED_FENCE_STATE_SHA256}" \ - --arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \ - --arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \ - --arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \ - '{v:1,operationId:$operationId,fenceCommit:$fenceCommit, - completedFenceRecovery:{attemptId:$attemptId,fenceCommit:$completedCommit, - gceOperation:$gceOperation,terraformStateSerial:($stateSerial|tonumber), - planObjectGeneration:$planGeneration, - terraformStateObjectGeneration:$stateGeneration, - terraformStateObjectSha256:$stateSha256}, - expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId, - requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')" - elif [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then - REQUEST="$(jq -cn \ - --arg operationId "${BROKER_OPERATION_ID}" \ - --arg fenceCommit "${GITHUB_SHA}" \ - --arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \ - --arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \ - --arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \ - '{v:1,operationId:$operationId,fenceCommit:$fenceCommit, - expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId, - requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')" - else - REQUEST="$(jq -cn \ - --arg operationId "${BROKER_OPERATION_ID}" \ - --arg fenceCommit "${GITHUB_SHA}" \ - '{v:1,operationId:$operationId,fenceCommit:$fenceCommit,confirmation:"SUPERSEDE_TARGET"}')" - fi - curl --fail-with-body --max-time 1790 \ - --request POST "${BROKER_URI}/v1/supersede-target" \ - --header "Authorization: Bearer ${BROKER_ID_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "${REQUEST}" - - - name: Invoke private source-fence broker - if: ${{ inputs.mode == 'fence-source' }} - env: - BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }} - BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }} - run: | - if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then - REQUEST="$(jq -cn \ - --arg operationId "${BROKER_OPERATION_ID}" \ - --arg fenceCommit "${GITHUB_SHA}" \ - --arg targetCellIds "${TARGET_CELL_IDS}" \ - --arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \ - --arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \ - --arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \ - '{v:1,operationId:$operationId,fenceCommit:$fenceCommit, - targetCellIds:($targetCellIds|split(",")), - expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId, - requestDigest:$leaseRequestDigest},confirmation:"FENCE_SOURCE"}')" - else - REQUEST="$(jq -cn \ - --arg operationId "${BROKER_OPERATION_ID}" \ - --arg fenceCommit "${GITHUB_SHA}" \ - --arg targetCellIds "${TARGET_CELL_IDS}" \ - '{v:1,operationId:$operationId,fenceCommit:$fenceCommit, - targetCellIds:($targetCellIds|split(",")),confirmation:"FENCE_SOURCE"}')" - fi - curl --fail-with-body --max-time 1790 \ - --request POST "${BROKER_URI}/v1/fence-source" \ - --header "Authorization: Bearer ${BROKER_ID_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "${REQUEST}" - - - name: Preflight or run multi-target evacuation - if: >- - ${{ inputs.mode != 'fence-source' && - inputs.mode != 'supersede-target' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/deploy-relay-gce-multi-target.mjs \ - --project "${GCP_PROJECT_ID}" \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --admin-audience "${ADMIN_AUDIENCE}" \ - --topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \ - --source-cell-id "${SOURCE_CELL_ID}" \ - --target-cell-ids "${TARGET_CELL_IDS}" \ - --general-cell-ids "${GENERAL_CELL_IDS}" \ - --unobserved-connection-bound "${UNOBSERVED_CONNECTION_BOUND}" \ - --director-region "${{ vars.PRODUCTION_GCP_REGION }}" \ - --director-service "orca-cloud-relay" \ - --director-min-instances "${DIRECTOR_MIN_INSTANCES}" \ - --selector-attempt-id "${SELECTOR_ATTEMPT_ID}" \ - --failed-target-cell-id "${FAILED_TARGET_CELL_ID}" \ - --replacement-target-cell-id "${REPLACEMENT_TARGET_CELL_ID}" \ - --runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \ - --environment production \ - --fence-commit "${GITHUB_SHA}" \ - --terraform-dir infra/terraform \ - --terraform-var-file environments/production.tfvars \ - --mode "${DEPLOY_MODE}" \ - --connection-ceiling 1000 \ - --minimum-lease-remaining-ms 600000 diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml deleted file mode 100644 index babe9c2a2fc..00000000000 --- a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml +++ /dev/null @@ -1,930 +0,0 @@ -name: Deploy Relay Production Same-Cap Job - -on: - workflow_call: - inputs: - mode: { required: true, type: string } - target-cell-id: { required: true, type: string } - target-image-digest: { required: true, type: string } - rollback-image-digest: { required: true, type: string } - target-rehome-protocol: { required: true, type: string } - rollback-rehome-protocol: { required: true, type: string } - expected-selector-generation: { required: true, type: string } - expected-existing-only-cells: { required: true, type: string } - expected-migration-only-cells: { required: true, type: string } - expected-general-cells: { required: true, type: string } - expected-rehome-generation: { required: true, type: string } - wave-index: { required: true, type: string } - drain-pace-window-ms: { required: true, type: string } - -permissions: - contents: read - id-token: write - -defaults: - run: - working-directory: cloud - -jobs: - rollout: - if: ${{ github.ref == 'refs/heads/main' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - # The pre-drain sample adds 3-8 min (20 at most); a timeout cancels the job and skips the failsafe. - # A 1200000 drain step alone may take 50 min. - timeout-minutes: 120 - environment: production - env: - GCP_PROJECT_ID: onorca-cloud - GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} - DIRECTOR_ORIGIN: https://relay.onorca.dev - IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay - TARGET_CELL_ID: ${{ inputs.target-cell-id }} - DEPLOY_MODE: ${{ inputs.mode }} - TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }} - ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }} - TARGET_REHOME_PROTOCOL: ${{ inputs.target-rehome-protocol }} - ROLLBACK_REHOME_PROTOCOL: ${{ inputs.rollback-rehome-protocol }} - EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }} - EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }} - EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }} - EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }} - EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }} - WAVE_INDEX: ${{ inputs.wave-index }} - # The cell spreads its drain sends over this window, so each drained host re-dials - # /v1/assign at hosts / window and is re-placed through the director's drain-return - # lane. The wave validator admits only its closed set, and anything below the 300000 - # default only on US general cells: Asia drains are bound by their targets' accept - # rate. The transition wait below is the 15-min migration lease plus this window. A - # cell on an older image drains at once, or refuses a window above 300000. - DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }} - steps: - - name: Require exact reusable-workflow configuration - working-directory: . - env: - DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - CAPACITY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} - run: | - [[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]] - [[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] - [[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] - test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}" - [[ "${TARGET_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]] - [[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]] - [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] - [[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] - # cell_1..cell_10 in the calling wave; the chain is static, so this range is too. - [[ "${WAVE_INDEX}" =~ ^[0-9]$ ]] - if test "${DEPLOY_MODE}" != verify && test "${GITHUB_RUN_ATTEMPT}" != 1; then - echo "mutations are single-dispatch: recover each remaining cell" >&2 - echo "with its own fresh dispatch instead of a re-run" >&2 - exit 1 - fi - test -n "${GCP_REGION}" - test -n "${DEPLOY_WIF}" - test -n "${DEPLOY_SERVICE_ACCOUNT}" - test -n "${CAPACITY_WIF}" - test -n "${CAPACITY_SERVICE_ACCOUNT}" - test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" - - - uses: actions/checkout@v4 - - - uses: ./.github/actions/setup-pnpm - with: { package-json-file: cloud/package.json } - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - run: pnpm install --frozen-lockfile - - - uses: hashicorp/setup-terraform@v3 - with: { terraform_wrapper: false } - - # One approved-cell table, in the wave validator the dispatch gate already uses, so - # a cell's class and its wave's selector delta cannot drift apart between the two. - # Also re-checks the drain pace against this cell, before anything reads production. - - name: Resolve this cell's admission class and wave selector delta - run: | - CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \ - --cell-id "${TARGET_CELL_ID}" --drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}")" - ENTRY_ADMISSION="$(jq -er '.entryAdmission' <<< "${CELL_CLASS}")" - SELECTOR_WAVE_DELTA="$(jq -er '.selectorWaveDelta' <<< "${CELL_CLASS}")" - if test "${DEPLOY_MODE}" = verify; then - EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}" - else - EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION \ - + (SELECTOR_WAVE_DELTA * WAVE_INDEX)))" - fi - { - echo "ENTRY_ADMISSION=${ENTRY_ADMISSION}" - echo "SELECTOR_WAVE_DELTA=${SELECTOR_WAVE_DELTA}" - echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}" - } >> "${GITHUB_ENV}" - - - id: deploy-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-terraform-state - object: terraform/state/cloud-sql-rollout/production.lock - release: 'false' - - - name: Recheck aggregate SQL, pool, reconnect, migration, and selector safety - if: ${{ inputs.mode != 'verify' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - # Freshness-only failures are publish lag, not health, so the CLI retries - # them a bounded number of times. The expected selector comes from the - # dispatch inputs, which the rehome inspect below verifies against the - # live director; the pre-drain sample later reads the same file. - jq -n \ - --arg existingOnly "${EXPECTED_EXISTING_ONLY_CELLS/none/}" \ - --arg migrationOnly "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \ - --arg general "${EXPECTED_GENERAL_CELLS/none/}" \ - '{existingOnly:$existingOnly,migrationOnly:$migrationOnly,general:$general} - | map_values(split(",") | map(select(length > 0)))' \ - > "${RUNNER_TEMP}/relay-same-cap-selector.json" - pnpm incident:relay-preflight -- \ - --no-monitor-state \ - --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \ - --wave-index "${WAVE_INDEX}" \ - --selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness - - - name: Require durable rehome disabled and exact selector - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - node dev/scripts/operate-relay-regional-rehome.mjs \ - --mode inspect \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --expected-selector-generation "${EFFECTIVE_SELECTOR_GENERATION}" \ - --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ - --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ - --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ - --expected-control-generation "${EXPECTED_REHOME_GENERATION}" \ - | jq -e '.control.enabled == false' >/dev/null - - - name: Initialize the exact production backend - run: node dev/scripts/infra.mjs init --env production - - - name: Resolve immutable same-cap cell configuration - shell: bash - run: | - TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}" - case "${TARGET_HOSTNAME}" in - c7|c8|c9|c10|c13|c14|c15|c16|c19|c20|c21|c22|c23|c24|c25|c26) - EXPECTED_HARD_CAP=1000 - EXPECTED_REGION=us-central1 - EXPECTED_DATABASE_POOL_MAX= - ;; - c17|c18) - EXPECTED_HARD_CAP=600 - EXPECTED_REGION=us-central1 - EXPECTED_DATABASE_POOL_MAX= - ;; - c27|c28|c29|c30|c31|c34|c35) - EXPECTED_HARD_CAP=3000 - EXPECTED_REGION=asia-east2 - EXPECTED_DATABASE_POOL_MAX=16 - ;; - c32|c33) - EXPECTED_HARD_CAP=3000 - EXPECTED_REGION=us-central1 - EXPECTED_DATABASE_POOL_MAX= - ;; - *) exit 1 ;; - esac - EXPECTED_UNOBSERVED_BOUND=60 - CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev" - CELLS_JSON="$(terraform -chdir=infra/terraform console \ - -var-file=environments/production.tfvars \ - <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')" - SOURCE_CELLS="$(terraform -chdir=infra/terraform console \ - -var-file=environments/production.tfvars \ - <<< 'jsonencode(var.relay_region_rehome_source_cell_ids)' | jq -er '.')" - CURRENT_SHAPE="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${CELLS_JSON}")" - test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}" - test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \ - "${EXPECTED_UNOBSERVED_BOUND}" - # The startup script emits a pool line only off the root default, so an unpinned cell - # must still be on that default or its plan would carry a line nothing reviews. - test "$(jq -r '.database_pool_max' <<< "${CURRENT_SHAPE}")" = \ - "${EXPECTED_DATABASE_POOL_MAX:-10}" - TARGET_ZONE="$(jq -r '.zone' <<< "${CURRENT_SHAPE}")" - MIG_NAME="orca-cloud-relay-gce-${TARGET_HOSTNAME}" - if test "${DEPLOY_MODE}" = rollback; then - DESIRED_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}" - CURRENT_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}" - DESIRED_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}" - CURRENT_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}" - else - DESIRED_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}" - CURRENT_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}" - DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}" - CURRENT_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}" - fi - # The startup template emits rehome trust lines only for a declared source cell, so - # require membership exactly when either side of this roll expects those lines. - if test "${EXPECTED_REGION}" = us-central1 && { - test "${DESIRED_REHOME_PROTOCOL}" != 0 || test "${CURRENT_REHOME_PROTOCOL}" != 0 - }; then - jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \ - <<< "${SOURCE_CELLS}" >/dev/null - fi - DESIRED_IMAGE="${IMAGE_REPOSITORY}@${DESIRED_IMAGE_DIGEST}" - OVERRIDE_CELLS_JSON="$(jq -ce --arg cell "${TARGET_CELL_ID}" \ - --arg image "${DESIRED_IMAGE}" '.[$cell].image = $image' <<< "${CELLS_JSON}")" - jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \ - '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-same-cap.tfvars.json" - SERVED_DIGEST="$(gcloud artifacts docker images describe "${DESIRED_IMAGE}" \ - --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" - test "${SERVED_DIGEST}" = "${DESIRED_IMAGE_DIGEST}" - { - echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}" - echo "CELL_ORIGIN=${CELL_ORIGIN}" - echo "TARGET_ZONE=${TARGET_ZONE}" - echo "MIG_NAME=${MIG_NAME}" - echo "EXPECTED_HARD_CAP=${EXPECTED_HARD_CAP}" - echo "EXPECTED_UNOBSERVED_BOUND=${EXPECTED_UNOBSERVED_BOUND}" - echo "EXPECTED_DATABASE_POOL_MAX=${EXPECTED_DATABASE_POOL_MAX}" - echo "EXPECTED_REGION=${EXPECTED_REGION}" - echo "DESIRED_IMAGE=${DESIRED_IMAGE}" - echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}" - echo "CURRENT_IMAGE_DIGEST=${CURRENT_IMAGE_DIGEST}" - echo "DESIRED_REHOME_PROTOCOL=${DESIRED_REHOME_PROTOCOL}" - echo "CURRENT_REHOME_PROTOCOL=${CURRENT_REHOME_PROTOCOL}" - } >> "${GITHUB_ENV}" - - - name: Verify exact current generation, digest, cap, and rollback point - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - # A single transient 5xx (LB warm-up behind a fresh instance) must not - # fail a canary; 4xx (auth, generation mismatch) still fails fast. - admin_post() { - local out="${RUNNER_TEMP}/$1.json" - if ! curl --fail-with-body --max-time 30 \ - --retry 3 --retry-delay 2 --retry-connrefused --output "${out}" \ - --request POST "$2" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' --data "$3"; then - cat "${out}" >&2 - return 1 - fi - cat "${out}" - } - CURRENT_RUNTIME="$(admin_post current-runtime \ - "${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')" - # Two different failures leave the cell on the rollback image, and the image - # alone cannot tell them apart. A rollback that failed between its template - # apply and its admission restore restarted the cell, so that cell is not - # draining and resumes. A wave that stopped after its drain and before its - # template apply never restarted anything, so its cell is still draining and - # is stranded: the drain flag only clears on a restart, so it has to be rolled. - LIVE_IMAGE_DIGEST="$(jq -r '.imageDigest' <<< "${CURRENT_RUNTIME}")" - LIVE_DRAINING="$(jq -r '.draining' <<< "${CURRENT_RUNTIME}")" - if test "${DEPLOY_MODE}" = rollback \ - && test "${LIVE_IMAGE_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"; then - if test "${LIVE_DRAINING}" = true; then - ROLLBACK_STAGE=stranded - else - ROLLBACK_STAGE=resume - fi - PREDECESSOR_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}" - PREDECESSOR_REHOME_PROTOCOL="${DESIRED_REHOME_PROTOCOL}" - else - ROLLBACK_STAGE=roll - PREDECESSOR_IMAGE_DIGEST="${CURRENT_IMAGE_DIGEST}" - PREDECESSOR_REHOME_PROTOCOL="${CURRENT_REHOME_PROTOCOL}" - fi - if test "${ROLLBACK_STAGE}" = resume; then - ROLLBACK_RESUME=true - else - ROLLBACK_RESUME=false - fi - # A stranded cell's template still carries the image the cell is serving, so that - # is the predecessor its plan is reviewed against. A template already moved on to - # the target is refused here rather than rolled backwards under a stale review. - if test "${ROLLBACK_STAGE}" = stranded; then - PLAN_ROLLBACK_IMAGE="${DESIRED_IMAGE}" - else - PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" - fi - # Rollback is the documented recovery from a failed canary, which - # leaves the cell migration-only (and possibly still marked - # draining); apply and verify still require the cell pristine in the - # class it is declared to serve in. - if test "${DEPLOY_MODE}" = rollback; then - PRECHECK_ADMISSION=general-or-migration-only - else - PRECHECK_ADMISSION="${ENTRY_ADMISSION}" - fi - # Draining sheds connections, and a migration-only cell holds none, so the flag - # carries no precondition there. It also outlives a failed wave, because the drain - # that set it is followed by no restart, which is the state a failed canary leaves. - if test "${DEPLOY_MODE}" = rollback \ - || test "${ENTRY_ADMISSION}" = migration-only; then - PRECHECK_DRAINING=either - else - PRECHECK_DRAINING=forbidden - fi - # A resumed rollback already restarted, so its cell has to come back not draining; - # that is what separates it from a wave that stopped before its template apply. - if test "${PRECHECK_DRAINING}" = either && test "${ROLLBACK_RESUME}" != true; then - PREDECESSOR_DRAINING_OK=true - else - PREDECESSOR_DRAINING_OK=false - fi - RESTORED_MIGRATION_CELLS="$(jq -rn \ - --arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \ - --arg target "${TARGET_CELL_ID}" \ - '$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')" - RESTORED_GENERAL_CELLS="$(jq -rn \ - --arg value "${EXPECTED_GENERAL_CELLS/none/}" \ - --arg target "${TARGET_CELL_ID}" \ - '$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')" - test -n "${RESTORED_MIGRATION_CELLS}" || RESTORED_MIGRATION_CELLS=none - test -n "${RESTORED_GENERAL_CELLS}" || RESTORED_GENERAL_CELLS=none - ISOLATED_MIGRATION_CELLS="$(jq -rn \ - --arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \ - --arg target "${TARGET_CELL_ID}" \ - '$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')" - ISOLATED_GENERAL_CELLS="$(jq -rn \ - --arg value "${EXPECTED_GENERAL_CELLS/none/}" \ - --arg target "${TARGET_CELL_ID}" \ - '$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')" - test -n "${ISOLATED_MIGRATION_CELLS}" || ISOLATED_MIGRATION_CELLS=none - test -n "${ISOLATED_GENERAL_CELLS}" || ISOLATED_GENERAL_CELLS=none - # A migration-only cell is already isolated and is handed back isolated, so both - # halves of its wave see exactly the membership it entered with. - if test "${ENTRY_ADMISSION}" = migration-only; then - RESTORED_MIGRATION_CELLS="${ISOLATED_MIGRATION_CELLS}" - RESTORED_GENERAL_CELLS="${ISOLATED_GENERAL_CELLS}" - fi - { - echo "ROLLBACK_RESUME=${ROLLBACK_RESUME}" - echo "ROLLBACK_STAGE=${ROLLBACK_STAGE}" - echo "PLAN_ROLLBACK_IMAGE=${PLAN_ROLLBACK_IMAGE}" - # The drain wait and the plan review both read the image this cell actually - # serves, which is the rollback image on a stranded cell and not the current one. - echo "PREDECESSOR_IMAGE_DIGEST=${PREDECESSOR_IMAGE_DIGEST}" - # The failsafe consumes these; deriving them here keeps them - # defined for a failure in any later step. - echo "ISOLATED_MIGRATION_CELLS=${ISOLATED_MIGRATION_CELLS}" - echo "ISOLATED_GENERAL_CELLS=${ISOLATED_GENERAL_CELLS}" - # No restart happens on resume, so isolate below is skipped and - # cannot advance the selector generation. - echo "SELECTOR_GENERATION_AFTER_ISOLATE=${EFFECTIVE_SELECTOR_GENERATION}" - # A failed-canary rollback enters with the target migration-only, - # so the restore inspect cannot reuse the entry membership inputs. - echo "RESTORED_MIGRATION_CELLS=${RESTORED_MIGRATION_CELLS}" - echo "RESTORED_GENERAL_CELLS=${RESTORED_GENERAL_CELLS}" - } >> "${GITHUB_ENV}" - if ! jq -e --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \ - --arg digest "${PREDECESSOR_IMAGE_DIGEST}" \ - --arg region "${EXPECTED_REGION}" \ - --argjson hardCap "${EXPECTED_HARD_CAP}" \ - --argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \ - --argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \ - --argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \ - '.role == "cell" and .cellId == $cell and .cellUrl == $origin and - (.region == $region or - ($region == "us-central1" and $protocol == 0 and .region == null)) and - .imageDigest == $digest and - .connectionCapacity.hardCap == $hardCap and - .connectionCapacity.unobservedBound == $unobservedBound and - (.draining == false or $drainingOk) and - (.regionalRehomeProtocol // 0) == $protocol' <<< "${CURRENT_RUNTIME}" >/dev/null - then - jq -r --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \ - --arg digest "${PREDECESSOR_IMAGE_DIGEST}" \ - --arg region "${EXPECTED_REGION}" \ - --argjson hardCap "${EXPECTED_HARD_CAP}" \ - --argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \ - --argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \ - --argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \ - '[ - if .role != "cell" then "role" else empty end, - if .cellId != $cell then "cellId" else empty end, - if .cellUrl != $origin then "cellUrl" else empty end, - if (.region != $region and - ($region != "us-central1" or $protocol != 0 or .region != null)) - then "region" else empty end, - if .imageDigest != $digest then "imageDigest" else empty end, - if .connectionCapacity.hardCap != $hardCap then "hardCap" else empty end, - if .connectionCapacity.unobservedBound != $unobservedBound then "unobservedBound" else empty end, - if (.draining != false and ($drainingOk | not)) then "draining" else empty end, - if (.regionalRehomeProtocol // 0) != $protocol then "regionalRehomeProtocol" else empty end - ] | "runtime predecessor mismatch fields=" + join(",")' \ - <<< "${CURRENT_RUNTIME}" >&2 - exit 1 - fi - # The exact legacy digest binds omitted pre-region fields to US and protocol 0. - jq -r '[ - if .region == null then "region" else empty end, - if .regionalRehomeProtocol == null then "regionalRehomeProtocol" else empty end - ] | if length > 0 then "runtime predecessor normalized legacy fields=" + join(",") else empty end' \ - <<< "${CURRENT_RUNTIME}" - if ! CURRENT_DIRECTOR_STATUS="$(admin_post current-cell-status \ - "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ - "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"; then - if jq -e '.error == "cell_in_reserve_mode"' \ - "${RUNNER_TEMP}/current-cell-status.json" >/dev/null 2>&1; then - echo "${TARGET_CELL_ID} is in reserve mode: flip it to admitMode=db with the cell-flags workflow and wait for it to record db before rolling it, then flip it back to reserve" >&2 - fi - exit 1 - fi - SOURCE_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \ - <<< "${CURRENT_DIRECTOR_STATUS}")" - if test "${ROLLBACK_RESUME}" = true && ! jq -e \ - '.status.admissionState == "migration-only"' \ - <<< "${CURRENT_DIRECTOR_STATUS}" >/dev/null; then - echo 'resume requires the isolated migration-only cell a failed rollback leaves' >&2 - exit 1 - fi - [[ "${SOURCE_INCARNATION}" =~ ^[0-9a-f-]{36}$ ]] - echo "SOURCE_INCARNATION=${SOURCE_INCARNATION}" >> "${GITHUB_ENV}" - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat fresh --admission "${PRECHECK_ADMISSION}" \ - --draining "${PRECHECK_DRAINING}" --activity allowed \ - --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" - - # Hosts drained with nowhere to go keep redialling and pin the cell empty (c28, 2026-10-01). - # Every run that drains (apply, rollback) checks first; verify reads the same answer; a resume drains nothing. - - name: Require free general-cell slots for the selected cell's hosts - if: ${{ env.ROLLBACK_RESUME != 'true' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - HEADROOM_STATUS=0 - HEADROOM="$(node dev/scripts/check-relay-same-cap-headroom.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --general-cells "${EXPECTED_GENERAL_CELLS}")" \ - || HEADROOM_STATUS=$? - echo "${HEADROOM}" - test "${HEADROOM_STATUS}" = 0 - # The pre-drain sample sizes its window from this count. - TARGET_HOSTS="$(jq -er '.targetHosts' <<< "${HEADROOM}")" - echo "TARGET_HOSTS=${TARGET_HOSTS}" >> "${GITHUB_ENV}" - - # The last read before isolation, so a roll drains on health measured minutes ago, not on a - # separate monitor run's verdict. Its window grows with the hosts the drain will re-place; - # every later wave samples again. Any trip fails here, before anything is changed. Rollback - # skips it: getting off a crash-looping image must not wait for the crashes to stop. - - name: Sample fleet health for a window sized to this drain - if: ${{ inputs.mode == 'apply' }} - timeout-minutes: 20 - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} - run: | - pnpm incident:relay-pre-drain-sample -- \ - --target-cell-id "${TARGET_CELL_ID}" \ - --target-hosts "${TARGET_HOSTS}" \ - --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \ - --wave-index "${WAVE_INDEX}" \ - --selector-wave-delta "${SELECTOR_WAVE_DELTA}" - - - name: Finish read-only verification - if: ${{ inputs.mode == 'verify' }} - run: echo 'Exact same-cap rollback point verified.' - - # The sample can age the first one-hour token by up to 20 min; the drain's restart-safe - # wait (20 min at the default pace, 35 at the slowest) gets a fresh one. - - id: drain-auth - if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Reversibly isolate and drain only the selected cell - if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }} - id: drain - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.drain-auth.outputs.id_token }} - run: | - # Before any mutation: a cell whose image caps the pace below this one stops untouched. - node dev/scripts/prepare-relay-production-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode pace-check \ - --pace-window-ms "${DRAIN_PACE_WINDOW_MS}" - # Opens the window the report-only shadow health gate below judges this cell over. - echo "drain-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}" - echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" - # A cell isolated by a failed canary is already migration-only, so - # isolate is a no-op there that does not advance the selector; the - # result's generation is authoritative either way. - ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)" - echo "${ISOLATE_RESULT}" - # Isolating a migration-only cell must be a read-only no-op; a change here would - # mean the live class is not the one this wave planned around. - if test "${ENTRY_ADMISSION}" = migration-only; then - jq -e '.changed == false' <<< "${ISOLATE_RESULT}" >/dev/null - fi - # The director re-places hosts off a cell only when the isolate stamped it, so a - # script too old to ask for the stamp produces today's behaviour and the canary - # reads as "the fix did nothing" with nothing to tell that from a wrong premise. - jq -e '.rollIsolated == true' <<< "${ISOLATE_RESULT}" >/dev/null - ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")" - echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}" - DRAIN_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain \ - --pace-window-ms "${DRAIN_PACE_WINDOW_MS}")" - echo "${DRAIN_RESULT}" - # What the cell accepted, which is 0 when an older image fell back to an unpaced drain. - echo "drain-applied-pace-window-ms=$(jq -er '.paceWindowMs' <<< "${DRAIN_RESULT}")" \ - >> "${GITHUB_OUTPUT}" - # Restart-safe means an empty runtime that lasts past the drain's pace window; leases - # held by unplaceable hosts are reported, not waited on (c28, 2026-10-01). - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat either --admission migration-only --draining required \ - --activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \ - --pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \ - --timeout-ms "$((900000 + DRAIN_PACE_WINDOW_MS))" - echo "drain-settled-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}" - - - id: capacity-auth - if: ${{ inputs.mode != 'verify' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - - - name: Require converged Terraform state and a stable MIG on resume - if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }} - shell: bash - env: - CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} - run: | - # A cell on the root pool default emits no pool line, so pin one only where it exists. - POOL_ARGUMENTS=() - if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then - POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}") - fi - # Zero resource changes prove the prior run's apply completed and no - # restart will follow, keeping the incarnation check honest. Root - # outputs may lag a targeted apply, so judge resource_changes only. - # The cell backend service is deliberately not targeted: the capacity role has no - # compute.backendServices.update, so naming it fails the apply after the MIG has - # rolled. Its drain and log_config settings ship in a root apply by a principal - # that holds the permission. - terraform -chdir=infra/terraform plan \ - -var-file=environments/production.tfvars \ - -var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \ - "-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ - "-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ - -out="${RUNNER_TEMP}/relay-same-cap-resume.tfplan" - if ! terraform -chdir=infra/terraform show -json \ - "${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \ - | jq -e '[.resource_changes[]? - | select(.change.actions | any(. != "no-op" and . != "read"))] - | length == 0' >/dev/null - then - # An apply that failed before its template apply also resumes here - # (the cell still serves the rollback image), and repo drift since - # the cell's last roll (for example newly added rehome trust - # config) then legitimately replaces the template. Nothing is - # applied on resume either way, so accept exactly the drift the - # reviewed validator would let a real apply ship for the image the - # cell already serves: the template leaves and re-enters the - # rollback image, as exactly the template-and-MIG change pair. - terraform -chdir=infra/terraform show -json \ - "${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \ - | jq -r '"resume found unconverged resources: " + - ([.resource_changes[]? - | select(.change.actions | any(. != "no-op" and . != "read")) - | .address] | join(","))' - echo 'requiring reviewed rollback-image drift' - RESUME_REVIEW="$(terraform -chdir=infra/terraform show -json \ - "${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \ - | node dev/scripts/validate-relay-capacity-plan.mjs \ - --mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --image "${DESIRED_IMAGE}" \ - --rollback-image "${DESIRED_IMAGE}" \ - --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \ - --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ - --rehome-audience https://relay.onorca.dev/v1/admin/host-drain \ - --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \ - "${POOL_ARGUMENTS[@]}")" - echo "${RESUME_REVIEW}" - jq -e '.changes == 2' <<< "${RESUME_REVIEW}" >/dev/null - fi - gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 - - - name: Apply only the selected same-cap template and MIG - if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }} - id: apply - shell: bash - env: - CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} - run: | - # A cell on the root pool default emits no pool line, so pin one only where it exists. - POOL_ARGUMENTS=() - if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then - POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}") - fi - terraform -chdir=infra/terraform plan \ - -var-file=environments/production.tfvars \ - -var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \ - "-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ - "-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \ - -out="${RUNNER_TEMP}/relay-same-cap.tfplan" - PLAN_REVIEW="$(terraform -chdir=infra/terraform show -json \ - "${RUNNER_TEMP}/relay-same-cap.tfplan" \ - | node dev/scripts/validate-relay-capacity-plan.mjs \ - --mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \ - --rollback-image "${PLAN_ROLLBACK_IMAGE}" \ - --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \ - --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ - --rehome-audience https://relay.onorca.dev/v1/admin/host-drain \ - --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \ - "${POOL_ARGUMENTS[@]}")" - echo "${PLAN_REVIEW}" - # Stamped before the apply, not after it: the new container announces its listener while - # the MIG is still converging, so a bound taken at the end of this step is already past - # the announcement the shadow gate looks for. - echo "apply-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}" - terraform -chdir=infra/terraform apply -auto-approve \ - "${RUNNER_TEMP}/relay-same-cap.tfplan" - gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 - # A stranded cell already runs the reviewed template, so the apply above replaces - # no instance and the drain flag, which only a restart clears, would survive the - # whole wave. Fewer than the template-and-MIG pair means the template stayed put, - # including a MIG-only reconciliation. Recreate its one instance from the MIG's - # current template; a rolling action would rewrite the MIG's version name outside - # Terraform, and every later targeted plan would carry that revert. - if test "${ROLLBACK_STAGE}" = stranded \ - && jq -e '.changes < 2' <<< "${PLAN_REVIEW}" >/dev/null; then - STRANDED_INSTANCE="$(gcloud compute instance-groups managed list-instances \ - "${MIG_NAME}" --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \ - --format=json \ - | jq -er 'if length == 1 then .[0].instance | split("/") | last - else error("stranded cell MIG does not have exactly one instance") end')" - gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \ - --instances "${STRANDED_INSTANCE}" \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --quiet - gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 - fi - # Recorded for the operator comparing verdicts; the gate's boot search opens at the - # apply-started-at stamp above, not here. - echo "apply-completed-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}" - - - id: post-auth - if: ${{ inputs.mode != 'verify' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Verify new incarnation, exact image, protocol, and durable safety - if: ${{ inputs.mode != 'verify' }} - id: verify-target - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }} - run: | - # Rides out one DB stall: 5-7 s, then readiness grace and two 10 s LB - # health checks (c18, 10-07 18:49Z: 503 "no healthy upstream", healthy - # 16 s later). A cell still down after 60 s fails; other 4xx fail fast. - # Retried in this loop: the runner's 7.81 client (Ubuntu 22.04) never - # retries under --fail-with-body, so its --retry flags were inert. - admin_post() { - local out="${RUNNER_TEMP}/$1.json" window=60 delay=5 status - local deadline=$((SECONDS + window)) - while true; do - # A refused or timed-out attempt writes no body; never report the last one's. - rm -f "${out}" - status="$(curl --silent --show-error --max-time 30 --output "${out}" \ - --write-out '%{http_code}' --request POST "$2" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' --data "$3")" || status=000 - case "${status}" in - 2??) cat "${out}"; return 0 ;; - 000|408|429|5??) ((SECONDS + delay < deadline)) || break ;; - *) break ;; - esac - sleep "${delay}" - done - echo "admin_post $1: HTTP ${status}" >&2 - cat "${out}" >&2 2>/dev/null || true - return 1 - } - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat fresh --admission migration-only --draining forbidden \ - --activity allowed --expected-image-digests "${DESIRED_IMAGE_DIGEST}" \ - --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" --timeout-ms 900000 - TARGET_RUNTIME="$(admin_post target-runtime \ - "${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')" - jq -e --arg digest "${DESIRED_IMAGE_DIGEST}" \ - --argjson protocol "${DESIRED_REHOME_PROTOCOL}" \ - '.imageDigest == $digest and (.regionalRehomeProtocol // 0) == $protocol' \ - <<< "${TARGET_RUNTIME}" >/dev/null - TARGET_DIRECTOR_STATUS="$(admin_post target-cell-status \ - "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ - "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" - TARGET_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \ - <<< "${TARGET_DIRECTOR_STATUS}")" - if test "${ROLLBACK_RESUME}" = true; then - echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" - # No restart happened; the incarnation legitimately stays put. - test "${TARGET_INCARNATION}" = "${SOURCE_INCARNATION}" - else - test "${TARGET_INCARNATION}" != "${SOURCE_INCARNATION}" - fi - echo "TARGET_INCARNATION=${TARGET_INCARNATION}" >> "${GITHUB_ENV}" - node dev/scripts/operate-relay-regional-rehome.mjs \ - --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \ - --expected-selector-generation "${SELECTOR_GENERATION_AFTER_ISOLATE}" \ - --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ - --expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \ - --expected-general-cells "${ISOLATED_GENERAL_CELLS}" \ - --expected-control-generation "${EXPECTED_REHOME_GENERATION}" \ - | jq -e '.control.enabled == false' >/dev/null - echo "verify-ended-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}" - - - name: Prove exact per-host trust and idempotent no-neighbor behavior - if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol != '0') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol != '0')) }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }} - run: | - node dev/scripts/probe-relay-rehome-trust.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-id "${TARGET_CELL_ID}" \ - --cell-incarnation "${TARGET_INCARNATION}" - - - name: Restore only the verified selected cell to its entry admission - if: ${{ inputs.mode != 'verify' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }} - run: | - echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" - # Activating a migration-only cell would promote it to a serving cell for good, so - # restore it with the idempotent isolate that reports the authoritative generation. - if test "${ENTRY_ADMISSION}" = migration-only; then - RESTORE_MODE=isolate - else - RESTORE_MODE=activate - fi - RESTORE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode "${RESTORE_MODE}")" - echo "${RESTORE_RESULT}" - SELECTOR_GENERATION_AFTER_RESTORE="$(jq -er '.generation' \ - <<< "${RESTORE_RESULT}")" - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat fresh --admission "${ENTRY_ADMISSION}" \ - --draining forbidden --activity allowed \ - --expected-image-digests "${DESIRED_IMAGE_DIGEST}" \ - --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" - node dev/scripts/operate-relay-regional-rehome.mjs \ - --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \ - --expected-selector-generation "${SELECTOR_GENERATION_AFTER_RESTORE}" \ - --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ - --expected-migration-only-cells "${RESTORED_MIGRATION_CELLS}" \ - --expected-general-cells "${RESTORED_GENERAL_CELLS}" \ - --expected-control-generation "${EXPECTED_REHOME_GENERATION}" \ - | jq -e '.control.enabled == false' >/dev/null - - # Report only: this evaluates the oracles an operator reads by hand after a cell and records - # what it would have called, so its verdicts can be compared with the operator's over a full - # roll before it is ever allowed to block. Two independent guarantees keep it inert: the - # script exits 0 on every verdict, and continue-on-error keeps even a crash off the job's - # outcome. The failsafe below therefore cannot fire on anything this step observes. - # - # It runs after the restore, not before it, for two reasons: the cell goes back into - # admission on exactly today's schedule rather than waiting out a minute of log reads, and - # the window it judges has closed by then, so Cloud Logging's ingestion lag is behind it. - # These are fleet oracles anyway; when this does gate, what it gates is the next cell. - - name: Shadow health gate (report only) - id: shadow-gate - if: ${{ inputs.mode != 'verify' }} - continue-on-error: true - # continue-on-error bounds this step's contribution to the job outcome, not its clock, - # and its reads are serialised. A timed-out step is a failed step, which continue-on-error - # absorbs; without this bound a Logging 429 storm or an expired credential makes every - # read cost its full retry budget and can push the job past timeout-minutes, and a - # cancelled job takes the whole wave with it. A healthy gate is already minutes of - # serial reads, so both bounds sit above that: the script settles at seven minutes and - # reaching this eight is the pathological case. Eight on top of a cell of ~14 minutes - # (~45 at the slowest pace) still leaves the job's 120 minutes intact. - timeout-minutes: 8 - env: - DRAIN_STARTED_AT: ${{ steps.drain.outputs.drain-started-at }} - DRAIN_SETTLED_AT: ${{ steps.drain.outputs.drain-settled-at }} - DRAIN_APPLIED_PACE_WINDOW_MS: ${{ steps.drain.outputs.drain-applied-pace-window-ms }} - APPLY_STARTED_AT: ${{ steps.apply.outputs.apply-started-at }} - APPLY_COMPLETED_AT: ${{ steps.apply.outputs.apply-completed-at }} - VERIFY_ENDED_AT: ${{ steps.verify-target.outputs.verify-ended-at }} - SHADOW_GATE_DIRECTORY: ${{ runner.temp }}/relay-same-cap-shadow-gate - SHADOW_GATE_NAME: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json - run: | - mkdir -p "${SHADOW_GATE_DIRECTORY}" - node dev/scripts/relay-same-cap-shadow-gate.mjs \ - --cell-id "${TARGET_CELL_ID}" \ - --cell-host "${TARGET_HOSTNAME}.relay.onorca.dev" \ - --project-id "${GCP_PROJECT_ID}" \ - --director-service orca-cloud-relay \ - --drain-started-at "${DRAIN_STARTED_AT}" \ - --apply-started-at "${APPLY_STARTED_AT}" \ - --apply-completed-at "${APPLY_COMPLETED_AT}" \ - --verify-ended-at "${VERIFY_ENDED_AT}" \ - --drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \ - --drain-applied-pace-window-ms "${DRAIN_APPLIED_PACE_WINDOW_MS}" \ - --drain-settled-at "${DRAIN_SETTLED_AT}" \ - --target-hosts "${TARGET_HOSTS:-}" \ - --summary-file "${GITHUB_STEP_SUMMARY}" \ - --output-file "${SHADOW_GATE_DIRECTORY}/${SHADOW_GATE_NAME}" - - - name: Publish the shadow health gate verdict - if: ${{ inputs.mode != 'verify' }} - continue-on-error: true - # One small JSON file; a retrying upload must not spend the wave's remaining minutes either. - timeout-minutes: 2 - uses: actions/upload-artifact@v4 - with: - name: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json - path: ${{ runner.temp }}/relay-same-cap-shadow-gate - if-no-files-found: warn - retention-days: 14 - overwrite: true - - - id: cleanup-auth - if: ${{ failure() && inputs.mode != 'verify' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Keep a failed cell isolated and rehome disabled - if: ${{ failure() && inputs.mode != 'verify' }} - continue-on-error: true - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }} - run: | - test "${MUTATION_STARTED:-false}" = true || exit 0 - ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)" - echo "${ISOLATE_RESULT}" - # Same reason as the isolate step above: a failed wave leaves this cell isolated - # deliberately, and the stamp is what lets its hosts leave. - jq -e '.rollIsolated == true' <<< "${ISOLATE_RESULT}" >/dev/null - # The isolate result carries the authoritative post-isolate generation; - # fixed offsets are wrong whenever an earlier isolate was a no-op. - FAILSAFE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")" - node dev/scripts/operate-relay-regional-rehome.mjs \ - --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \ - --expected-selector-generation "${FAILSAFE_GENERATION}" \ - --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ - --expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \ - --expected-general-cells "${ISOLATED_GENERAL_CELLS}" \ - --expected-control-generation "${EXPECTED_REHOME_GENERATION}" diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap.yml b/.github/workflows/cloud-deploy-relay-production-same-cap.yml deleted file mode 100644 index 82a84648c22..00000000000 --- a/.github/workflows/cloud-deploy-relay-production-same-cap.yml +++ /dev/null @@ -1,433 +0,0 @@ -name: Deploy Relay Production Same-Cap - -on: - workflow_dispatch: - inputs: - mode: - description: Verify, roll one canary, roll a bounded batch, or roll back - required: true - default: verify - type: choice - options: [verify, canary-apply, batch-apply, rollback] - cell-ids: - description: Ordered comma-separated serving cells; one canary or two to ten batch cells - required: true - type: string - target-image-digest: - description: Exact immutable compatibility image digest - required: true - type: string - rollback-image-digest: - description: Exact immutable currently serving rollback digest - required: true - type: string - target-rehome-protocol: - description: Exact target regional-rehome protocol - required: true - default: '1' - type: choice - options: ['0', '1', '3'] - rollback-rehome-protocol: - description: Exact rollback regional-rehome protocol - required: true - default: '0' - type: choice - options: ['0', '1', '3'] - expected-selector-generation: - description: Exact selector generation before the first cell - required: true - type: string - expected-existing-only-cells: - description: Exact existing-only membership, or none - required: true - type: string - expected-migration-only-cells: - description: Exact migration-only membership, or none - required: true - type: string - expected-general-cells: - description: Exact general membership, or none - required: true - type: string - expected-rehome-generation: - description: Exact durable regional-rehome control generation; it must be disabled - required: true - type: string - drain-pace-window-ms: - description: Window the cell spreads its drain over; below 300000 only for US general cells; any other value named in the confirmation - required: true - default: '300000' - type: choice - options: ['300000', '60000', '30000', '900000', '1200000'] - canary-run-id: - description: Successful same-code canary in this rehome control generation; reusable across batches - required: false - type: string - confirmation: - description: Exact digest-and-cell-bound mutation confirmation - required: false - type: string - -permissions: - actions: read - contents: read - id-token: write - -concurrency: - group: production-cloud-sql-rollout - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - gate: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - # Headroom for the full-history checkout the canary provenance check needs. - timeout-minutes: 15 - environment: production - outputs: - cells: ${{ steps.wave.outputs.cells }} - job-mode: ${{ steps.wave.outputs.job-mode }} - steps: - # Full history: the canary authority a batch verifies is sealed at an ancestor commit, and - # the provenance check fails closed on a commit a shallow clone left out. - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/setup-node@v4 - with: { node-version: 24, check-latest: true } - - - id: wave - env: - MODE: ${{ inputs.mode }} - CELL_IDS: ${{ inputs.cell-ids }} - TARGET_DIGEST: ${{ inputs.target-image-digest }} - ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }} - CONFIRMATION: ${{ inputs.confirmation }} - CANARY_RUN_ID: ${{ inputs.canary-run-id }} - DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }} - run: | - CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \ - --mode "${MODE}" --cell-ids "${CELL_IDS}" \ - --target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \ - --confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}" \ - --drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}")" - echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}" - if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then - echo 'job-mode=apply' >> "${GITHUB_OUTPUT}" - else - echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}" - fi - - - name: Download exact prior canary authority - if: ${{ inputs.mode == 'batch-apply' }} - uses: actions/download-artifact@v4 - with: - name: relay-same-cap-canary-${{ inputs.canary-run-id }} - path: ${{ runner.temp }}/relay-same-cap-canary - github-token: ${{ github.token }} - run-id: ${{ inputs.canary-run-id }} - - - name: Verify canary authority against this batch - if: ${{ inputs.mode == 'batch-apply' }} - env: - CANARY_RUN_ID: ${{ inputs.canary-run-id }} - CELL_IDS: ${{ inputs.cell-ids }} - TARGET_DIGEST: ${{ inputs.target-image-digest }} - ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }} - SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }} - REHOME_GENERATION: ${{ inputs.expected-rehome-generation }} - DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }} - run: | - node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \ - --file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \ - --commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \ - --cell-ids "${CELL_IDS}" \ - --target-digest "${TARGET_DIGEST}" \ - --rollback-digest "${ROLLBACK_DIGEST}" \ - --drain-pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \ - --selector-generation "${SELECTOR_GENERATION}" \ - --rehome-generation "${REHOME_GENERATION}" - - cell_1: - needs: gate - uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml - with: - mode: ${{ needs.gate.outputs.job-mode }} - target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[0] }} - target-image-digest: ${{ inputs.target-image-digest }} - rollback-image-digest: ${{ inputs.rollback-image-digest }} - target-rehome-protocol: ${{ inputs.target-rehome-protocol }} - rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - expected-rehome-generation: ${{ inputs.expected-rehome-generation }} - drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }} - wave-index: '0' - secrets: inherit - - cell_2: - if: ${{ needs.cell_1.result == 'success' && fromJSON(needs.gate.outputs.cells)[1] != null }} - needs: [gate, cell_1] - uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml - with: - mode: ${{ needs.gate.outputs.job-mode }} - target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[1] }} - target-image-digest: ${{ inputs.target-image-digest }} - rollback-image-digest: ${{ inputs.rollback-image-digest }} - target-rehome-protocol: ${{ inputs.target-rehome-protocol }} - rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - expected-rehome-generation: ${{ inputs.expected-rehome-generation }} - drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }} - wave-index: '1' - secrets: inherit - - cell_3: - if: ${{ needs.cell_2.result == 'success' && fromJSON(needs.gate.outputs.cells)[2] != null }} - needs: [gate, cell_2] - uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml - with: - mode: ${{ needs.gate.outputs.job-mode }} - target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[2] }} - target-image-digest: ${{ inputs.target-image-digest }} - rollback-image-digest: ${{ inputs.rollback-image-digest }} - target-rehome-protocol: ${{ inputs.target-rehome-protocol }} - rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - expected-rehome-generation: ${{ inputs.expected-rehome-generation }} - drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }} - wave-index: '2' - secrets: inherit - - cell_4: - if: ${{ needs.cell_3.result == 'success' && fromJSON(needs.gate.outputs.cells)[3] != null }} - needs: [gate, cell_3] - uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml - with: - mode: ${{ needs.gate.outputs.job-mode }} - target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[3] }} - target-image-digest: ${{ inputs.target-image-digest }} - rollback-image-digest: ${{ inputs.rollback-image-digest }} - target-rehome-protocol: ${{ inputs.target-rehome-protocol }} - rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - expected-rehome-generation: ${{ inputs.expected-rehome-generation }} - drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }} - wave-index: '3' - secrets: inherit - - cell_5: - if: ${{ needs.cell_4.result == 'success' && fromJSON(needs.gate.outputs.cells)[4] != null }} - needs: [gate, cell_4] - uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml - with: - mode: ${{ needs.gate.outputs.job-mode }} - target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[4] }} - target-image-digest: ${{ inputs.target-image-digest }} - rollback-image-digest: ${{ inputs.rollback-image-digest }} - target-rehome-protocol: ${{ inputs.target-rehome-protocol }} - rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - expected-rehome-generation: ${{ inputs.expected-rehome-generation }} - drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }} - wave-index: '4' - secrets: inherit - - cell_6: - if: ${{ needs.cell_5.result == 'success' && fromJSON(needs.gate.outputs.cells)[5] != null }} - needs: [gate, cell_5] - uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml - with: - mode: ${{ needs.gate.outputs.job-mode }} - target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[5] }} - target-image-digest: ${{ inputs.target-image-digest }} - rollback-image-digest: ${{ inputs.rollback-image-digest }} - target-rehome-protocol: ${{ inputs.target-rehome-protocol }} - rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - expected-rehome-generation: ${{ inputs.expected-rehome-generation }} - drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }} - wave-index: '5' - secrets: inherit - - cell_7: - if: ${{ needs.cell_6.result == 'success' && fromJSON(needs.gate.outputs.cells)[6] != null }} - needs: [gate, cell_6] - uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml - with: - mode: ${{ needs.gate.outputs.job-mode }} - target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[6] }} - target-image-digest: ${{ inputs.target-image-digest }} - rollback-image-digest: ${{ inputs.rollback-image-digest }} - target-rehome-protocol: ${{ inputs.target-rehome-protocol }} - rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - expected-rehome-generation: ${{ inputs.expected-rehome-generation }} - drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }} - wave-index: '6' - secrets: inherit - - cell_8: - if: ${{ needs.cell_7.result == 'success' && fromJSON(needs.gate.outputs.cells)[7] != null }} - needs: [gate, cell_7] - uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml - with: - mode: ${{ needs.gate.outputs.job-mode }} - target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[7] }} - target-image-digest: ${{ inputs.target-image-digest }} - rollback-image-digest: ${{ inputs.rollback-image-digest }} - target-rehome-protocol: ${{ inputs.target-rehome-protocol }} - rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - expected-rehome-generation: ${{ inputs.expected-rehome-generation }} - drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }} - wave-index: '7' - secrets: inherit - - cell_9: - if: ${{ needs.cell_8.result == 'success' && fromJSON(needs.gate.outputs.cells)[8] != null }} - needs: [gate, cell_8] - uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml - with: - mode: ${{ needs.gate.outputs.job-mode }} - target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[8] }} - target-image-digest: ${{ inputs.target-image-digest }} - rollback-image-digest: ${{ inputs.rollback-image-digest }} - target-rehome-protocol: ${{ inputs.target-rehome-protocol }} - rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - expected-rehome-generation: ${{ inputs.expected-rehome-generation }} - drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }} - wave-index: '8' - secrets: inherit - - cell_10: - if: ${{ needs.cell_9.result == 'success' && fromJSON(needs.gate.outputs.cells)[9] != null }} - needs: [gate, cell_9] - uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml - with: - mode: ${{ needs.gate.outputs.job-mode }} - target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[9] }} - target-image-digest: ${{ inputs.target-image-digest }} - rollback-image-digest: ${{ inputs.rollback-image-digest }} - target-rehome-protocol: ${{ inputs.target-rehome-protocol }} - rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - expected-rehome-generation: ${{ inputs.expected-rehome-generation }} - drain-pace-window-ms: ${{ inputs.drain-pace-window-ms }} - wave-index: '9' - secrets: inherit - - seal_canary: - if: ${{ inputs.mode == 'canary-apply' }} - needs: [gate, cell_1] - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 5 - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: { node-version: 24, check-latest: true } - - # Its pace checks are what let a batch drain faster than the default; a missing artifact - # seals UNVERIFIED, which only the default accepts. Named from the gate's normalized cell, - # as cell_1 uploaded it, so a padded cell-ids input cannot miss it. - - name: Download the canary cell's shadow health verdict - continue-on-error: true - uses: actions/download-artifact@v4 - with: - name: relay-same-cap-shadow-gate-${{ fromJSON(needs.gate.outputs.cells)[0] }}-${{ github.run_id }}.json - path: ${{ runner.temp }}/relay-same-cap-shadow-gate - - - name: Seal exact successful canary authority - run: | - mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary" - node dev/scripts/relay-production-same-cap-wave.mjs create-canary \ - --cell-id "${{ fromJSON(needs.gate.outputs.cells)[0] }}" \ - --target-digest "${{ inputs.target-image-digest }}" \ - --rollback-digest "${{ inputs.rollback-image-digest }}" \ - --confirmation "${{ inputs.confirmation }}" \ - --drain-pace-window-ms "${{ inputs.drain-pace-window-ms }}" \ - --shadow-report "${RUNNER_TEMP}/relay-same-cap-shadow-gate/relay-same-cap-shadow-gate-${{ fromJSON(needs.gate.outputs.cells)[0] }}-${GITHUB_RUN_ID}.json" \ - --commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \ - --selector-generation "${{ inputs.expected-selector-generation }}" \ - --rehome-generation "${{ inputs.expected-rehome-generation }}" \ - > "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" - - - uses: actions/upload-artifact@v4 - with: - name: relay-same-cap-canary-${{ github.run_id }} - path: ${{ runner.temp }}/relay-same-cap-canary/authority.json - retention-days: 30 - if-no-files-found: error - - # Every cell job re-enters the run's lease with release: 'false'; only this job frees it. - release_lease: - if: always() - needs: - - gate - - cell_1 - - cell_2 - - cell_3 - - cell_4 - - cell_5 - - cell_6 - - cell_7 - - cell_8 - - cell_9 - - cell_10 - - seal_canary - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 10 - environment: production - steps: - - uses: actions/checkout@v4 - - - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-terraform-state - object: terraform/state/cloud-sql-rollout/production.lock - release: 'true' diff --git a/.github/workflows/cloud-deploy-relay-production.yml b/.github/workflows/cloud-deploy-relay-production.yml deleted file mode 100644 index 6f6257cf176..00000000000 --- a/.github/workflows/cloud-deploy-relay-production.yml +++ /dev/null @@ -1,220 +0,0 @@ -name: Deploy Relay Production Candidate - -on: - workflow_dispatch: - inputs: - source-cell-id: - description: Existing Terraform cell ID to evacuate - required: true - type: string - target-cell-id: - description: Distinct Terraform candidate cell ID - required: true - type: string - mode: - description: Audit/preflight are read-only; recover/continue resume committed work; disable/enable/reset/execute mutate admission - required: true - default: preflight - type: choice - options: - - audit - - preflight - - recover-forward - - continue-evacuation - - disable-cell - - enable-empty-cell - - reset-empty-candidate - - execute - confirmation: - description: Enter RECOVER_FORWARD, CONTINUE_EVACUATION, DISABLE_CELL, ENABLE_CELL, RESET_CANDIDATE, or EVACUATE for the matching mutation - required: false - type: string - monitor-run-id: - description: Successful fresh dry-run monitor workflow run ID - required: false - type: string - monitor-run-attempt: - description: Exact dry-run monitor workflow attempt - required: false - type: string - -permissions: - actions: read - contents: read - id-token: write - -concurrency: - group: production-cloud-sql-rollout - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - candidate: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - environment: production - env: - GCP_PROJECT_ID: onorca-cloud - DIRECTOR_ORIGIN: https://relay.onorca.dev - ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain - SOURCE_CELL_ID: ${{ inputs.source-cell-id }} - TARGET_CELL_ID: ${{ inputs.target-cell-id }} - DEPLOY_MODE: ${{ inputs.mode }} - MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} - MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} - steps: - - uses: actions/checkout@v4 - - - name: Require fresh dry-run evidence reference - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} - run: | - [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]] - [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] - - - name: Download private dry-run evidence - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} - uses: actions/download-artifact@v4 - with: - name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ runner.temp }}/relay-monitor-evidence - github-token: ${{ github.token }} - run-id: ${{ inputs.monitor-run-id }} - - - uses: ./.github/actions/setup-pnpm - with: - package-json-file: cloud/package.json - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - run: pnpm install --frozen-lockfile - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_wrapper: false - - - name: Verify dry-run artifact before cloud authentication - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} - run: | - node dev/scripts/relay-monitor-evidence.mjs verify-restore \ - --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ - --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ - --run-id "${MONITOR_RUN_ID}" \ - --run-attempt "${MONITOR_RUN_ATTEMPT}" \ - --commit-sha "${GITHUB_SHA}" \ - --mode dry-run - - - name: Reject previously consumed dry-run evidence - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} - env: - GH_TOKEN: ${{ github.token }} - run: | - MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - COUNT="$(gh api \ - "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \ - --jq '.total_count')" - test "${COUNT}" = "0" - - - id: google-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-terraform-state - object: terraform/state/cloud-sql-rollout/production.lock - - - name: Require explicit mutation confirmation - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - if [[ "${DEPLOY_MODE}" = "execute" ]]; then - test "${CONFIRMATION}" = "EVACUATE" - elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then - test "${CONFIRMATION}" = "RECOVER_FORWARD" - elif [[ "${DEPLOY_MODE}" = "continue-evacuation" ]]; then - test "${CONFIRMATION}" = "CONTINUE_EVACUATION" - elif [[ "${DEPLOY_MODE}" = "disable-cell" ]]; then - test "${CONFIRMATION}" = "DISABLE_CELL" - elif [[ "${DEPLOY_MODE}" = "enable-empty-cell" ]]; then - test "${CONFIRMATION}" = "ENABLE_CELL" - else - test "${CONFIRMATION}" = "RESET_CANDIDATE" - fi - - - name: Read reviewed Terraform topology - run: | - node dev/scripts/infra.mjs init --env production - terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json" - RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)" - echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}" - - - name: Verify fresh dry-run evidence against live selector - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/relay-monitor-evidence.mjs verify-mutation \ - --directory "${RUNNER_TEMP}/relay-monitor-evidence" \ - --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ - --run-id "${MONITOR_RUN_ID}" \ - --run-attempt "${MONITOR_RUN_ATTEMPT}" \ - --commit-sha "${GITHUB_SHA}" \ - --mode dry-run \ - --mutation-mode "${DEPLOY_MODE}" \ - --source-cell-id "${SOURCE_CELL_ID}" \ - --director-origin "${DIRECTOR_ORIGIN}" - - - name: Recheck all live safety signals - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - pnpm incident:relay-preflight -- \ - --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" - - - name: Create single-use dry-run marker - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} - run: | - MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption" - printf '%s\n' "${GITHUB_RUN_ID}" \ - > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}" - - - name: Consume dry-run evidence - if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }} - uses: actions/upload-artifact@v4 - with: - name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - retention-days: 90 - if-no-files-found: error - - - name: Preflight or evacuate exact GCE candidate - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/deploy-relay-gce-candidate.mjs \ - --project "${GCP_PROJECT_ID}" \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --admin-audience "${ADMIN_AUDIENCE}" \ - --topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \ - --source-cell-id "${SOURCE_CELL_ID}" \ - --target-cell-id "${TARGET_CELL_ID}" \ - --runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \ - --mode "${DEPLOY_MODE}" diff --git a/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml b/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml deleted file mode 100644 index 832fb3bc03f..00000000000 --- a/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: Deploy Relay Staging GCE Candidate - -on: - workflow_dispatch: - inputs: - source-cell-id: - description: Existing Terraform cell ID to evacuate - required: true - type: string - target-cell-id: - description: Distinct Terraform candidate cell ID - required: true - type: string - mode: - description: Preflight is read-only; reset repairs an empty candidate; execute evacuates - required: true - default: preflight - type: choice - options: - - preflight - - reset-empty-candidate - - execute - confirmation: - description: Enter RESET_CANDIDATE for reset or EVACUATE for execute - required: false - type: string - -permissions: - contents: read - id-token: write - -concurrency: - group: relay-staging-mutation - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - candidate: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - environment: staging - env: - GCP_PROJECT_ID: onorca-cloud-staging - DIRECTOR_ORIGIN: https://relay-staging.onorca.dev - ADMIN_AUDIENCE: https://relay-staging.onorca.dev/v1/admin/drain - SOURCE_CELL_ID: ${{ inputs.source-cell-id }} - TARGET_CELL_ID: ${{ inputs.target-cell-id }} - DEPLOY_MODE: ${{ inputs.mode }} - steps: - - uses: actions/checkout@v4 - - - id: google-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-staging-terraform-state - object: terraform/state/cloud-sql-rollout/staging.lock - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_wrapper: false - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - name: Require explicit mutation confirmation - if: ${{ inputs.mode != 'preflight' }} - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - if [[ "${DEPLOY_MODE}" = "execute" ]]; then - test "${CONFIRMATION}" = "EVACUATE" - else - test "${CONFIRMATION}" = "RESET_CANDIDATE" - fi - - - name: Read reviewed Terraform topology - run: | - node dev/scripts/infra.mjs init --env staging - terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json" - RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)" - echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}" - - - name: Preflight or evacuate exact GCE candidate - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/deploy-relay-gce-candidate.mjs \ - --project "${GCP_PROJECT_ID}" \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --admin-audience "${ADMIN_AUDIENCE}" \ - --topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \ - --source-cell-id "${SOURCE_CELL_ID}" \ - --target-cell-id "${TARGET_CELL_ID}" \ - --runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \ - --mode "${DEPLOY_MODE}" diff --git a/.github/workflows/cloud-deploy-relay-staging.yml b/.github/workflows/cloud-deploy-relay-staging.yml deleted file mode 100644 index 66b99096500..00000000000 --- a/.github/workflows/cloud-deploy-relay-staging.yml +++ /dev/null @@ -1,242 +0,0 @@ -name: Deploy Relay Staging - -on: - workflow_dispatch: - inputs: - expected-image-digest: - description: Exact checked-in production Relay sha256 digest to deploy - required: true - type: string - reserve-placement: - description: Preserve step-5 reserve placement, or set it; turning it off is refused while any cell is reserve - required: true - default: preserve - type: choice - options: [preserve, 'off', dry-run, 'on'] - expected-rehome-generation: - description: Exact durable regional-rehome generation; it must remain disabled - required: true - type: string - bootstrap-runtime-identity: - description: One-time move from the stamped-cell identity to the director identity - required: true - default: false - type: boolean - predecessor-image-digest: - description: Bootstrap only. Exact immutable serving predecessor digest - required: false - type: string - capacity-cell-id: - description: Register this cell's reviewed staging.tfvars cap and bound with the director - required: true - default: none - type: choice - options: [none, staging-gce-c1, staging-gce-c2, staging-gce-c3, staging-gce-c4] - min-instances: - description: Director floor; preserve keeps the serving revision's, so a reserve run's 2 survives - required: true - default: preserve - type: choice - options: [preserve, '0', '2'] - confirmation: - description: BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY for a bootstrap, ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE with a shape baseline - required: false - type: string - shape-baseline-revision: - description: Only after a Terraform apply changed the director template. The apply's 0% revision, whose shape the deploy adopts - required: false - type: string - shadow-seat-feed-cells: - description: Preserve the step-3 shadow seat-feed cells, or set all or a comma list of cell ids; dropping a reserve cell is refused - required: true - default: preserve - type: string - -permissions: - contents: read - id-token: write - -concurrency: - # Staging deploy, candidate, auth, and power operations must never overlap. - group: relay-staging-mutation - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - deploy: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - environment: staging - env: - GCP_PROJECT_ID: onorca-cloud-staging - GCP_REGION: ${{ vars.STAGING_GCP_REGION }} - DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging - REPOSITORY_ID: orca-cloud - IMAGE_NAME: relay - EXPECTED_IMAGE_DIGEST: ${{ inputs.expected-image-digest }} - CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - # The director must trust the identity every staging Relay workflow authenticates as. - DEPLOY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - ASIA_PROOF_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }} - REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled - DIRECTOR_RUNTIME_SERVICE_ACCOUNT: orca-cloud-staging-relay-dir@onorca-cloud-staging.iam.gserviceaccount.com - PREDECESSOR_RUNTIME_SERVICE_ACCOUNT: orca-cloud-staging-relay@onorca-cloud-staging.iam.gserviceaccount.com - REHOME_AUDIENCE: https://relay-staging.onorca.dev/v1/admin/host-drain - EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }} - BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }} - PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }} - CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }} - MIN_INSTANCES: ${{ inputs.min-instances }} - SHAPE_BASELINE_REVISION: ${{ inputs.shape-baseline-revision }} - steps: - - uses: actions/checkout@v4 - - - name: Require the expected immutable image and identity request - shell: bash - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - set -euo pipefail - [[ "${EXPECTED_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] - [[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] - [[ "${BOOTSTRAP_RUNTIME_IDENTITY}" =~ ^(true|false)$ ]] - [[ "${MIN_INSTANCES}" =~ ^(preserve|0|2)$ ]] - if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then - [[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] - test "${CAPACITY_CELL_ID}" = none - test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY - else - test -z "${PREDECESSOR_IMAGE_DIGEST}" - fi - if test -n "${SHAPE_BASELINE_REVISION}"; then - [[ "${SHAPE_BASELINE_REVISION}" =~ ^orca-cloud-relay-staging-[0-9]{5}-[a-z0-9]{3}$ ]] - test "${BOOTSTRAP_RUNTIME_IDENTITY}" = false - test "${CAPACITY_CELL_ID}" = none - test "${CONFIRMATION}" = ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE - fi - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.17.0 - terraform_wrapper: false - - - id: google-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Bind the request to the checked-in staging C4 image - shell: bash - run: | - set -euo pipefail - terraform -chdir=infra/terraform init -reconfigure \ - -backend-config=backend/staging.hcl -input=false - IMAGE="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars \ - <<< 'var.relay_gce_cells["staging-gce-c4"].image' | jq -er '.')" - test "${IMAGE}" = \ - "${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${EXPECTED_IMAGE_DIGEST}" - echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}" - if test "${CAPACITY_CELL_ID}" != none; then - # The deploy script refuses any topology change beyond this one cell's cap and bound. - DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars \ - <<< 'local.relay_director_cells_json' | jq -er '.')" - jq -e --arg cell "${CAPACITY_CELL_ID}" \ - 'any(.[]; .id == $cell and .connectionHardCap != null)' \ - <<< "${DESIRED_CELLS_JSON}" >/dev/null - echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" >> "${GITHUB_ENV}" - fi - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-staging-terraform-state - object: terraform/state/cloud-sql-rollout/staging.lock - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - name: Require the mirrored immutable image - run: | - DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \ - --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" - test "${DIGEST}" = "${EXPECTED_IMAGE_DIGEST}" - - - name: Deploy director blue/green - env: - # An image change reads every cell's admit mode before the traffic move. - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - RESERVE_PLACEMENT: ${{ inputs.reserve-placement }} - SHADOW_SEAT_FEED_CELLS: ${{ inputs.shadow-seat-feed-cells }} - run: | - regional_version="$(gcloud secrets versions describe latest \ - --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \ - --format='value(name)' | awk -F/ '{print $NF}')" - [[ "${regional_version}" =~ ^[1-9][0-9]*$ ]] - RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" - identity_args=() - if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then - identity_args=( - --bootstrap-runtime-identity true - --predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" - --predecessor-image-digest "${PREDECESSOR_IMAGE_DIGEST}" - ) - fi - # Omitted, the deploy script inherits the serving revision's floor. - floor_args=() - if test "${MIN_INSTANCES}" != preserve; then - floor_args=(--min-instances "${MIN_INSTANCES}") - fi - shape_args=() - if test -n "${SHAPE_BASELINE_REVISION}"; then - shape_args=(--shape-baseline-revision "${SHAPE_BASELINE_REVISION}") - fi - topology_args=() - if test "${CAPACITY_CELL_ID}" != none; then - topology_args=( - --capacity-cell-id "${CAPACITY_CELL_ID}" - --director-cells-json "${DESIRED_CELLS_JSON}" - ) - fi - node dev/scripts/deploy-relay-blue-green.mjs \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --service "${DIRECTOR_SERVICE_NAME}" \ - --image "${IMAGE}" \ - --role director \ - --max-instances 2 \ - --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \ - --deploy-service-account "${DEPLOY_SERVICE_ACCOUNT}" \ - --asia-proof-service-account "${ASIA_PROOF_SERVICE_ACCOUNT}" \ - --regional-placement-secret-version "${regional_version}" \ - "${floor_args[@]}" \ - --runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ - "${identity_args[@]}" \ - "${shape_args[@]}" \ - --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \ - --rehome-audience "${REHOME_AUDIENCE}" \ - --rehome-control-origin https://relay-staging.onorca.dev \ - --expected-rehome-generation "${EXPECTED_REHOME_GENERATION}" \ - "${topology_args[@]}" \ - --admin-audience https://relay-staging.onorca.dev/v1/admin/drain \ - --reserve-placement "${RESERVE_PLACEMENT}" \ - --shadow-seat-feed-cells "${SHADOW_SEAT_FEED_CELLS}" \ - --release-id "${RELEASE_ID}" - - - name: Smoke director health - run: | - URL="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(status.url)')" - node dev/scripts/smoke-relay.mjs "${URL}" diff --git a/.github/workflows/cloud-monitor-relay-clock-skew.yml b/.github/workflows/cloud-monitor-relay-clock-skew.yml deleted file mode 100644 index 6bcca3f8124..00000000000 --- a/.github/workflows/cloud-monitor-relay-clock-skew.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Monitor Relay Cell Clock Skew - -# Why: the 2026-08-01 sustained HOST_OFFLINE incident traced to one cell's -# clock running ~100ms ahead, which deterministically failed every host -# challenge under a zero-tolerance freshness check. /health and /ready cannot -# see clock skew; this monitor alarms before drift reaches the (now 2s) -# client tolerance. - -on: - workflow_dispatch: - schedule: - - cron: '17 * * * *' - -permissions: - contents: read - -defaults: - run: - working-directory: cloud - -jobs: - skew: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - sparse-checkout: cloud/infra/terraform/environments/production.tfvars - sparse-checkout-cone-mode: false - persist-credentials: false - - - name: Measure Date-header skew for every relay cell - run: | - set -u - # Date headers carry whole seconds; compare floored seconds on both - # sides so perfect sync reads 0/±1 and never flaps. An absolute - # floor-skew of >= 2 means real drift of at least ~1s — approaching - # the client's 2s challenge tolerance. Millisecond-precision deltas - # come from the desktop's named-check logging when activations fail. - ALARM_S=2 - failures=0 - reachable=0 - unserved=0 - # Every relay_gce_cells hostname (the block relayOpsCellsFromTerraform reads), fenced - # cells included, so a newly declared cell is probed from its first run. - cells="$(awk ' - /^relay_gce_cells[ \t]*=[ \t]*\{/ { inside = 1; next } - inside && /^\}/ { exit } - inside && /^[ \t]*hostname[ \t]*=/ { split($0, quoted, "\""); print quoted[2] } - ' infra/terraform/environments/production.tfvars || true)" - if [ -z "${cells}" ]; then - echo "::error::read no relay cells from production.tfvars; monitor blind" - exit 1 - fi - for cell in ${cells}; do - url="https://${cell}.relay.onorca.dev/health" - # Why: *.relay.onorca.dev is a wildcard, so the load balancer answers with its own - # accurate Date for a fenced, dead, or never-provisioned cell. Timing that reads as - # perfect sync. Only an HTTP 200 is the relay process itself answering, so only a - # 200 carries a clock worth judging. - response="$(curl -sS -D - -o /dev/null --max-time 8 "${url}" 2>/dev/null || true)" - status="$(printf '%s' "${response}" | awk 'NR==1 {print $2}')" - header="$(printf '%s' "${response}" | tr -d '\r' | grep -i '^date:' || true)" - if [ "${status:-000}" != "200" ] || [ -z "${header}" ]; then - # Expected for the fenced cells; a dead unfenced cell is caught by the heartbeat - # and readiness alerts, not here. Named either way so it is never invisible. - echo "${cell}: not serving (status ${status:-none}); no relay clock to judge" - unserved=$((unserved + 1)) - continue - fi - reachable=$((reachable + 1)) - server_s="$(date -d "${header#*: }" +%s)" - local_s="$(date +%s)" - skew=$((server_s - local_s)) - abs=${skew#-} - if [ "${abs}" -ge "${ALARM_S}" ]; then - echo "::error::${cell}: clock skew ${skew}s reaches ±${ALARM_S}s alarm" - failures=$((failures + 1)) - else - echo "${cell}: skew ${skew}s" - fi - done - echo "cells serving: ${reachable}, not serving: ${unserved}, alarms: ${failures}" - if [ "${reachable}" -eq 0 ]; then - # Now meaningful: previously the load balancer answered for every name, so this - # could never fire and a total fleet outage reported "all healthy". - echo "::error::no relay cell is serving; monitor blind" - exit 1 - fi - exit "$((failures > 0 ? 1 : 0))" diff --git a/.github/workflows/cloud-monitor-relay-production-job.yml b/.github/workflows/cloud-monitor-relay-production-job.yml deleted file mode 100644 index 3bc5ba8294d..00000000000 --- a/.github/workflows/cloud-monitor-relay-production-job.yml +++ /dev/null @@ -1,216 +0,0 @@ -name: Monitor Relay Production Job - -on: - workflow_call: - inputs: - mode: - required: true - type: string - expected-selector-generation: - required: true - type: string - expected-existing-only-cells: - required: true - type: string - expected-migration-only-cells: - required: true - type: string - expected-general-cells: - required: true - type: string - migration-policy: - required: true - type: string - recovery-source-cell-id: - required: true - type: string - capacity-cell-id: - required: true - type: string - -permissions: - actions: read - contents: read - id-token: write - -defaults: - run: - working-directory: cloud - -jobs: - monitor: - if: ${{ github.ref == 'refs/heads/main' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 100 - environment: production - env: - EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }} - EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }} - EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }} - EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }} - MIGRATION_POLICY: ${{ inputs.migration-policy }} - RECOVERY_SOURCE_CELL_ID: ${{ inputs.recovery-source-cell-id }} - CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }} - INCIDENT_ID: relay-${{ github.run_id }}-${{ inputs.mode }} - MONITOR_MODE: ${{ inputs.mode }} - OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-incident - steps: - - uses: actions/checkout@v4 - - - uses: ./.github/actions/setup-pnpm - with: - package-json-file: cloud/package.json - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - run: pnpm install --frozen-lockfile - - - id: prior-attempt - if: ${{ github.run_attempt > 1 }} - run: echo "value=$((GITHUB_RUN_ATTEMPT - 1))" >> "${GITHUB_OUTPUT}" - - - name: Restore prior private monitor state - if: ${{ github.run_attempt > 1 }} - uses: actions/download-artifact@v4 - with: - name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ steps.prior-attempt.outputs.value }} - path: ${{ github.workspace }}/relay-incident - github-token: ${{ github.token }} - run-id: ${{ github.run_id }} - - - name: Verify restored state provenance - if: ${{ github.run_attempt > 1 }} - run: | - node dev/scripts/relay-monitor-evidence.mjs verify-restore \ - --directory "${OUTPUT_DIRECTORY}" \ - --incident-id "${INCIDENT_ID}" \ - --run-id "${GITHUB_RUN_ID}" \ - --run-attempt "${{ steps.prior-attempt.outputs.value }}" \ - --commit-sha "${GITHUB_SHA}" \ - --mode "${MONITOR_MODE}" - echo "RESTART_FLAG=--restart" >> "${GITHUB_ENV}" - - - id: google-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - name: Verify exact-audience admin identity - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)" - - - name: Run read-only relay dry-run - if: ${{ inputs.mode == 'dry-run' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - pnpm --filter @orca-cloud/relay-ops incident:monitor \ - --environment production \ - --incident-id "${INCIDENT_ID}" \ - --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ - --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ - --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ - --migration-policy "${MIGRATION_POLICY}" \ - --recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \ - --capacity-cell-id "${CAPACITY_CELL_ID}" \ - --interval-seconds 60 \ - --output-directory "${OUTPUT_DIRECTORY}" \ - --duration-minutes 15 \ - --pre-drain-dry-run \ - ${RESTART_FLAG:-} - - - name: Run first relay monitor segment - if: ${{ inputs.mode == 'monitor' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - pnpm --filter @orca-cloud/relay-ops incident:monitor \ - --environment production \ - --incident-id "${INCIDENT_ID}" \ - --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ - --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ - --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ - --migration-policy "${MIGRATION_POLICY}" \ - --recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \ - --capacity-cell-id "${CAPACITY_CELL_ID}" \ - --interval-seconds 60 \ - --output-directory "${OUTPUT_DIRECTORY}" \ - --duration-minutes 90 \ - --max-samples-this-run 45 \ - ${RESTART_FLAG:-} - - - id: google-auth-refresh - if: ${{ inputs.mode == 'monitor' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Run remaining relay monitor window - if: ${{ inputs.mode == 'monitor' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-refresh.outputs.id_token }} - run: | - node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)" - pnpm --filter @orca-cloud/relay-ops incident:monitor \ - --environment production \ - --incident-id "${INCIDENT_ID}" \ - --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ - --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ - --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ - --migration-policy "${MIGRATION_POLICY}" \ - --recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \ - --capacity-cell-id "${CAPACITY_CELL_ID}" \ - --interval-seconds 60 \ - --output-directory "${OUTPUT_DIRECTORY}" \ - --duration-minutes 90 \ - --restart - - - name: Seal private evidence provenance - if: ${{ always() }} - run: | - node dev/scripts/relay-monitor-evidence.mjs create \ - --directory "${OUTPUT_DIRECTORY}" \ - --incident-id "${INCIDENT_ID}" \ - --run-id "${GITHUB_RUN_ID}" \ - --run-attempt "${GITHUB_RUN_ATTEMPT}" \ - --commit-sha "${GITHUB_SHA}" \ - --mode "${MONITOR_MODE}" - - - name: Publish aggregate job summary - if: ${{ always() }} - run: | - if [[ -f "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" ]]; then - cat "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" >> "${GITHUB_STEP_SUMMARY}" - else - echo "Relay monitor failed before its first aggregate checkpoint." \ - >> "${GITHUB_STEP_SUMMARY}" - fi - - - name: Upload private aggregate evidence - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ github.workspace }}/relay-incident - if-no-files-found: error - retention-days: 14 diff --git a/.github/workflows/cloud-monitor-relay-production.yml b/.github/workflows/cloud-monitor-relay-production.yml deleted file mode 100644 index 402e2aa9f69..00000000000 --- a/.github/workflows/cloud-monitor-relay-production.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Monitor Relay Production - -on: - workflow_dispatch: - inputs: - mode: - description: Run the required 15-minute pre-drain gate or a 90-minute incident watch - required: true - default: dry-run - type: choice - options: - - dry-run - - monitor - expected-selector-generation: - description: Exact durable admission-selector generation - required: true - type: string - expected-existing-only-cells: - description: Exact comma-separated existing-only cells, or none - required: true - type: string - expected-migration-only-cells: - description: Exact comma-separated migration-only cells, or none - required: true - type: string - expected-general-cells: - description: Exact comma-separated general cells, or none - required: true - type: string - migration-policy: - description: Migration checks matched to the intended mutation - required: true - default: strict - type: choice - options: - - strict - - recover-forward - - capacity-transition - recovery-source-cell-id: - description: Existing-only recovery source cell, or none for strict monitoring - required: true - default: none - type: string - capacity-cell-id: - description: General cell for a capacity-transition monitor, or none - required: true - default: none - type: string - -permissions: - actions: read - contents: read - id-token: write - -concurrency: - group: production-cloud-sql-rollout - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - monitor: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - uses: ./.github/workflows/cloud-monitor-relay-production-job.yml - with: - mode: ${{ inputs.mode }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - migration-policy: ${{ inputs.migration-policy }} - recovery-source-cell-id: ${{ inputs.recovery-source-cell-id }} - capacity-cell-id: ${{ inputs.capacity-cell-id }} diff --git a/.github/workflows/cloud-operate-relay-asia-admission.yml b/.github/workflows/cloud-operate-relay-asia-admission.yml deleted file mode 100644 index b1dc8b5e198..00000000000 --- a/.github/workflows/cloud-operate-relay-asia-admission.yml +++ /dev/null @@ -1,558 +0,0 @@ -name: Operate Relay Asia Admission - -on: - workflow_dispatch: - inputs: - environment: - description: Target Relay environment - required: true - type: choice - options: [staging, production] - mode: - description: Inspect, initialize, verify, atomically register, promote, or roll back admission - required: true - default: verify - type: choice - options: [inspect, initialize, verify, register, configure, promote, rollback] - cell-ids: - description: Exact reviewed comma-separated cell wave (Asia, or a US cell at the Asia shape) - required: true - type: string - selector-generation: - description: Exact live selector generation; leave empty only for inspect - required: false - type: string - selector-membership-sha256: - description: Exact fingerprint printed by inspect; required only for initialize - required: false - type: string - selector-attempt-id: - description: Durable unique attempt ID; empty for inspect, verify, and configure - required: false - type: string - image-digest: - description: Expected compatible Relay sha256 digest - required: true - type: string - director-image-digest: - description: Director sha256 digest; required only for configure - required: false - type: string - evidence-run-id: - description: Staging evidence run ID for C27, C27 canary run ID for C28/C29; C30-C35 take none and each proves itself by its own canary - required: false - type: string - evidence-run-attempt: - description: Exact evidence workflow run attempt; required with an evidence run ID - required: false - type: string - confirmation: - description: Exact typed confirmation for a mutation - required: false - type: string - -permissions: - actions: read - contents: read - id-token: write - -concurrency: - group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }} - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - admission: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 30 - environment: ${{ inputs.environment }} - env: - DIRECTOR_ORIGIN: ${{ inputs.environment == 'production' && 'https://relay.onorca.dev' || 'https://relay-staging.onorca.dev' }} - AUTH_ORIGIN: ${{ inputs.environment == 'production' && 'https://login.onorca.dev' || 'https://auth-staging.onorca.dev' }} - DIRECTOR_SERVICE: ${{ inputs.environment == 'production' && 'orca-cloud-relay' || 'orca-cloud-relay-staging' }} - REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled - GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }} - GCP_REGION: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_REGION || vars.STAGING_GCP_REGION }} - DIRECTOR_MAX_INSTANCES: ${{ inputs.environment == 'production' && '5' || '2' }} - TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }} - TARGET_ENVIRONMENT: ${{ inputs.environment }} - OPERATION_MODE: ${{ inputs.mode }} - TARGET_CELL_IDS: ${{ inputs.cell-ids }} - EXPECTED_SELECTOR_GENERATION: ${{ inputs.selector-generation }} - EXPECTED_SELECTOR_MEMBERSHIP_SHA256: ${{ inputs.selector-membership-sha256 }} - SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }} - IMAGE_DIGEST: ${{ inputs.image-digest }} - DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }} - EVIDENCE_RUN_ID: ${{ inputs.evidence-run-id }} - EVIDENCE_RUN_ATTEMPT: ${{ inputs.evidence-run-attempt }} - OPERATION_CONFIRMATION: ${{ inputs.confirmation }} - DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - DEPLOY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - steps: - - uses: actions/checkout@v4 - - - name: Validate exact operation inputs before authentication - id: inputs - shell: bash - run: | - set -euo pipefail - test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}" - test -n "${DEPLOY_SERVICE_ACCOUNT}" - [[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] - evidence_kind=none - canary_cell=none - canary_region=asia-east2 - artifact_name=none - if test "${OPERATION_MODE}" = inspect; then - test -z "${EXPECTED_SELECTOR_GENERATION}" - test -z "${SELECTOR_ATTEMPT_ID}" - test -z "${OPERATION_CONFIRMATION}" - test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" - test -z "${DIRECTOR_IMAGE_DIGEST}" - else - [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] - fi - if test "${OPERATION_MODE}" = initialize; then - test "${EXPECTED_SELECTOR_GENERATION}" = 0 - [[ "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" =~ ^[a-f0-9]{64}$ ]] - test -z "${DIRECTOR_IMAGE_DIGEST}" - [[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] - test "${OPERATION_CONFIRMATION}" = INITIALIZE_ADMISSION_SELECTOR - elif test "${OPERATION_MODE}" = verify; then - test -z "${SELECTOR_ATTEMPT_ID}" - test -z "${OPERATION_CONFIRMATION}" - test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" - test -z "${DIRECTOR_IMAGE_DIGEST}" - elif test "${OPERATION_MODE}" = inspect; then - : - elif test "${OPERATION_MODE}" = configure; then - test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" - test -z "${SELECTOR_ATTEMPT_ID}" - [[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] - test "${OPERATION_CONFIRMATION}" = CONFIGURE_ASIA_DIRECTOR - else - test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" - test -z "${DIRECTOR_IMAGE_DIGEST}" - [[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] - case "${OPERATION_MODE}:${OPERATION_CONFIRMATION}" in - register:REGISTER_ASIA_MIGRATION_ONLY) ;; - promote:PROMOTE_ASIA_GENERAL) ;; - rollback:ROLLBACK_ASIA_MIGRATION_ONLY) ;; - *) echo "typed confirmation does not match the requested mutation" >&2; exit 1 ;; - esac - fi - if test "${TARGET_ENVIRONMENT}:${OPERATION_MODE}" = production:promote; then - case "${TARGET_CELL_IDS}" in - production-gce-c27) - evidence_kind=staging - artifact_name="relay-asia-staging-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}" - canary_cell=production-gce-c27 - ;; - production-gce-c28,production-gce-c29) - evidence_kind=c27 - artifact_name="relay-asia-c27-canary-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}" - ;; - production-gce-c30|production-gce-c31|production-gce-c34|production-gce-c35) - # No earlier proof binds a later cell's generation; its own canary below rolls it back on failure. - canary_cell="${TARGET_CELL_IDS}" - ;; - production-gce-c32|production-gce-c33) - # A US cell at the Asia shape proves itself the same way, with load aimed at its own region. - canary_cell="${TARGET_CELL_IDS}" - canary_region=us-central1 - ;; - *) echo "production promotion wave is not reviewed" >&2; exit 1 ;; - esac - fi - if test "${evidence_kind}" = none; then - test -z "${EVIDENCE_RUN_ID}" - test -z "${EVIDENCE_RUN_ATTEMPT}" - else - [[ "${EVIDENCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]] - [[ "${EVIDENCE_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] - fi - canary=false - if test "${canary_cell}" != none; then - canary=true - # Leaves room for the rollback attempt's -rollback suffix within 128 characters. - test "${#SELECTOR_ATTEMPT_ID}" -le 119 - fi - { - echo "evidence_kind=${evidence_kind}" - echo "artifact_name=${artifact_name}" - echo "canary=${canary}" - echo "canary_cell=${canary_cell}" - echo "canary_region=${canary_region}" - echo "canary_hostname=${canary_cell##*-}" - } >> "${GITHUB_OUTPUT}" - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - uses: ./.github/actions/setup-pnpm - with: - package-json-file: cloud/package.json - if: ${{ steps.inputs.outputs.canary == 'true' }} - - - name: Install exact canary dependencies - if: ${{ steps.inputs.outputs.canary == 'true' }} - run: pnpm install --frozen-lockfile - - - name: Build the canary Relay contract - if: ${{ steps.inputs.outputs.canary == 'true' }} - run: pnpm --filter @orca-cloud/relay-contract build - - - name: Download immutable rollout evidence - if: ${{ steps.inputs.outputs.evidence_kind != 'none' }} - uses: actions/download-artifact@v4 - with: - name: ${{ steps.inputs.outputs.artifact_name }} - path: ${{ runner.temp }}/relay-asia-input-evidence - github-token: ${{ github.token }} - run-id: ${{ inputs.evidence-run-id }} - - - name: Verify evidence provenance and rollout binding before authentication - if: ${{ steps.inputs.outputs.evidence_kind != 'none' }} - env: - GH_TOKEN: ${{ github.token }} - EVIDENCE_KIND: ${{ steps.inputs.outputs.evidence_kind }} - shell: bash - run: | - set -euo pipefail - run_json="${RUNNER_TEMP}/relay-asia-evidence-run.json" - verified="${RUNNER_TEMP}/relay-asia-evidence-verified" - gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${EVIDENCE_RUN_ID}/attempts/${EVIDENCE_RUN_ATTEMPT}" > "${run_json}" - evidence_commit_sha="$( - jq -er '.head_sha | select(type == "string" and test("^[a-f0-9]{40}$"))' "${run_json}" - )" - command=(node dev/scripts/relay-asia-rollout-evidence.mjs "verify-${EVIDENCE_KIND}" - --evidence "${RUNNER_TEMP}/relay-asia-input-evidence/evidence.json" - --run-json "${run_json}" - --commit-sha "${evidence_commit_sha}" - --image-digest "${IMAGE_DIGEST}" - --now "$(date -u +%Y-%m-%dT%H:%M:%SZ)" - --output "${verified}") - if test "${EVIDENCE_KIND}" = c27; then - command+=(--selector-generation "${EXPECTED_SELECTOR_GENERATION}") - fi - "${command[@]}" - test "$(< "${verified}")" = verified - - - id: auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ env.DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ env.DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: ${{ env.DIRECTOR_ORIGIN }}/v1/admin/drain - id_token_include_email: true - - - name: Require the exact director image before promotion - if: ${{ inputs.mode == 'promote' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - runtime="$(curl --fail-with-body --max-time 30 --request POST \ - "${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' --data '{"v":1}')" - test "$(jq -r '.role' <<< "${runtime}")" = director - test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}" - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }} - object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }} - if: ${{ inputs.mode == 'configure' || steps.inputs.outputs.canary == 'true' }} - - - uses: hashicorp/setup-terraform@v3 - if: ${{ inputs.mode == 'configure' }} - with: - terraform_version: 1.17.0 - terraform_wrapper: false - - - name: Run the exact generation-bound admission operation - id: admission-operation - if: ${{ inputs.mode != 'configure' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - run: | - result="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment "${TARGET_ENVIRONMENT}" \ - --mode "${OPERATION_MODE}" \ - --cell-ids "${TARGET_CELL_IDS}" \ - --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --expected-membership-sha256 "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" \ - --attempt-id "${SELECTOR_ATTEMPT_ID}" \ - --image-digest "${IMAGE_DIGEST}")" - generation="$(jq -er '.generation' <<< "${result}")" - states="$(jq -cS '.states // {}' <<< "${result}")" - membership="$(jq -cS '.membership // empty' <<< "${result}")" - membership_sha256="$(jq -r '.membershipSha256 // empty' <<< "${result}")" - echo "generation=${generation}" >> "${GITHUB_OUTPUT}" - result_dir="${RUNNER_TEMP}/relay-asia-admission-result" - mkdir -p "${result_dir}" - node dev/scripts/sanitize-relay-asia-admission-result.mjs \ - <<< "${result}" > "${result_dir}/result.json" - { - echo "### Relay Asia admission" - echo "- Mode: ${OPERATION_MODE}" - echo "- Cells: ${TARGET_CELL_IDS}" - echo "- Result generation: ${generation}" - echo "- States: \`${states}\`" - if test -n "${membership}"; then echo "- Membership: \`${membership}\`"; fi - if test -n "${membership_sha256}"; then - echo "- Membership SHA-256: \`${membership_sha256}\`" - fi - } >> "${GITHUB_STEP_SUMMARY}" - - - name: Verify the canary cell state and start the timed canary - id: canary-start - if: ${{ steps.inputs.outputs.canary == 'true' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }} - shell: bash - run: | - set -euo pipefail - # A later cell's launch cells were checked general by the promotion; verify reads only its digest. - case "${CANARY_CELL}" in - production-gce-c27) - verify_cells=production-gce-c27,production-gce-c28,production-gce-c29 - expected_states='{"production-gce-c27":"general","production-gce-c28":"migration-only","production-gce-c29":"migration-only"}' - ;; - production-gce-c30|production-gce-c31|production-gce-c32|production-gce-c33|production-gce-c34|production-gce-c35) - verify_cells="${CANARY_CELL}" - expected_states="{\"${CANARY_CELL}\":\"general\"}" - ;; - *) exit 1 ;; - esac - result="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment production \ - --mode verify \ - --cell-ids "${verify_cells}" \ - --expected-generation "${{ steps.admission-operation.outputs.generation }}" \ - --image-digest "${IMAGE_DIGEST}")" - test "$(jq -cS '.states' <<< "${result}")" = "$(jq -cS '.' <<< "${expected_states}")" - echo "verify_cells=${verify_cells}" >> "${GITHUB_OUTPUT}" - echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}" - - - name: Run a real five-minute canary control and splice - id: canary-load - if: ${{ steps.inputs.outputs.canary == 'true' }} - env: - CANARY_REGION: ${{ steps.inputs.outputs.canary_region }} - shell: bash - run: | - set -euo pipefail - log="${RUNNER_TEMP}/relay-asia-canary-load.jsonl" - report="${RUNNER_TEMP}/relay-asia-canary-load.json" - node dev/scripts/load-relay-controls.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --auth-origin "${AUTH_ORIGIN}" \ - --preferred-region "${CANARY_REGION}" \ - --relay-asia-load-principals 1 \ - --controls 1 \ - --splices 1 \ - --capacity-hard-cap 3000 \ - --ramp-seconds 0 \ - --duration-seconds 300 \ - --splice-hold-seconds 60 \ - --required-lease-horizons 2 > "${log}" - jq -cer 'select(.event == "relay_load_complete")' "${log}" | tail -n 1 > "${report}" - echo "ended_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}" - - - name: Collect regional, Relay SQL, and Cloud SQL canary evidence - if: ${{ steps.inputs.outputs.canary == 'true' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }} - CANARY_STARTED_AT: ${{ steps.canary-start.outputs.started_at }} - CANARY_VERIFY_CELLS: ${{ steps.canary-start.outputs.verify_cells }} - shell: bash - run: | - set -euo pipefail - result="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment production \ - --mode verify \ - --cell-ids "${CANARY_VERIFY_CELLS}" \ - --expected-generation "${{ steps.admission-operation.outputs.generation }}" \ - --image-digest "${IMAGE_DIGEST}")" - test "$(jq -r --arg cell "${CANARY_CELL}" '.states[$cell]' <<< "${result}")" = general - ended_at="${{ steps.canary-load.outputs.ended_at }}" - sleep 60 - output="${RUNNER_TEMP}/relay-asia-output-evidence" - logs="${RUNNER_TEMP}/relay-asia-canary-runtime-metrics.json" - mkdir -p "${output}" - gcloud logging read \ - "timestamp>=\"${CANARY_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \ - --project "${GCP_PROJECT_ID}" \ - --limit 20000 \ - --format json > "${logs}" - node dev/scripts/relay-asia-rollout-evidence.mjs create-canary \ - --cell-id "${CANARY_CELL}" \ - --repository "${GITHUB_REPOSITORY}" \ - --run-id "${GITHUB_RUN_ID}" \ - --run-attempt "${GITHUB_RUN_ATTEMPT}" \ - --commit-sha "${GITHUB_SHA}" \ - --image-digest "${IMAGE_DIGEST}" \ - --selector-generation "${{ steps.admission-operation.outputs.generation }}" \ - --started-at "${CANARY_STARTED_AT}" \ - --ended-at "${ended_at}" \ - --load-report "${RUNNER_TEMP}/relay-asia-canary-load.json" \ - --logs-json "${logs}" \ - --output "${output}/evidence.json" - jq -r '.metrics | to_entries[] | "- \(.key): \(.value)"' \ - "${output}/evidence.json" >> "${GITHUB_STEP_SUMMARY}" - - - name: Upload immutable canary evidence - id: canary-evidence-upload - if: ${{ steps.inputs.outputs.canary == 'true' }} - uses: actions/upload-artifact@v4 - with: - name: relay-asia-${{ steps.inputs.outputs.canary_hostname }}-canary-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/relay-asia-output-evidence/evidence.json - if-no-files-found: error - retention-days: 7 - - - name: Upload sanitized admission result - if: ${{ inputs.mode != 'configure' && steps.admission-operation.outcome == 'success' }} - uses: actions/upload-artifact@v4 - with: - name: relay-asia-admission-result-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/relay-asia-admission-result/result.json - if-no-files-found: error - retention-days: 7 - - - name: Return an unproven canary cell to migration-only - if: ${{ always() && steps.inputs.outputs.canary == 'true' && steps.admission-operation.outcome != 'skipped' && steps.canary-evidence-upload.outcome != 'success' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - CANARY_CELL: ${{ steps.inputs.outputs.canary_cell }} - shell: bash - run: | - set -euo pipefail - promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment production \ - --mode recover-promotion \ - --cell-ids "${CANARY_CELL}" \ - --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --attempt-id "${SELECTOR_ATTEMPT_ID}" \ - --image-digest "${IMAGE_DIGEST}")" - if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi - promoted_generation="$(jq -er '.generation' <<< "${promoted}")" - result="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment production \ - --mode rollback \ - --cell-ids "${CANARY_CELL}" \ - --expected-generation "${promoted_generation}" \ - --attempt-id "${SELECTOR_ATTEMPT_ID}-rollback" \ - --image-digest "${IMAGE_DIGEST}")" - test "$(jq -r --arg cell "${CANARY_CELL}" '.states[$cell]' <<< "${result}")" = migration-only - - - name: Require registered migration-only cells before director configuration - if: ${{ inputs.mode == 'configure' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - run: | - result="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment "${TARGET_ENVIRONMENT}" \ - --mode registered \ - --cell-ids "${TARGET_CELL_IDS}" \ - --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --image-digest "${IMAGE_DIGEST}")" - test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true - - - name: Build the additive director cell configuration - if: ${{ inputs.mode == 'configure' }} - id: director-config - shell: bash - run: | - set -euo pipefail - terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}" - topology="${RUNNER_TEMP}/relay-asia-state-topology.json" - current="${RUNNER_TEMP}/relay-current-director-cells.json" - desired="${RUNNER_TEMP}/relay-asia-director-cells.json" - terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${topology}" - service="$(gcloud run services describe "${DIRECTOR_SERVICE}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" - revision="$(jq -er '[.status.traffic[] | select((.percent // 0) > 0)] | - if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end' \ - <<< "${service}")" - gcloud run revisions describe "${revision}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -er '.spec.containers[0].env[] | select(.name == "ORCA_RELAY_CELLS_JSON") | .value | fromjson' \ - > "${current}" - node dev/scripts/prepare-relay-asia-director-cells.mjs \ - --current-json "${current}" \ - --topology-json "${topology}" \ - --output "${desired}" \ - --cell-ids "${TARGET_CELL_IDS}" \ - --image-digest "${IMAGE_DIGEST}" - echo "file=${desired}" >> "${GITHUB_OUTPUT}" - - - name: Deploy the registered additive director topology - if: ${{ inputs.mode == 'configure' }} - env: - DIRECTOR_CELLS_FILE: ${{ steps.director-config.outputs.file }} - run: | - current="$(gcloud secrets versions access latest \ - --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")" - [[ "${current}" =~ ^(true|false)$ ]] - image="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}" - release_id="asia-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" - node dev/scripts/deploy-relay-blue-green.mjs \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --service "${DIRECTOR_SERVICE}" \ - --image "${image}" \ - --role director \ - --max-instances "${DIRECTOR_MAX_INSTANCES}" \ - --release-id "${release_id}" \ - --director-cells-json "$(< "${DIRECTOR_CELLS_FILE}")" \ - --prune-revisions false - - - name: Verify selector and heartbeats after director configuration - if: ${{ inputs.mode == 'configure' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - run: | - result="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment "${TARGET_ENVIRONMENT}" \ - --mode verify \ - --cell-ids "${TARGET_CELL_IDS}" \ - --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --image-digest "${IMAGE_DIGEST}")" - test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true - revision="$(gcloud run services describe "${DIRECTOR_SERVICE}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -er '[.status.traffic[] | select((.percent // 0) > 0)] | - if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end')" - revision_json="$(gcloud run revisions describe "${revision}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" - jq -e --arg image "us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}" \ - '.spec.containers[0].image == $image' <<< "${revision_json}" > /dev/null - jq -er --arg secret "${REGIONAL_PLACEMENT_SECRET}" \ - '[.spec.containers[0].env[] | - select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") | - (.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) | - {secret: (.secret // .name), version: (.version // .key)} | - select(.secret == $secret and (.version | test("^[1-9][0-9]*$")))] | - if length == 1 then .[0].version else error("regional switch version missing") end' \ - <<< "${revision_json}" > "${RUNNER_TEMP}/relay-regional-placement-version" - regional_version="$(< "${RUNNER_TEMP}/relay-regional-placement-version")" - [[ "$(gcloud secrets versions access "${regional_version}" --project "${GCP_PROJECT_ID}" \ - --secret "${REGIONAL_PLACEMENT_SECRET}")" =~ ^(true|false)$ ]] - echo "Director configuration now lists the registered migration-only Asia cells." >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/cloud-operate-relay-production-cell-flags.yml b/.github/workflows/cloud-operate-relay-production-cell-flags.yml deleted file mode 100644 index 148ebb7fd48..00000000000 --- a/.github/workflows/cloud-operate-relay-production-cell-flags.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: Operate Relay Production Cell Flags - -# Writes one cell's runtime switch object and waits for the cell to report it applied. -# One cell per run; never fleet-wide. dry-run writes nothing. -on: - workflow_dispatch: - inputs: - mode: - description: dry-run prints the change; write applies it - required: true - default: dry-run - type: choice - options: [dry-run, write] - cell-id: - description: Exact cell id, e.g. production-gce-c26 - required: true - type: string - break-glass: - description: >- - Only for a dead or unreachable reserve cell, with set admitMode=default. Typed as - BREAK GLASS ; records db without the cell's read-back - required: false - default: '' - type: string - confirm-reserve: - description: Required for admitMode=reserve, typed as RESERVE - required: false - default: '' - type: string - set: - description: >- - Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a - key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck, - admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs, - reregisterInFlight, phonePath (memory only takes effect in reserve; a reserve flip - must name phonePath) - required: true - type: string - expected-generation: - description: Exact current object generation from a dry run, or 0 if none exists - required: true - type: string - confirmation: - description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS " - required: false - type: string - -permissions: - contents: read - id-token: write - -# One write at a time across the fleet keeps the audit trail in order. -concurrency: - group: relay-production-cell-flags - cancel-in-progress: false - -defaults: - run: - # Explicit bash runs with pipefail, so the script's exit code survives the tee. - shell: bash - working-directory: cloud - -jobs: - operate: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - # A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls. - timeout-minutes: 60 - environment: production - env: - MODE: ${{ inputs.mode }} - CELL_ID: ${{ inputs.cell-id }} - FLAG_CHANGES: ${{ inputs.set }} - EXPECTED_GENERATION: ${{ inputs.expected-generation }} - CONFIRMATION: ${{ inputs.confirmation }} - CONFIRM_RESERVE: ${{ inputs.confirm-reserve }} - BREAK_GLASS: ${{ inputs.break-glass }} - GCP_PROJECT_ID: onorca-cloud - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - id: google-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - name: Write or preview the cell's switch object - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - [[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]] - ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)" - echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}" - export ORCA_RELAY_CONTROL_ACCESS_TOKEN - node dev/scripts/operate-relay-cell-flags.mjs \ - --mode "${MODE}" --cell-id "${CELL_ID}" \ - --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \ - --project-id "${GCP_PROJECT_ID}" \ - --set "${FLAG_CHANGES}" \ - --director-origin "https://relay.onorca.dev" --confirm-reserve "${CONFIRM_RESERVE}" \ - --break-glass "${BREAK_GLASS}" \ - --expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \ - | tee "${RUNNER_TEMP}/relay-cell-flags.jsonl" - - - name: Audit summary - if: ${{ always() }} - run: | - { - echo "### Relay cell flags: ${MODE} ${CELL_ID}" - echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}" - echo '```json' - cat "${RUNNER_TEMP}/relay-cell-flags.jsonl" 2>/dev/null || echo '(no output)' - echo '```' - } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/cloud-operate-relay-production-rehome-job.yml b/.github/workflows/cloud-operate-relay-production-rehome-job.yml deleted file mode 100644 index 4dc45665083..00000000000 --- a/.github/workflows/cloud-operate-relay-production-rehome-job.yml +++ /dev/null @@ -1,342 +0,0 @@ -name: Operate Relay Production Rehome Job - -on: - workflow_call: - inputs: - mode: { required: true, type: string } - director-image-digest: { required: true, type: string } - rollback-image-digest: { required: true, type: string } - expected-selector-generation: { required: true, type: string } - expected-existing-only-cells: { required: true, type: string } - expected-migration-only-cells: { required: true, type: string } - expected-general-cells: { required: true, type: string } - expected-control-generation: { required: true, type: string } - not-before: { required: true, type: string } - rate-per-minute: { required: true, type: string } - preference-max-age-ms: { required: true, type: string } - host-cooldown-ms: { required: true, type: string } - drain-grace-ms: { required: true, type: string } - confirmation: { required: true, type: string } - monitor-run-id: { required: true, type: string } - monitor-run-attempt: { required: true, type: string } - -permissions: - actions: read - contents: read - id-token: write - -defaults: - run: - # `shell: bash` adds pipefail; without it `node ... | tee` reports tee's exit code and a - # thrown inspect/apply passed green (Aug 28-29 and Sep 3 2026 runs). - shell: bash - working-directory: cloud - -jobs: - control: - if: ${{ github.ref == 'refs/heads/main' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 30 - environment: production - env: - GCP_PROJECT_ID: onorca-cloud - GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} - DIRECTOR_SERVICE: orca-cloud-relay - DIRECTOR_ORIGIN: https://relay.onorca.dev - REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain - MODE: ${{ inputs.mode }} - DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }} - ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }} - EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }} - EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }} - EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }} - EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }} - EXPECTED_CONTROL_GENERATION: ${{ inputs.expected-control-generation }} - NOT_BEFORE: ${{ inputs.not-before }} - RATE_PER_MINUTE: ${{ inputs.rate-per-minute }} - PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }} - HOST_COOLDOWN_MS: ${{ inputs.host-cooldown-ms }} - DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }} - CONFIRMATION: ${{ inputs.confirmation }} - MONITOR_RUN_ID: ${{ inputs.monitor-run-id }} - MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }} - OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence - steps: - - name: Require exact reusable-workflow configuration - working-directory: . - env: - DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} - run: | - [[ "${MODE}" =~ ^(inspect|enable|pause|disable)$ ]] - [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] - [[ "${EXPECTED_CONTROL_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] - test -n "${DEPLOY_WIF}" - test -n "${DEPLOY_SERVICE_ACCOUNT}" - if [[ "${MODE}" =~ ^(inspect|enable)$ ]]; then - [[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] - [[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] - test -n "${GCP_REGION}" - test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" - fi - case "${MODE}" in - inspect) - test -z "${CONFIRMATION}" - ;; - enable) - test "${CONFIRMATION}" = ENABLE_REGIONAL_REHOMING - test "${RATE_PER_MINUTE}" = 10 - [[ "${NOT_BEFORE}" =~ ^[1-9][0-9]*$ ]] - ;; - pause) - test "${CONFIRMATION}" = PAUSE_REGIONAL_REHOMING - ;; - disable) - test "${CONFIRMATION}" = DISABLE_REGIONAL_REHOMING - ;; - esac - - # Full history: the monitor evidence this job verifies is sealed at an ancestor commit, - # and the provenance check fails closed on a commit a shallow clone left out. - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - id: google-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Apply emergency durable pause or disable before diagnostics - if: ${{ inputs.mode == 'pause' || inputs.mode == 'disable' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/operate-relay-regional-rehome.mjs \ - --mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \ - --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ - --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ - --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ - --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \ - --not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \ - --preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \ - --host-cooldown-ms "${HOST_COOLDOWN_MS}" \ - --drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \ - | tee "${RUNNER_TEMP}/relay-rehome-control.json" - - - uses: ./.github/actions/setup-pnpm - if: ${{ inputs.mode == 'enable' }} - with: { package-json-file: cloud/package.json } - - - run: pnpm install --frozen-lockfile - if: ${{ inputs.mode == 'enable' }} - - - name: Download fresh aggregate safety evidence - if: ${{ inputs.mode == 'enable' }} - uses: actions/download-artifact@v4 - with: - name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ github.workspace }}/relay-monitor-evidence - github-token: ${{ github.token }} - run-id: ${{ inputs.monitor-run-id }} - - - name: Verify enable evidence provenance - if: ${{ inputs.mode == 'enable' }} - run: | - [[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]] - [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] - node dev/scripts/relay-monitor-evidence.mjs verify-authority \ - --directory "${OUTPUT_DIRECTORY}" \ - --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \ - --run-id "${MONITOR_RUN_ID}" --run-attempt "${MONITOR_RUN_ATTEMPT}" \ - --commit-sha "${GITHUB_SHA}" --mode dry-run \ - --required-migration-policy strict - - - name: Reject previously consumed enable safety evidence - if: ${{ inputs.mode == 'enable' }} - env: - GH_TOKEN: ${{ github.token }} - run: | - MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - COUNT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \ - --jq '.total_count')" - test "${COUNT}" = 0 - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-terraform-state - object: terraform/state/cloud-sql-rollout/production.lock - - - name: Verify exact serving and rollback director identities - if: ${{ inputs.mode == 'inspect' || inputs.mode == 'enable' }} - env: - DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }} - run: | - SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" - SERVING_REVISION="$(jq -er \ - '[.status.traffic[] | select((.percent // 0) > 0)] | - if length == 1 and .[0].percent == 100 then .[0].revisionName - else error("director does not have one serving revision") end' \ - <<< "${SERVICE_JSON}")" - ROLLBACK_REVISION="$(jq -er \ - '[.status.traffic[] | select(.tag == "selector-rollback")] | - if length == 1 then .[0].revisionName else error("rollback tag missing") end' \ - <<< "${SERVICE_JSON}")" - verify_revision() { - local revision="$1" expected_digest="$2" - local json - json="$(gcloud run revisions describe "${revision}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" - test "$(jq -r '.spec.serviceAccountName' <<< "${json}")" = \ - "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" - test "$(jq -r '.spec.containers[0].image | split("@") | last' <<< "${json}")" = \ - "${expected_digest}" - test "$(jq -r '[.spec.containers[0].env[] | select(.name == - "ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT") | .value] | if length == 1 - then .[0] else empty end' <<< "${json}")" = "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" - test "$(jq -r '[.spec.containers[0].env[] | select(.name == - "ORCA_RELAY_REHOME_AUDIENCE") | .value] | if length == 1 then .[0] - else empty end' <<< "${json}")" = "${REHOME_AUDIENCE}" - } - verify_revision "${SERVING_REVISION}" "${DIRECTOR_IMAGE_DIGEST}" - verify_revision "${ROLLBACK_REVISION}" "${ROLLBACK_IMAGE_DIGEST}" - - - name: Seal 24-hour aggregate region observation evidence - if: ${{ inputs.mode == 'enable' }} - run: | - mkdir -p "${RUNNER_TEMP}/relay-region-observation" - # 6 director instances x 120 samples/hour x 25h = 18000; a clipped - # read empties the oldest hourly buckets and fails the seal. - gcloud logging read \ - 'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND jsonPayload.event="orca_relay_runtime_metrics" AND jsonPayload.role="director"' \ - --project "${GCP_PROJECT_ID}" --freshness=25h --limit=30000 --format=json \ - | node dev/scripts/relay-region-observation-evidence.mjs create \ - --commit-sha "${GITHUB_SHA}" \ - --director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \ - --selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --control-generation "${EXPECTED_CONTROL_GENERATION}" \ - > "${RUNNER_TEMP}/relay-region-observation/evidence.json" - - - name: Upload sealed 24-hour aggregate region evidence - if: ${{ inputs.mode == 'enable' }} - uses: actions/upload-artifact@v4 - with: - name: relay-region-observation-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/relay-region-observation/evidence.json - retention-days: 90 - if-no-files-found: error - - - name: Verify sealed 24-hour enable authority - if: ${{ inputs.mode == 'enable' }} - run: | - node dev/scripts/relay-region-observation-evidence.mjs verify \ - --file "${RUNNER_TEMP}/relay-region-observation/evidence.json" \ - --commit-sha "${GITHUB_SHA}" \ - --director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \ - --selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --control-generation "${EXPECTED_CONTROL_GENERATION}" - - - name: Recheck every aggregate safety signal before enable - if: ${{ inputs.mode == 'enable' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - pnpm incident:relay-preflight -- \ - --state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" - - - name: Seal single-use enable safety authority - if: ${{ inputs.mode == 'enable' }} - run: | - MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}" - mkdir -p "${RUNNER_TEMP}/relay-rehome-enable-authority" - printf '%s\n' "${GITHUB_RUN_ID}" \ - > "${RUNNER_TEMP}/relay-rehome-enable-authority/${MARKER_NAME}" - - - name: Consume enable safety evidence before durable mutation - if: ${{ inputs.mode == 'enable' }} - uses: actions/upload-artifact@v4 - with: - name: relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - path: ${{ runner.temp }}/relay-rehome-enable-authority/relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }} - retention-days: 90 - if-no-files-found: error - - - name: Inspect regional rehome control - if: ${{ inputs.mode == 'inspect' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/operate-relay-regional-rehome.mjs \ - --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \ - --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ - --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ - --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ - --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \ - | tee "${RUNNER_TEMP}/relay-rehome-control.json" - - - name: Apply exact durable regional rehome enable - if: ${{ inputs.mode == 'enable' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/operate-relay-regional-rehome.mjs \ - --mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \ - --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \ - --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \ - --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \ - --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \ - --not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \ - --preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \ - --host-cooldown-ms "${HOST_COOLDOWN_MS}" \ - --drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \ - | tee "${RUNNER_TEMP}/relay-rehome-control.json" - - - name: Read fresh aggregate completion and abort evidence - run: | - gcloud logging read \ - 'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND textPayload:"[orca-relay] regional rehome inventory"' \ - --project "${GCP_PROJECT_ID}" --freshness=15m --limit=20 --format=json \ - | node dev/scripts/relay-rehome-aggregate-evidence.mjs --max-age-ms 900000 \ - | tee "${RUNNER_TEMP}/relay-rehome-inventory.json" - - - name: Publish aggregate control evidence - run: | - { - echo '### Regional rehome control' - jq -r '"- mode: `\(.mode)`\n- generation: `\(.control.generation)`\n- enabled: `\(.control.enabled)`"' \ - "${RUNNER_TEMP}/relay-rehome-control.json" - jq -r '"- active: `\(.active)`\n- awaiting receipt: `\(.awaitingReceipt)`\n- target registered: `\(.targetRegistered)`\n- completed (24h): `\(.completedLast24Hours)`\n- aborted (24h): `\(.abortedLast24Hours)`\n- host not arrived (24h): `\(.hostNotArrivedLast24Hours // "not reported")`\n- oldest active age (ms): `\(.oldestActiveAgeMs // "none")`"' \ - "${RUNNER_TEMP}/relay-rehome-inventory.json" - } >> "${GITHUB_STEP_SUMMARY}" - - - name: Fail closed after an unsuccessful enable run - if: ${{ failure() && inputs.mode == 'enable' && steps.google-auth.outcome == 'success' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/operate-relay-regional-rehome.mjs \ - --mode recover-enable --director-origin "${DIRECTOR_ORIGIN}" \ - --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \ - --confirmation RECOVER_FAILED_REGIONAL_REHOME_ENABLE \ - | tee "${RUNNER_TEMP}/relay-rehome-enable-recovery.json" - jq -e \ - '.mode == "recover-enable" and .control.enabled == false' \ - "${RUNNER_TEMP}/relay-rehome-enable-recovery.json" >/dev/null diff --git a/.github/workflows/cloud-operate-relay-production-rehome.yml b/.github/workflows/cloud-operate-relay-production-rehome.yml deleted file mode 100644 index 0615b197c11..00000000000 --- a/.github/workflows/cloud-operate-relay-production-rehome.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: Operate Relay Production Rehome - -on: - workflow_dispatch: - inputs: - mode: - description: Inspect or apply the durable regional-rehome switch - required: true - default: inspect - type: choice - options: [inspect, enable, pause, disable] - director-image-digest: - description: Exact immutable serving director digest - required: true - type: string - rollback-image-digest: - description: Exact immutable selector-rollback director digest - required: true - type: string - expected-selector-generation: - description: Exact admission selector generation - required: true - type: string - expected-existing-only-cells: - description: Exact existing-only membership, or none - required: true - type: string - expected-migration-only-cells: - description: Exact migration-only membership, or none - required: true - type: string - expected-general-cells: - description: Exact general membership, or none - required: true - type: string - expected-control-generation: - description: Exact durable rehome generation - required: true - type: string - not-before: - description: Exact epoch milliseconds; ignored only by inspect - required: true - default: '0' - type: string - rate-per-minute: - description: Exact global host rate; initial enable is fixed at 10 - required: true - default: '10' - type: string - preference-max-age-ms: - description: Maximum fresh preference age - required: true - default: '86400000' - type: string - host-cooldown-ms: - description: Minimum gap between two rehomes of the same host - required: true - default: '604800000' - type: string - drain-grace-ms: - description: Per-host source drain grace - required: true - default: '3600000' - type: string - monitor-run-id: - description: Fresh successful aggregate dry-run required only by enable - required: false - type: string - monitor-run-attempt: - description: Exact monitor attempt required only by enable - required: false - type: string - confirmation: - description: ENABLE_REGIONAL_REHOMING, PAUSE_REGIONAL_REHOMING, or DISABLE_REGIONAL_REHOMING - required: false - type: string - -permissions: - actions: read - contents: read - id-token: write - -concurrency: - group: production-cloud-sql-rollout - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - operate: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - uses: ./.github/workflows/cloud-operate-relay-production-rehome-job.yml - with: - mode: ${{ inputs.mode }} - director-image-digest: ${{ inputs.director-image-digest }} - rollback-image-digest: ${{ inputs.rollback-image-digest }} - expected-selector-generation: ${{ inputs.expected-selector-generation }} - expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} - expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} - expected-general-cells: ${{ inputs.expected-general-cells }} - expected-control-generation: ${{ inputs.expected-control-generation }} - not-before: ${{ inputs.not-before }} - rate-per-minute: ${{ inputs.rate-per-minute }} - preference-max-age-ms: ${{ inputs.preference-max-age-ms }} - host-cooldown-ms: ${{ inputs.host-cooldown-ms }} - drain-grace-ms: ${{ inputs.drain-grace-ms }} - confirmation: ${{ inputs.confirmation }} - monitor-run-id: ${{ inputs.monitor-run-id }} - monitor-run-attempt: ${{ inputs.monitor-run-attempt }} - secrets: inherit diff --git a/.github/workflows/cloud-operate-relay-production-reserve-watch.yml b/.github/workflows/cloud-operate-relay-production-reserve-watch.yml deleted file mode 100644 index 78e24a1d567..00000000000 --- a/.github/workflows/cloud-operate-relay-production-reserve-watch.yml +++ /dev/null @@ -1,341 +0,0 @@ -name: Operate Relay Production Reserve Watch - -# Watches one cell already switched to reserve mode and flips it back to db on the first stop -# rule it trips (step 5 design §9). Manual only. dry-run evaluates and logs, writes nothing. -# An operator watches the run. A refused watch or a flip back that did not land raises an ::error:: -# annotation, as the clock-skew monitor does, and in write mode logs the line the reserve-watch -# alert pages on. Refuses to start while a rollout holds the Cloud SQL rollout lease. -on: - workflow_dispatch: - inputs: - mode: - description: dry-run logs what a trip would write; write flips the cell back on a trip - required: true - default: dry-run - type: choice - options: [dry-run, write] - cell-id: - description: Exact cell id already in reserve mode, e.g. production-gce-c26 - required: true - type: string - duration-minutes: - description: How long to watch, 5 to 300 - required: true - default: '120' - type: string - flipped-at: - description: >- - When the reserve (or phonePath=memory) write applied, ISO with a zone, e.g. - 2026-10-14T12:00:00Z. The non-drain 503 background and the phone rules' background are - the 10 minutes before it - required: false - default: '' - type: string - background-503-per-minute: - description: >- - Director non-drain 503s per minute before the flip (all 503s less drain-return deferrals - and a host's own early or duplicate re-dials, from the director's runtime metrics); - overrides the one measured before flipped-at. One of the two is required - required: false - default: '' - type: string - unverifiable-limit: - description: Consecutive samples with any unreadable signal that count as a trip - required: true - default: '3' - type: string - confirmation: - description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS " - required: false - type: string - -permissions: - contents: read - id-token: write - -# One watch per cell; why not the flag workflow's group: cloud/docs/relay-workflows.md. -concurrency: - group: relay-production-reserve-watch-${{ inputs.cell-id }} - cancel-in-progress: false - -defaults: - run: - # Explicit bash runs with pipefail, so the script's exit code survives the tee. - shell: bash - working-directory: cloud - -env: - MODE: ${{ inputs.mode }} - CELL_ID: ${{ inputs.cell-id }} - CONFIRMATION: ${{ inputs.confirmation }} - GCP_PROJECT_ID: onorca-cloud - -jobs: - watch: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - # Six 55-minute segments cover the longest watch; under the 360-minute job limit. - timeout-minutes: 350 - environment: production - outputs: - state: ${{ steps.outcome.outputs.state }} - verdict: ${{ steps.outcome.outputs.verdict }} - # phonePath when a step-6 phone rule tripped; anything else flips the cell back whole. - flip: ${{ steps.outcome.outputs.flip }} - env: - DURATION_MINUTES: ${{ inputs.duration-minutes }} - FLIPPED_AT: ${{ inputs.flipped-at }} - BACKGROUND_503_PER_MINUTE: ${{ inputs.background-503-per-minute }} - UNVERIFIABLE_LIMIT: ${{ inputs.unverifiable-limit }} - # One segment, after the auth step before it: fixed text, the inputs arrive as env vars. - WATCH_SEGMENT: | - set -euo pipefail - [[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]] - node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)" - # A phone-rule trip writes phonePath=db mid-watch, through the flag tool. - ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)" - echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}" - export ORCA_RELAY_CONTROL_ACCESS_TOKEN - node dev/scripts/watch-relay-reserve-cell.mjs watch \ - --mode "${MODE}" --cell-id "${CELL_ID}" \ - --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \ - --project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \ - --director-service orca-cloud-relay --confirmation "${CONFIRMATION}" \ - --duration-minutes "${DURATION_MINUTES}" --segment-minutes 55 \ - --flipped-at "${FLIPPED_AT}" --background-503-per-minute "${BACKGROUND_503_PER_MINUTE}" \ - --unverifiable-limit "${UNVERIFIABLE_LIMIT}" \ - --state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}" \ - | tee -a "${RUNNER_TEMP}/relay-reserve-watch.jsonl" - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - id: google-auth-1 - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - id: watch-1 - name: Watch segment 1 of 6 - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-1.outputs.id_token }} - run: bash -c "${WATCH_SEGMENT}" - - - id: google-auth-2 - if: ${{ steps.watch-1.outputs.state == 'watching' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - id: watch-2 - name: Watch segment 2 of 6 - if: ${{ steps.watch-1.outputs.state == 'watching' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-2.outputs.id_token }} - run: bash -c "${WATCH_SEGMENT}" - - - id: google-auth-3 - if: ${{ steps.watch-2.outputs.state == 'watching' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - id: watch-3 - name: Watch segment 3 of 6 - if: ${{ steps.watch-2.outputs.state == 'watching' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-3.outputs.id_token }} - run: bash -c "${WATCH_SEGMENT}" - - - id: google-auth-4 - if: ${{ steps.watch-3.outputs.state == 'watching' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - id: watch-4 - name: Watch segment 4 of 6 - if: ${{ steps.watch-3.outputs.state == 'watching' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-4.outputs.id_token }} - run: bash -c "${WATCH_SEGMENT}" - - - id: google-auth-5 - if: ${{ steps.watch-4.outputs.state == 'watching' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - id: watch-5 - name: Watch segment 5 of 6 - if: ${{ steps.watch-4.outputs.state == 'watching' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-5.outputs.id_token }} - run: bash -c "${WATCH_SEGMENT}" - - - id: google-auth-6 - if: ${{ steps.watch-5.outputs.state == 'watching' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - id: watch-6 - name: Watch segment 6 of 6 - if: ${{ steps.watch-5.outputs.state == 'watching' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-6.outputs.id_token }} - run: bash -c "${WATCH_SEGMENT}" - - - id: outcome - name: Record the watch outcome - if: ${{ always() }} - run: | - node dev/scripts/watch-relay-reserve-cell.mjs outcome \ - --state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}" - - # The documented flow writes reserve before dispatching, so a refusal can leave it unwatched. - - name: Alarm on a refused watch - if: ${{ always() && steps.outcome.outputs.state == 'refused' }} - env: - WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }} - run: | - node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind refused \ - --mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${WATCH_VERDICT}" - - - name: Audit summary - if: ${{ always() }} - env: - WATCH_STATE: ${{ steps.outcome.outputs.state }} - WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }} - run: | - { - echo "### Relay reserve watch: ${MODE} ${CELL_ID}" - echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}" - echo "Outcome: ${WATCH_STATE}: ${WATCH_VERDICT}" - echo '```json' - tail -n 20 "${RUNNER_TEMP}/relay-reserve-watch.jsonl" 2>/dev/null || echo '(no output)' - echo '```' - } >> "${GITHUB_STEP_SUMMARY}" - - flip-back: - needs: watch - # Missing data is never healthy: a watch that crashed, timed out or was cancelled (by an - # operator too) flips back like a trip. In dry-run this only plans the write. - if: ${{ always() && vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && needs.watch.result != 'skipped' && !contains(fromJSON('["completed","stopped","refused"]'), needs.watch.outputs.state) }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - # A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls. - timeout-minutes: 60 - environment: production - # Only the run's per-cell group above, which a running run never yields (cancel-in-progress off). - # Every step runs on always() too, so an ordinary cancel cannot skip the switch-file write; a force-cancel - # can, and leaves the cell in reserve for the operator to flip back by hand. A cancel during the - # lease wait can still kill the process before Postgres records db: the file says db, Postgres reserve. - env: - VERDICT: ${{ needs.watch.outputs.verdict || format('the watch ended {0} without a verdict', needs.watch.result) }} - FLIP: ${{ needs.watch.outputs.flip || 'admitMode' }} - steps: - - uses: actions/checkout@v4 - if: ${{ always() }} - with: - persist-credentials: false - - - uses: actions/setup-node@v4 - if: ${{ always() }} - with: - node-version: 24 - check-latest: true - - - id: google-auth - if: ${{ always() }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - if: ${{ always() }} - with: - skip_install: true - - - name: Flip the cell back to db, or preview it - if: ${{ always() }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - [[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]] - ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)" - echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}" - export ORCA_RELAY_CONTROL_ACCESS_TOKEN - node dev/scripts/watch-relay-reserve-cell.mjs flip-back \ - --mode "${MODE}" --cell-id "${CELL_ID}" \ - --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \ - --project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \ - --confirmation "${CONFIRMATION}" --verdict "${VERDICT}" --flip "${FLIP}" \ - | tee "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl" - - # The flip-back line is logged only once the write (or its dry-run plan) finished; without - # it a missing token, a failed login or a silent cell left the cell in reserve, unwatched. - - name: Alarm when the flip back did not land - if: ${{ always() }} - run: | - if grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"; then - exit 0 - fi - if [[ "${FLIP}" == phonePath ]]; then - echo "## FLIP BACK FAILED: ${CELL_ID} may still have phonePath=memory" >> "${GITHUB_STEP_SUMMARY}" - echo "Set phonePath=default by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}" - else - echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode" >> "${GITHUB_STEP_SUMMARY}" - echo "Flip it to db by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}" - fi - node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind flip-back-failed \ - --mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}" --flip "${FLIP}" - - - name: Audit summary - if: ${{ always() }} - run: | - { - echo "### Relay reserve watch flip back: ${MODE} ${CELL_ID}" - echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}" - echo "Verdict: ${VERDICT}" - echo '```json' - cat "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl" 2>/dev/null || echo '(no output)' - echo '```' - } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/cloud-operate-relay-staging-cell-flags.yml b/.github/workflows/cloud-operate-relay-staging-cell-flags.yml deleted file mode 100644 index 6e2f747f0b1..00000000000 --- a/.github/workflows/cloud-operate-relay-staging-cell-flags.yml +++ /dev/null @@ -1,129 +0,0 @@ -name: Operate Relay Staging Cell Flags - -# The staging twin of Operate Relay Production Cell Flags: writes one staging cell's runtime -# switch object and waits for the cell to report it applied. One cell per run. dry-run writes -# nothing. -on: - workflow_dispatch: - inputs: - mode: - description: dry-run prints the change; write applies it - required: true - default: dry-run - type: choice - options: [dry-run, write] - cell-id: - description: Exact cell id, e.g. staging-gce-c2 - required: true - type: string - break-glass: - description: >- - Only for a dead or unreachable reserve cell, with set admitMode=default. Typed as - BREAK GLASS ; records db without the cell's read-back - required: false - default: '' - type: string - confirm-reserve: - description: Required for admitMode=reserve, typed as RESERVE - required: false - default: '' - type: string - set: - description: >- - Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a - key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck, - admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs, - reregisterInFlight, phonePath (memory only takes effect in reserve; a reserve flip - must name phonePath) - required: true - type: string - expected-generation: - description: Exact current object generation from a dry run, or 0 if none exists - required: true - type: string - confirmation: - description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS " - required: false - type: string - -permissions: - contents: read - id-token: write - -# Not relay-staging-mutation: a switch write must run while a staging deploy or load run is -# in flight, and it touches neither Cloud SQL nor a revision. -concurrency: - group: relay-staging-cell-flags - cancel-in-progress: false - -defaults: - run: - # Explicit bash runs with pipefail, so the script's exit code survives the tee. - shell: bash - working-directory: cloud - -jobs: - operate: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: ubuntu-latest - # A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls. - timeout-minutes: 60 - environment: staging - env: - MODE: ${{ inputs.mode }} - CELL_ID: ${{ inputs.cell-id }} - FLAG_CHANGES: ${{ inputs.set }} - EXPECTED_GENERATION: ${{ inputs.expected-generation }} - CONFIRMATION: ${{ inputs.confirmation }} - CONFIRM_RESERVE: ${{ inputs.confirm-reserve }} - BREAK_GLASS: ${{ inputs.break-glass }} - GCP_PROJECT_ID: onorca-cloud-staging - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - id: google-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - name: Write or preview the cell's switch object - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - [[ "${CELL_ID}" =~ ^staging-gce-c[0-9]{1,3}$ ]] - ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)" - echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}" - export ORCA_RELAY_CONTROL_ACCESS_TOKEN - node dev/scripts/operate-relay-cell-flags.mjs \ - --mode "${MODE}" --cell-id "${CELL_ID}" \ - --cell-origin "https://${CELL_ID#staging-gce-}.relay-staging.onorca.dev" \ - --project-id "${GCP_PROJECT_ID}" \ - --set "${FLAG_CHANGES}" \ - --director-origin "https://relay-staging.onorca.dev" --confirm-reserve "${CONFIRM_RESERVE}" \ - --break-glass "${BREAK_GLASS}" \ - --expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \ - | tee "${RUNNER_TEMP}/relay-cell-flags.jsonl" - - - name: Audit summary - if: ${{ always() }} - run: | - { - echo "### Relay staging cell flags: ${MODE} ${CELL_ID}" - echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}" - echo '```json' - cat "${RUNNER_TEMP}/relay-cell-flags.jsonl" 2>/dev/null || echo '(no output)' - echo '```' - } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/cloud-power-relay-staging.yml b/.github/workflows/cloud-power-relay-staging.yml deleted file mode 100644 index 90134d01ffa..00000000000 --- a/.github/workflows/cloud-power-relay-staging.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: Power Relay Staging - -on: - schedule: - # A zero-activity guard makes this a no-op when an internal test is still running. - - cron: '0 9 * * *' - workflow_dispatch: - inputs: - mode: - description: Inspect, wake, or sleep the staging Relay data plane - required: true - default: status - type: choice - options: - - status - - wake - - sleep - wake-cells: - description: Wake configured admission cells, or include disabled candidate cells - required: true - default: configured - type: choice - options: - - configured - - all - confirmation: - description: Enter WAKE_STAGING or SLEEP_STAGING for a manual mutation - required: false - type: string - -permissions: - contents: read - id-token: write - -concurrency: - group: relay-staging-mutation - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - power: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - environment: staging - env: - POWER_MODE: ${{ github.event_name == 'schedule' && 'sleep' || inputs.mode }} - WAKE_CELLS: ${{ inputs.wake-cells || 'configured' }} - steps: - - uses: actions/checkout@v4 - - - id: google-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-staging-terraform-state - object: terraform/state/cloud-sql-rollout/staging.lock - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_wrapper: false - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - name: Require explicit manual mutation confirmation - if: ${{ github.event_name == 'workflow_dispatch' && inputs.mode != 'status' }} - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - if [[ "${POWER_MODE}" = "wake" ]]; then - test "${CONFIRMATION}" = "WAKE_STAGING" - else - test "${CONFIRMATION}" = "SLEEP_STAGING" - fi - - - name: Read reviewed staging topology - run: | - node dev/scripts/infra.mjs init --env staging - terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json" - - - name: Inspect or change staging power state - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/power-staging-relay.mjs \ - --mode "${POWER_MODE}" \ - --wake-cells "${WAKE_CELLS}" \ - --topology-file "${RUNNER_TEMP}/relay-gce-topology.json" diff --git a/.github/workflows/cloud-prove-relay-asia-staging.yml b/.github/workflows/cloud-prove-relay-asia-staging.yml deleted file mode 100644 index d471027e85f..00000000000 --- a/.github/workflows/cloud-prove-relay-asia-staging.yml +++ /dev/null @@ -1,500 +0,0 @@ -name: Prove Relay Asia Staging - -on: - workflow_dispatch: - inputs: - mode: - description: prove runs the fixed C4 proof; load runs the generator with free-form arguments - required: true - default: prove - type: choice - options: [prove, load] - image-digest: - description: Prove only. Exact immutable Relay digest deployed on staging C4 - required: false - type: string - selector-generation: - description: Prove only. Exact selector generation with C4 migration-only - required: false - type: string - promote-attempt-id: - description: Prove only. Durable unique C4 promotion attempt ID - required: false - type: string - rollback-attempt-id: - description: Prove only. Durable unique C4 rollback attempt ID - required: false - type: string - load-region: - description: >- - Load only. asia-east2 runs on the Asia load runner; asia-east2-hosted asks for asia-east2 - from a hosted runner; us-central1 runs on a hosted runner - required: false - default: asia-east2 - type: choice - options: [asia-east2, asia-east2-hosted, us-central1] - load-shards: - description: Load only. Generator processes, each with its own shard of the 32 load principals - required: false - default: '4' - type: choice - options: ['1', '2', '3', '4'] - load-runners: - description: >- - Load only, hosted runners only. Runner machines, each with load-shards generators; one - hosted runner holds about 2,000 connections to one cell - required: false - default: '1' - type: choice - options: ['1', '2'] - load-args: - description: >- - Load only. Per-shard generator arguments, e.g. --controls 150 --splices 15 - --reconnect-max-seconds 30 --duration-seconds 900. Origins, region, principals and - sharding are set by the workflow - required: false - default: '' - type: string - confirmation: - description: Enter PROVE_ASIA_STAGING (prove) or LOAD_RELAY_STAGING (load) - required: true - type: string - -permissions: - contents: read - id-token: write - -# A load run drives traffic only, so it must not queue behind (or hold up) the staging deploys -# and switch writes it is there to exercise. -concurrency: - group: ${{ inputs.mode == 'load' && 'relay-staging-load' || 'relay-staging-mutation' }} - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - prove: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') && inputs.mode == 'prove' }} - runs-on: [self-hosted, linux, x64, relay-asia-east2-load] - timeout-minutes: 75 - environment: staging - env: - GCP_PROJECT_ID: onorca-cloud-staging - DIRECTOR_ORIGIN: https://relay-staging.onorca.dev - AUTH_ORIGIN: https://auth-staging.onorca.dev - IMAGE_DIGEST: ${{ inputs.image-digest }} - INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }} - PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }} - ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }} - steps: - - uses: actions/checkout@v4 - - - name: Validate the exact staging proof request - shell: bash - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - set -euo pipefail - test "${CONFIRMATION}" = PROVE_ASIA_STAGING - [[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] - [[ "${INITIAL_SELECTOR_GENERATION}" =~ ^[1-9][0-9]*$ ]] - [[ "${PROMOTE_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] - [[ "${ROLLBACK_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] - test "${PROMOTE_ATTEMPT_ID}" != "${ROLLBACK_ATTEMPT_ID}" - - - id: auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-staging-terraform-state - object: terraform/state/cloud-sql-rollout/staging.lock - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - uses: ./.github/actions/setup-pnpm - with: - package-json-file: cloud/package.json - - - name: Require the exact staging director image before promotion - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - runtime="$(curl --fail-with-body --max-time 30 --request POST \ - "${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' --data '{"v":1}')" - test "$(jq -r '.role' <<< "${runtime}")" = director - test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}" - - - name: Install exact load-harness dependencies - run: pnpm install --frozen-lockfile - - - name: Build the Relay load-harness contract - run: pnpm --filter @orca-cloud/relay-contract build - - - name: Promote only staging C4 - id: promote - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - result="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment staging \ - --mode promote \ - --cell-ids staging-gce-c4 \ - --expected-generation "${INITIAL_SELECTOR_GENERATION}" \ - --attempt-id "${PROMOTE_ATTEMPT_ID}" \ - --image-digest "${IMAGE_DIGEST}")" - generation="$(jq -er '.generation' <<< "${result}")" - test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = general - echo "generation=${generation}" >> "${GITHUB_OUTPUT}" - echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}" - - - name: Run sharded two-horizon and mixed splice proofs - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof" - mkdir -p "${proof_dir}" - fd_limit="$(ulimit -n)" - if test "${fd_limit}" != unlimited; then - [[ "${fd_limit}" =~ ^[0-9]+$ ]] - test "${fd_limit}" -ge 4096 - fi - run_phase() { - phase="$1" - controls="$2" - splices="$3" - pids=() - stop_shards() { - for pid in "${pids[@]}"; do kill "${pid}" 2>/dev/null || true; done - for pid in "${pids[@]}"; do wait "${pid}" 2>/dev/null || true; done - } - trap stop_shards EXIT - for shard in 0 1 2 3; do - slow=0 - wedged=0 - boundary_args=() - request_unit_args=() - if test "${phase}" = launch && test "${shard}" = 0; then - slow=4 - wedged=1 - fi - if test "${phase}" = launch && test "${shard}" = 0; then - boundary_args=( - --region-behavior-probes 1 - --capacity-cell-id staging-gce-c4 - --capacity-cell-origin https://c4.relay-staging.onorca.dev - --capacity-unobserved-bound 60 - --rebind-probes 2 - --skip-rebind-overflow-check - ) - fi - node dev/scripts/load-relay-controls.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --auth-origin "${AUTH_ORIGIN}" \ - --preferred-region asia-east2 \ - --relay-asia-load-principals 32 \ - --controls "${controls}" \ - --splices "${splices}" \ - --slow-reader-splices "${slow}" \ - --wedged-reader-splices "${wedged}" \ - --capacity-hard-cap 3000 \ - "${boundary_args[@]}" \ - "${request_unit_args[@]}" \ - --phase-barrier-dir "${proof_dir}/${phase}-barrier" \ - --aggregate-controls "$((controls * 4))" \ - --aggregate-splices "$((splices * 4))" \ - --aggregate-reader-splices "$([[ "${phase}" = launch ]] && echo 5 || echo 0)" \ - --aggregate-reader-bytes "$([[ "${phase}" = launch ]] && echo 12582912 || echo 0)" \ - --required-lease-horizons 2 \ - --splice-ramp-seconds 120 \ - --max-generator-rss-growth-mib 512 \ - --ramp-seconds 180 \ - --duration-seconds 210 \ - --shard-count 4 \ - --shard-index "${shard}" \ - > "${proof_dir}/${phase}-${shard}.jsonl" & - pids+=("$!") - done - failed=0 - for pid in "${pids[@]}"; do - if ! wait "${pid}"; then failed=1; break; fi - done - if test "${failed}" = 1; then - stop_shards - for shard in 0 1 2 3; do - jq -cer 'select(.event == "relay_load_progress" or .event == "relay_load_complete") | - {event, shardIndex, active, peakActive, connected, connectionFailures, - rampConnectionFailures, connectionFailuresByReason, unexpectedCloses, - protocolErrors, refreshErrors, socketErrors, elapsedSeconds}' \ - "${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1 || true - done - trap - EXIT - return 1 - fi - trap - EXIT - for shard in 0 1 2 3; do - jq -cer 'select(.event == "relay_load_complete")' \ - "${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1 - done | jq -s . > "${proof_dir}/${phase}.json" - } - run_phase launch 5 5 - - - name: Collect and validate aggregate staging proof evidence - env: - PROOF_STARTED_AT: ${{ steps.promote.outputs.started_at }} - shell: bash - run: | - set -euo pipefail - proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof" - ended_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - sleep 60 - gcloud logging read \ - "timestamp>=\"${PROOF_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \ - --project "${GCP_PROJECT_ID}" --limit 20000 --format json \ - > "${proof_dir}/runtime-metrics.json" - node dev/scripts/relay-asia-rollout-evidence.mjs create-staging \ - --repository "${GITHUB_REPOSITORY}" \ - --run-id "${GITHUB_RUN_ID}" \ - --run-attempt "${GITHUB_RUN_ATTEMPT}" \ - --commit-sha "${GITHUB_SHA}" \ - --image-digest "${IMAGE_DIGEST}" \ - --selector-generation "${{ steps.promote.outputs.generation }}" \ - --started-at "${PROOF_STARTED_AT}" \ - --ended-at "${ended_at}" \ - --launch-report "${proof_dir}/launch.json" \ - --logs-json "${proof_dir}/runtime-metrics.json" \ - --output "${proof_dir}/evidence.json" - - - name: Return staging C4 to migration-only - if: ${{ always() && steps.promote.outcome != 'skipped' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment staging \ - --mode recover-promotion \ - --cell-ids staging-gce-c4 \ - --expected-generation "${INITIAL_SELECTOR_GENERATION}" \ - --attempt-id "${PROMOTE_ATTEMPT_ID}" \ - --image-digest "${IMAGE_DIGEST}")" - if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi - promoted_generation="$(jq -er '.generation' <<< "${promoted}")" - result="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment staging \ - --mode rollback \ - --cell-ids staging-gce-c4 \ - --expected-generation "${promoted_generation}" \ - --attempt-id "${ROLLBACK_ATTEMPT_ID}" \ - --image-digest "${IMAGE_DIGEST}")" - test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only - - - name: Upload immutable staging readiness evidence - if: ${{ success() }} - uses: actions/upload-artifact@v4 - with: - name: relay-asia-staging-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/relay-asia-staging-proof/evidence.json - if-no-files-found: error - retention-days: 7 - - recover: - if: ${{ always() && github.ref == 'refs/heads/main' && inputs.mode == 'prove' }} - needs: prove - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 15 - environment: staging - env: - IMAGE_DIGEST: ${{ inputs.image-digest }} - INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }} - PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }} - ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }} - steps: - - uses: actions/checkout@v4 - - - id: auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - name: Recover staging C4 with a fresh identity - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment staging \ - --mode recover-promotion \ - --cell-ids staging-gce-c4 \ - --expected-generation "${INITIAL_SELECTOR_GENERATION}" \ - --attempt-id "${PROMOTE_ATTEMPT_ID}" \ - --image-digest "${IMAGE_DIGEST}")" - if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi - promoted_generation="$(jq -er '.generation' <<< "${promoted}")" - result="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment staging \ - --mode rollback \ - --cell-ids staging-gce-c4 \ - --expected-generation "${promoted_generation}" \ - --attempt-id "${ROLLBACK_ATTEMPT_ID}" \ - --image-digest "${IMAGE_DIGEST}")" - test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only - - # The staging auth service mints relay load principals only for this workflow file, so the - # free-form staging load lives here rather than in a workflow of its own. - load: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') && inputs.mode == 'load' }} - runs-on: ${{ inputs.load-region == 'asia-east2' && fromJSON('["self-hosted","linux","x64","relay-asia-east2-load"]') || 'ubuntu-latest' }} - strategy: - # One runner's failure must not drop the other half of the load mid-run. - fail-fast: false - matrix: - runner: ${{ fromJSON(inputs.load-runners == '2' && '[0, 1]' || '[0]') }} - timeout-minutes: 180 - environment: staging - env: - DIRECTOR_ORIGIN: https://relay-staging.onorca.dev - AUTH_ORIGIN: https://auth-staging.onorca.dev - LOAD_REGION: ${{ inputs.load-region }} - LOAD_SHARDS: ${{ inputs.load-shards }} - LOAD_RUNNERS: ${{ inputs.load-runners }} - LOAD_RUNNER_INDEX: ${{ matrix.runner }} - LOAD_ARGS: ${{ inputs.load-args }} - steps: - - uses: actions/checkout@v4 - - - name: Validate the staging load request - shell: bash - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - set -euo pipefail - test "${CONFIRMATION}" = LOAD_RELAY_STAGING - [[ "${LOAD_REGION}" =~ ^(asia-east2|asia-east2-hosted|us-central1)$ ]] - [[ "${LOAD_SHARDS}" =~ ^[1-4]$ ]] - [[ "${LOAD_RUNNERS}" =~ ^[12]$ ]] - # The Asia runner is one machine; a second matrix job would only queue behind the first. - if test "${LOAD_RUNNERS}" != 1 && test "${LOAD_REGION}" = asia-east2; then - echo "load-runners 2 needs a hosted runner" >&2 - exit 1 - fi - read -r -a args <<< "${LOAD_ARGS}" - for arg in "${args[@]}"; do - [[ "${arg}" =~ ^[A-Za-z0-9=:/._,-]+$ ]] || { echo "unsupported characters in load-args" >&2; exit 1; } - case "${arg}" in - --target-origin|--director-origin|--auth-origin|--signing-key-file|--preferred-region|\ - --relay-asia-load-principals|--shard-count|--shard-index|--phase-barrier-dir) - echo "${arg} is set by the workflow" >&2 - exit 1 - ;; - esac - done - - - id: auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - uses: ./.github/actions/setup-pnpm - with: - package-json-file: cloud/package.json - - - name: Install exact load-harness dependencies - run: pnpm install --frozen-lockfile - - - name: Build the Relay load-harness contract - run: pnpm --filter @orca-cloud/relay-contract build - - - name: Run the sharded staging load - env: - # Probes that read director capacity need an admin token; it lasts about an hour. - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - load_dir="${RUNNER_TEMP}/relay-staging-load" - mkdir -p "${load_dir}" - fd_limit="$(ulimit -n)" - if test "${fd_limit}" != unlimited; then - [[ "${fd_limit}" =~ ^[0-9]+$ ]] - test "${fd_limit}" -ge 4096 - fi - read -r -a args <<< "${LOAD_ARGS}" - pids=() - stop_shards() { - for pid in "${pids[@]}"; do kill "${pid}" 2>/dev/null || true; done - for pid in "${pids[@]}"; do wait "${pid}" 2>/dev/null || true; done - } - trap stop_shards EXIT - for ((shard = 0; shard < LOAD_SHARDS; shard++)); do - node dev/scripts/load-relay-controls.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --auth-origin "${AUTH_ORIGIN}" \ - --preferred-region "${LOAD_REGION%-hosted}" \ - --relay-asia-load-principals 32 \ - --shard-count "$((LOAD_SHARDS * LOAD_RUNNERS))" \ - --shard-index "$((LOAD_RUNNER_INDEX * LOAD_SHARDS + shard))" \ - "${args[@]}" \ - > "${load_dir}/shard-${shard}.jsonl" & - pids+=("$!") - done - failed=0 - for pid in "${pids[@]}"; do - if ! wait "${pid}"; then failed=1; fi - done - trap - EXIT - for ((shard = 0; shard < LOAD_SHARDS; shard++)); do - jq -c 'select(.event == "relay_load_complete" or .event == "relay_load_progress")' \ - "${load_dir}/shard-${shard}.jsonl" | tail -n 1 || true - done - exit "${failed}" - - - name: Upload the load reports - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: relay-staging-load-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.runner }} - path: ${{ runner.temp }}/relay-staging-load/ - if-no-files-found: warn - retention-days: 7 diff --git a/.github/workflows/cloud-prove-relay-staging-capacity.yml b/.github/workflows/cloud-prove-relay-staging-capacity.yml deleted file mode 100644 index 1eb2265a3ef..00000000000 --- a/.github/workflows/cloud-prove-relay-staging-capacity.yml +++ /dev/null @@ -1,928 +0,0 @@ -name: Prove Relay Staging Capacity - -on: - workflow_dispatch: - inputs: - mode: - description: Verify or change C3 capacity, restore admission, or refresh empty Asia C4 - required: true - default: verify - type: choice - options: - - verify - - apply - - restore-admission - - refresh-asia-c4-image - expected-hard-cap: - description: Exact cap declared for staging-gce-c3 in the reviewed staging tfvars - required: true - default: '600' - type: choice - options: - - '1000' - - '600' - expected-unobserved-bound: - description: Exact bound declared for staging-gce-c3 in the reviewed staging tfvars - required: true - default: '60' - type: choice - options: - - '60' - - '0' - confirmation: - description: Exact confirmation required for a mutation - required: false - type: string - expected-selector-generation: - description: Exact staging selector generation for a C4 image refresh - required: false - type: string - predecessor-image-digest: - description: Exact current C4 sha256 digest - required: false - type: string - target-image-digest: - description: Exact desired C4 sha256 digest - required: false - type: string - -permissions: - contents: read - id-token: write - -concurrency: - group: relay-staging-mutation - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - capacity: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && inputs.mode != 'refresh-asia-c4-image' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - environment: staging - env: - GCP_PROJECT_ID: onorca-cloud-staging - GCP_REGION: ${{ vars.STAGING_GCP_REGION }} - DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging - DIRECTOR_ORIGIN: https://relay-staging.onorca.dev - CELL_ORIGIN: https://c3.relay-staging.onorca.dev - # These drains are unpaced, but restart-safe still waits out the production pace window - # from the drain, the same one the same-cap drain uses. - RESTART_SAFE_PACE_WINDOW_MS: '300000' - TARGET_CELL_ID: staging-gce-c3 - FALLBACK_CELL_ORIGIN: https://c2.relay-staging.onorca.dev - FALLBACK_CELL_ID: staging-gce-c2 - CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - EXPECTED_HARD_CAP: ${{ inputs.expected-hard-cap }} - EXPECTED_UNOBSERVED_BOUND: ${{ inputs.expected-unobserved-bound }} - steps: - - uses: actions/checkout@v4 - - - id: google-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-staging-terraform-state - object: terraform/state/cloud-sql-rollout/staging.lock - - - uses: hashicorp/setup-terraform@v3 - if: ${{ inputs.mode != 'restore-admission' }} - with: - terraform_wrapper: false - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - name: Initialize the exact staging backend - if: ${{ inputs.mode != 'restore-admission' }} - run: node dev/scripts/infra.mjs init --env staging - - - name: Require reviewed desired capacity and image - if: ${{ inputs.mode != 'restore-admission' }} - shell: bash - run: | - CAP_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_hard_cap" - BOUND_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_unobserved_bound" - IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image" - ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone" - DESIRED_CAP="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars <<< "${CAP_EXPRESSION}")" - DESIRED_BOUND="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars <<< "${BOUND_EXPRESSION}")" - DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars <<< "${IMAGE_EXPRESSION}" | jq -r '.')" - TARGET_ZONE="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars <<< "${ZONE_EXPRESSION}" | jq -r '.')" - MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \ - | jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')" - DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars \ - <<< 'local.relay_director_cells_json' | jq -r '.')" - CELL_ORIGIN="$(jq -r --arg cell "${TARGET_CELL_ID}" \ - '.[] | select(.id == $cell) | .url' <<< "${DESIRED_CELLS_JSON}")" - test "${DESIRED_CAP}" = "${EXPECTED_HARD_CAP}" - test "${DESIRED_BOUND}" = "${EXPECTED_UNOBSERVED_BOUND}" - [[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]] - [[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]] - [[ "${MIG_NAME}" =~ ^[a-z0-9-]+$ ]] - test "${CELL_ORIGIN}" = "https://c3.relay-staging.onorca.dev" - jq -e \ - --arg cell "${TARGET_CELL_ID}" \ - --arg fallback "${FALLBACK_CELL_ID}" \ - --argjson cap "${EXPECTED_HARD_CAP}" \ - --argjson bound "${EXPECTED_UNOBSERVED_BOUND}" \ - '(any(.[]; .id == $cell and .connectionHardCap == $cap and .connectionUnobservedBound == $bound)) and - (any(.[]; .id == $fallback and .connectionHardCap == 600 and .connectionUnobservedBound == 60))' \ - <<< "${DESIRED_CELLS_JSON}" >/dev/null - echo "DESIRED_IMAGE=${DESIRED_IMAGE}" >> "${GITHUB_ENV}" - echo "TARGET_ZONE=${TARGET_ZONE}" >> "${GITHUB_ENV}" - echo "MIG_NAME=${MIG_NAME}" >> "${GITHUB_ENV}" - echo "CELL_ORIGIN=${CELL_ORIGIN}" >> "${GITHUB_ENV}" - echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" >> "${GITHUB_ENV}" - - - name: Verify exact compatible director image - if: ${{ inputs.mode != 'restore-admission' }} - shell: bash - run: | - SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" - ACTIVE_REVISION="$(jq -r \ - '[.status.traffic[] | select((.percent // 0) > 0)] - | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \ - <<< "${SERVICE_JSON}")" - test -n "${ACTIVE_REVISION}" - ACTIVE_IMAGE="$(gcloud run revisions describe "${ACTIVE_REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --format='value(spec.containers[0].image)')" - test "${ACTIVE_IMAGE}" = "${DESIRED_IMAGE}" - echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}" >> "${GITHUB_ENV}" - - - name: Require explicit transition confirmation - if: ${{ inputs.mode == 'apply' }} - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: test "${CONFIRMATION}" = "FENCE_AND_TRANSITION_STAGING_C3" - - - name: Require explicit admission restore confirmation - if: ${{ inputs.mode == 'restore-admission' }} - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: test "${CONFIRMATION}" = "RESTORE_STAGING_C2_C3_GENERAL" - - - name: Require capacity identity and exact predecessor state - if: ${{ inputs.mode == 'apply' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - case "${EXPECTED_HARD_CAP}/${EXPECTED_UNOBSERVED_BOUND}" in - 1000/0) - PREDECESSOR_C3_CAP=600 - PREDECESSOR_C3_BOUND=60 - ;; - 1000/60) - PREDECESSOR_C3_CAP=1000 - PREDECESSOR_C3_BOUND=0 - ;; - 600/60) - PREDECESSOR_C3_CAP=1000 - PREDECESSOR_C3_BOUND=60 - ;; - *) - echo "Unsupported staging capacity transition" >&2 - exit 1 - ;; - esac - ACTIVE_REVISION="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -r ' - [.status.traffic[] | select((.percent // 0) > 0)] | - if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" - test -n "${ACTIVE_REVISION}" - CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -cer ' - [.spec.containers[0].env[]? | - select(.name == "ORCA_RELAY_CELLS_JSON") | .value] | - if length == 1 then .[0] | fromjson else error("missing director topology") end')" - PREDECESSOR_CELLS_JSON="$(jq -ce \ - --arg cell "${TARGET_CELL_ID}" \ - --argjson cap "${PREDECESSOR_C3_CAP}" \ - --argjson bound "${PREDECESSOR_C3_BOUND}" \ - 'map(if .id == $cell then . + { - connectionHardCap: $cap, - connectionUnobservedBound: $bound - } else . end)' <<< "${DESIRED_CELLS_JSON}")" - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${FALLBACK_CELL_ORIGIN}" \ - --cell-id "${FALLBACK_CELL_ID}" \ - --hard-cap 600 \ - --unobserved-bound 60 \ - --heartbeat fresh \ - --admission either \ - --draining forbidden \ - --activity allowed - if jq -ne \ - --argjson current "${CURRENT_CELLS_JSON}" \ - --argjson expected "${DESIRED_CELLS_JSON}" \ - '$current == $expected'; then - if node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat fresh \ - --admission general \ - --draining forbidden \ - --activity allowed; then - TRANSITION_PHASE=cell-active - elif node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat fresh \ - --admission migration-only \ - --draining forbidden \ - --activity allowed; then - TRANSITION_PHASE=cell-ready - else - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --heartbeat either \ - --admission migration-only \ - --draining required \ - --activity restart-safe \ - --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000 - TRANSITION_PHASE=director-ready - fi - else - jq -ne \ - --argjson current "${CURRENT_CELLS_JSON}" \ - --argjson expected "${PREDECESSOR_CELLS_JSON}" \ - '$current == $expected' - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${PREDECESSOR_C3_CAP}" \ - --unobserved-bound "${PREDECESSOR_C3_BOUND}" \ - --heartbeat fresh \ - --admission either \ - --draining either \ - --activity allowed - TRANSITION_PHASE=predecessor - fi - echo "TRANSITION_PHASE=${TRANSITION_PHASE}" >> "${GITHUB_ENV}" - - - name: Restore C2 as the safe placement fallback - if: ${{ inputs.mode == 'restore-admission' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${FALLBACK_CELL_ORIGIN}" \ - --cell-id "${FALLBACK_CELL_ID}" \ - --heartbeat fresh \ - --admission either \ - --draining forbidden \ - --activity allowed - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --mode restore-fallback \ - --general-cell-ids "${FALLBACK_CELL_ID}" - - - name: Require a healthy non-draining C3 before restoring it - if: ${{ inputs.mode == 'restore-admission' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat fresh \ - --admission either \ - --draining forbidden \ - --activity allowed - - - name: Require a healthy general C2 before isolating C3 - if: ${{ inputs.mode == 'apply' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - if test "${TRANSITION_PHASE}" = cell-active; then - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --mode restore-fallback \ - --general-cell-ids "${FALLBACK_CELL_ID}" - fi - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${FALLBACK_CELL_ORIGIN}" \ - --cell-id "${FALLBACK_CELL_ID}" \ - --hard-cap 600 \ - --unobserved-bound 60 \ - --heartbeat fresh \ - --admission general \ - --draining forbidden \ - --activity allowed - - - name: Reversibly isolate and drain C3 - if: ${{ inputs.mode == 'apply' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - if test "${TRANSITION_PHASE}" = cell-ready; then - exit 0 - fi - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --mode isolate - - - name: Verify restart-safe migration-only target - if: ${{ inputs.mode == 'apply' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - if test "${TRANSITION_PHASE}" = cell-ready; then - exit 0 - fi - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --heartbeat either \ - --admission migration-only \ - --draining required \ - --activity restart-safe \ - --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000 - - - name: Verify current capacity - if: ${{ inputs.mode == 'verify' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat fresh \ - --admission general \ - --draining forbidden \ - --activity allowed - - - name: Deploy reviewed director topology and remove pre-protocol revisions - if: ${{ inputs.mode == 'apply' }} - run: | - if test "${TRANSITION_PHASE}" != predecessor; then - echo "DIRECTOR_CONFIG_CHANGED=false" >> "${GITHUB_ENV}" - exit 0 - fi - RELEASE_ID="capacity-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}" - DEPLOY_RESULT="$(node dev/scripts/deploy-relay-blue-green.mjs \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --service "${DIRECTOR_SERVICE_NAME}" \ - --image "${ACTIVE_IMAGE}" \ - --role director \ - --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \ - --capacity-cell-id "${TARGET_CELL_ID}" \ - --director-cells-json "${DESIRED_CELLS_JSON}" \ - --prune-revisions true \ - --release-id "${RELEASE_ID}")" - echo "${DEPLOY_RESULT}" - DIRECTOR_CONFIG_CHANGED="$(jq -r '.topologyChanged' <<< "${DEPLOY_RESULT}")" - [[ "${DIRECTOR_CONFIG_CHANGED}" =~ ^(true|false)$ ]] - echo "DIRECTOR_CONFIG_CHANGED=${DIRECTOR_CONFIG_CHANGED}" >> "${GITHUB_ENV}" - - - name: Require fail-closed director transition - if: ${{ inputs.mode == 'apply' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - if test "${TRANSITION_PHASE}" = cell-ready; then - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat fresh \ - --admission migration-only \ - --draining forbidden \ - --activity allowed - exit 0 - fi - HEARTBEAT_EXPECTATION=either - if test "${DIRECTOR_CONFIG_CHANGED}" = true; then - HEARTBEAT_EXPECTATION=stale - fi - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat "${HEARTBEAT_EXPECTATION}" \ - --admission migration-only \ - --draining required \ - --activity restart-safe \ - --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000 - - - name: Plan and apply only the exact empty cell - if: ${{ inputs.mode == 'apply' }} - shell: bash - run: | - recreate_fixed_one_instance() { - local instance - instance="$(gcloud compute instance-groups managed list-instances \ - "${MIG_NAME}" \ - --project "${GCP_PROJECT_ID}" \ - --zone "${TARGET_ZONE}" \ - --format=json \ - | jq -er ' - if length == 1 and .[0].instanceStatus == "RUNNING" and - .[0].currentAction == "NONE" - then .[0].instance | split("/") | last - else error("capacity cell does not have one stable running instance") end')" - gcloud compute instance-groups managed recreate-instances \ - "${MIG_NAME}" \ - --instances "${instance}" \ - --project "${GCP_PROJECT_ID}" \ - --zone "${TARGET_ZONE}" \ - --quiet - } - - terraform -chdir=infra/terraform plan \ - -var-file=environments/staging.tfvars \ - '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c3"]' \ - '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]' \ - -out="${RUNNER_TEMP}/relay-capacity-cell.tfplan" - PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \ - "${RUNNER_TEMP}/relay-capacity-cell.tfplan" \ - | node dev/scripts/validate-relay-capacity-plan.mjs \ - --mode cell \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --image "${DESIRED_IMAGE}")" - echo "${PLAN_RESULT}" - CELL_PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")" - [[ "${CELL_PLAN_CHANGES}" =~ ^(0|2)$ ]] - if test "${TRANSITION_PHASE}" = cell-ready; then - test "${CELL_PLAN_CHANGES}" = 0 - elif test "${TRANSITION_PHASE}" = cell-active; then - test "${CELL_PLAN_CHANGES}" = 0 - recreate_fixed_one_instance - elif test "${CELL_PLAN_CHANGES}" = 0; then - recreate_fixed_one_instance - else - terraform -chdir=infra/terraform apply \ - -auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan" - fi - gcloud compute instance-groups managed wait-until \ - "${MIG_NAME}" \ - --stable \ - --project "${GCP_PROJECT_ID}" \ - --zone "${TARGET_ZONE}" \ - --timeout 900 - - - name: Verify exact live cap and fresh matching heartbeat - if: ${{ inputs.mode == 'apply' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat fresh \ - --admission migration-only \ - --draining forbidden \ - --activity allowed - - - name: Make C3 the only staging placement cell - if: ${{ inputs.mode == 'apply' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --mode activate - - - name: Verify the sole general canary after transition - if: ${{ inputs.mode == 'apply' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat fresh \ - --admission general \ - --draining forbidden \ - --activity allowed - - - name: Restore the reviewed C2 and C3 placement set - if: ${{ inputs.mode == 'restore-admission' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --mode restore \ - --general-cell-ids staging-gce-c2,staging-gce-c3 - - - name: Verify restored C3 admission and capacity - if: ${{ inputs.mode == 'restore-admission' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --hard-cap "${EXPECTED_HARD_CAP}" \ - --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ - --heartbeat fresh \ - --admission general \ - --draining forbidden \ - --activity allowed - - - name: Preserve C2 as the safe fallback after a failed transition - if: ${{ failure() && inputs.mode == 'apply' }} - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }} - run: | - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" \ - --mode restore-fallback \ - --general-cell-ids "${FALLBACK_CELL_ID}" - - refresh-asia-c4-image: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && github.ref == 'refs/heads/main' && inputs.mode == 'refresh-asia-c4-image' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 90 - environment: staging - env: - GCP_PROJECT_ID: onorca-cloud-staging - GCP_REGION: ${{ vars.STAGING_GCP_REGION }} - DIRECTOR_ORIGIN: https://relay-staging.onorca.dev - CELL_ORIGIN: https://c4.relay-staging.onorca.dev - TARGET_CELL_ID: staging-gce-c4 - EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }} - PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }} - TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }} - APPROVED_PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7 - steps: - - uses: actions/checkout@v4 - - - name: Require the exact bounded C4 refresh - env: - CONFIRMATION: ${{ inputs.confirmation }} - shell: bash - run: | - set -euo pipefail - test "${CONFIRMATION}" = REFRESH_STAGING_ASIA_C4_IMAGE - [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] - [[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] - [[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] - test "${PREDECESSOR_IMAGE_DIGEST}" != "${TARGET_IMAGE_DIGEST}" - test "${PREDECESSOR_IMAGE_DIGEST}" = "${APPROVED_PREDECESSOR_IMAGE_DIGEST}" - - - id: google-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-staging-terraform-state - object: terraform/state/cloud-sql-rollout/staging.lock - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.17.0 - terraform_wrapper: false - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - name: Initialize the exact staging backend - run: node dev/scripts/infra.mjs init --env staging - - - name: Resolve the reviewed C4 image and shape - shell: bash - run: | - set -euo pipefail - cells="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars \ - <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')" - shape="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${cells}")" - test "$(jq -r '.hostname' <<< "${shape}")" = c4 - test "$(jq -r '.region' <<< "${shape}")" = asia-east2 - test "$(jq -r '.zone' <<< "${shape}")" = asia-east2-a - test "$(jq -r '.machine_type' <<< "${shape}")" = e2-standard-4 - test "$(jq -r '.capacity_requests' <<< "${shape}")" = 6000 - test "$(jq -r '.database_pool_max' <<< "${shape}")" = 10 - test "$(jq -r '.connection_hard_cap' <<< "${shape}")" = 3000 - test "$(jq -r '.connection_unobserved_bound' <<< "${shape}")" = 60 - test "$(jq -r '.initially_enabled' <<< "${shape}")" = false - desired_image="$(jq -r '.image' <<< "${shape}")" - test "${desired_image}" = \ - "us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${TARGET_IMAGE_DIGEST}" - served_digest="$(gcloud artifacts docker images describe "${desired_image}" \ - --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" - test "${served_digest}" = "${TARGET_IMAGE_DIGEST}" - mig_name="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \ - | jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')" - test "${mig_name}" = orca-cloud-staging-relay-gce-c4 - { - echo "DESIRED_IMAGE=${desired_image}" - echo "ROLLBACK_IMAGE=us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${PREDECESSOR_IMAGE_DIGEST}" - echo "TARGET_ZONE=asia-east2-a" - echo "MIG_NAME=${mig_name}" - } >> "${GITHUB_ENV}" - - - name: Save, validate, and classify the exact C4 plan - id: plan - shell: bash - run: | - set -euo pipefail - plan="${RUNNER_TEMP}/relay-c4-image-refresh.tfplan" - terraform -chdir=infra/terraform plan \ - -var-file=environments/staging.tfvars \ - '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \ - '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \ - -out="${plan}" - result="$(terraform -chdir=infra/terraform show -json "${plan}" \ - | node dev/scripts/validate-relay-capacity-plan.mjs \ - --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \ - --unobserved-bound 60 --image "${DESIRED_IMAGE}" \ - --rollback-image "${ROLLBACK_IMAGE}")" - case "$(jq -r '[.changes,.changeKind] | join(":")' <<< "${result}")" in - 2:replacement) refresh_phase=predecessor ;; - 0:none|*:obsolete-template-delete) refresh_phase=applied ;; - *:manager-convergence|*:replacement-with-obsolete-template) refresh_phase=converging ;; - *) exit 1 ;; - esac - { - echo "PLAN_CHANGES=$(jq -r '.changes' <<< "${result}")" - echo "REFRESH_PHASE=${refresh_phase}" - } >> "${GITHUB_ENV}" - - - id: state-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Verify the exact selector and current C4 state - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.state-auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \ - --expected-generation '' --expected-membership-sha256 '' --attempt-id '' \ - --image-digest "${TARGET_IMAGE_DIGEST}")" - test "$(jq -r '.generation' <<< "${inspect}")" = "${EXPECTED_SELECTOR_GENERATION}" - test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \ - migration-only - expected_digests="${TARGET_IMAGE_DIGEST}" - if test "${REFRESH_PHASE}" = predecessor; then - expected_digests="${PREDECESSOR_IMAGE_DIGEST}" - elif test "${REFRESH_PHASE}" = converging; then - expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}" - fi - if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \ - "${CELL_ORIGIN}/v1/admin/runtime-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' --data '{"v":1}')"; then - current_digest="$(jq -er '.imageDigest' <<< "${runtime}")" - case ",${expected_digests}," in - *,"${current_digest}",*) ;; - *) exit 1 ;; - esac - source_incarnation='' - draining=forbidden - if test "${REFRESH_PHASE}" != applied; then - status="$(curl --fail-with-body --max-time 30 --request POST \ - "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" - source_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")" - [[ "${source_incarnation}" =~ ^[0-9a-f-]{36}$ ]] - if test "$(jq -r '.draining' <<< "${runtime}")" = true; then - draining=required - fi - fi - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \ - --heartbeat fresh --admission migration-only --draining "${draining}" \ - --activity quiescent --expected-image-digests "${expected_digests}" - runtime_available=true - else - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \ - --admission migration-only --draining either --activity restart-safe \ - --timeout-ms 300000 - source_incarnation='' - runtime_available=false - fi - { - echo "MIG_STABLE_AT_MS=0" - echo "MUTATION_STARTED=false" - echo "REPLACEMENT_STARTED_AT_MS=0" - echo "RUNTIME_AVAILABLE=${runtime_available}" - echo "SOURCE_INCARNATION=${source_incarnation}" - } >> "${GITHUB_ENV}" - - - id: fence-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Fence C4 and prove it stayed empty before replacement - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - if test "${REFRESH_PHASE}" = applied; then exit 0; fi - echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" - if test "${RUNTIME_AVAILABLE}" = false; then exit 0; fi - node dev/scripts/prepare-relay-capacity-canary.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --mode isolate - fence_digests="${PREDECESSOR_IMAGE_DIGEST}" - if test "${REFRESH_PHASE}" = converging; then - fence_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}" - fi - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \ - --heartbeat fresh --admission migration-only --draining required \ - --activity quiescent --expected-image-digests "${fence_digests}" - - - name: Apply the exact saved C4 plan - shell: bash - run: | - set -euo pipefail - if test "${PLAN_CHANGES}" = 0 && test "${RUNTIME_AVAILABLE}" = true; then exit 0; fi - if test "${REFRESH_PHASE}" = applied && test "${RUNTIME_AVAILABLE}" = false; then - echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}" - fi - if test "${REFRESH_PHASE}" != applied; then - replacement_started_at_ms="$(date -u +%s%3N)" - echo "REPLACEMENT_STARTED_AT_MS=${replacement_started_at_ms}" >> "${GITHUB_ENV}" - fi - if test "${PLAN_CHANGES}" != 0; then - terraform -chdir=infra/terraform apply -auto-approve \ - "${RUNNER_TEMP}/relay-c4-image-refresh.tfplan" - fi - gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 - if test "${RUNTIME_AVAILABLE}" = false && test "${REFRESH_PHASE}" = applied; then - instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \ - | jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and - .[0].currentAction == "NONE" then .[0].instance | split("/") | last - else error("C4 is not one stable running instance") end')" - gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \ - --instances "${instance}" --project "${GCP_PROJECT_ID}" \ - --zone "${TARGET_ZONE}" --quiet - gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 - fi - echo "MIG_STABLE_AT_MS=$(date -u +%s%3N)" >> "${GITHUB_ENV}" - - - id: post-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Verify new C4 incarnation, image, and unchanged isolation - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \ - --heartbeat fresh --admission migration-only --draining forbidden \ - --activity quiescent --expected-image-digests "${TARGET_IMAGE_DIGEST}" \ - --timeout-ms 240000 - result="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \ - --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \ - --expected-membership-sha256 '' --attempt-id '' \ - --image-digest "${TARGET_IMAGE_DIGEST}")" - test "$(jq -r '.generation' <<< "${result}")" = "${EXPECTED_SELECTOR_GENERATION}" - test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \ - migration-only - for _ in $(seq 1 36); do - status="$(curl --fail-with-body --max-time 30 --request POST \ - "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" - if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \ - test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \ - -ge "${MIG_STABLE_AT_MS}"; then break; fi - sleep 5 - done - target_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")" - test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true - test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \ - -ge "${MIG_STABLE_AT_MS}" - if test "${REFRESH_PHASE}" != applied; then - if test -n "${SOURCE_INCARNATION}"; then - test "${target_incarnation}" != "${SOURCE_INCARNATION}" - fi - test "$(jq -r '.status.runtime.startedAt' <<< "${status}")" \ - -ge "${REPLACEMENT_STARTED_AT_MS}" - fi - - - name: Require an empty targeted Terraform readback - shell: bash - run: | - set -euo pipefail - plan="${RUNNER_TEMP}/relay-c4-image-readback.tfplan" - terraform -chdir=infra/terraform plan \ - -var-file=environments/staging.tfvars \ - '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \ - '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \ - -out="${plan}" - result="$(terraform -chdir=infra/terraform show -json "${plan}" \ - | node dev/scripts/validate-relay-capacity-plan.mjs \ - --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \ - --unobserved-bound 60 --image "${DESIRED_IMAGE}" \ - --rollback-image "${ROLLBACK_IMAGE}")" - test "$(jq -r '.changes' <<< "${result}")" = 0 diff --git a/.github/workflows/cloud-publish-relay-production.yml b/.github/workflows/cloud-publish-relay-production.yml deleted file mode 100644 index 4a96f6a47c8..00000000000 --- a/.github/workflows/cloud-publish-relay-production.yml +++ /dev/null @@ -1,133 +0,0 @@ -name: Publish Relay Production Image - -on: - workflow_dispatch: - inputs: - mode: - description: Publish a new production image or mirror an existing immutable image to staging - required: true - default: publish - type: choice - options: [publish, mirror-staging] - image-digest: - description: Exact existing production digest for mirror-staging mode - required: false - type: string - confirmation: - description: Enter MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING for mirror-staging mode - required: false - type: string - -permissions: - contents: read - id-token: write - -concurrency: - group: publish-relay-production - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - publish: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - environment: production - env: - GCP_PROJECT_ID: onorca-cloud - GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} - REPOSITORY_ID: orca-cloud - IMAGE_NAME: relay - PUBLISH_MODE: ${{ inputs.mode }} - MIRROR_DIGEST: ${{ inputs.image-digest }} - MIRROR_CONFIRMATION: ${{ inputs.confirmation }} - steps: - - uses: actions/checkout@v4 - - - name: Validate the exact publish request before authentication - shell: bash - run: | - set -euo pipefail - if test "${PUBLISH_MODE}" = mirror-staging; then - [[ "${MIRROR_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] - test "${MIRROR_CONFIRMATION}" = MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING - else - test "${PUBLISH_MODE}" = publish - test -z "${MIRROR_DIGEST}" - test -z "${MIRROR_CONFIRMATION}" - fi - - - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }} - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: docker/setup-buildx-action@v3 - - - name: Configure Docker auth - run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet - - - name: Build and publish immutable image - if: ${{ inputs.mode == 'publish' }} - run: | - IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${GITHUB_SHA}" - docker build -f apps/relay/Dockerfile -t "${IMAGE_TAG}" . - docker push "${IMAGE_TAG}" - DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')" - test -n "${DIGEST}" - IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${DIGEST}" - { - echo '### Terraform candidate image' - echo - echo "\`${IMAGE}\`" - echo - echo 'Declare this digest on a distinct disabled candidate cell in a reviewed Terraform PR.' - } >> "${GITHUB_STEP_SUMMARY}" - - - name: Build and publish immutable fence broker - if: ${{ inputs.mode == 'publish' }} - run: | - IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker:sha-${GITHUB_SHA}" - docker build \ - --build-arg "ORCA_RELAY_FENCE_IMAGE_COMMIT=${GITHUB_SHA}" \ - -f apps/relay-fence-broker/Dockerfile \ - -t "${IMAGE_TAG}" . - docker push "${IMAGE_TAG}" - DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')" - test -n "${DIGEST}" - IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker@${DIGEST}" - { - echo - echo '### Terraform fence broker image' - echo - echo "\`${IMAGE}\`" - } >> "${GITHUB_STEP_SUMMARY}" - - - name: Mirror the exact production manifest to staging - if: ${{ inputs.mode == 'mirror-staging' }} - shell: bash - run: | - set -euo pipefail - source_image="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${MIRROR_DIGEST}" - target_tag="${GCP_REGION}-docker.pkg.dev/onorca-cloud-staging/${REPOSITORY_ID}/${IMAGE_NAME}:production-${MIRROR_DIGEST#sha256:}" - source_digest="$(gcloud artifacts docker images describe "${source_image}" \ - --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" - test "${source_digest}" = "${MIRROR_DIGEST}" - docker pull "${source_image}" - docker tag "${source_image}" "${target_tag}" - docker push "${target_tag}" - target_digest="$(gcloud artifacts docker images describe "${target_tag}" \ - --project onorca-cloud-staging --format='value(image_summary.digest)')" - test "${target_digest}" = "${MIRROR_DIGEST}" - { - echo '### Mirrored Relay image' - echo - printf 'Production and staging now resolve the same immutable digest: %s.\n' \ - "${MIRROR_DIGEST}" - } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/cloud-push-deploy.yml b/.github/workflows/cloud-push-deploy.yml deleted file mode 100644 index b54a48092f8..00000000000 --- a/.github/workflows/cloud-push-deploy.yml +++ /dev/null @@ -1,568 +0,0 @@ -name: Deploy Push Gateway Production - -on: - workflow_dispatch: - inputs: - source_sha: - description: Full reviewed commit SHA to build (feature may remain unmerged) - required: true - type: string - confirmation: - description: Enter DEPLOY_PUSH_GATEWAY to shift production traffic - required: true - type: string - -permissions: - contents: read - id-token: write - -# Serialize push traffic changes independently of Relay and the shared database. -concurrency: - group: production-push-rollout - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - deploy: - if: >- - ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && - github.ref == 'refs/heads/main' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - environment: production - env: - GCP_PROJECT_ID: onorca-cloud - GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }} - SERVICE_NAME: orca-cloud-push - REPOSITORY_ID: orca-cloud - IMAGE_NAME: push - PUSH_ORIGIN: https://push.onorca.dev - PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud.iam.gserviceaccount.com - # Scaling the serving revision must already hold, matching push_min_instances and - # push_max_instances. Terraform owns both, and the candidate inherits them from the - # service, so this deploy never passes a scaling flag: doing so would write a - # Terraform-owned field that `lifecycle.ignore_changes` does not cover, and a later - # `push_max_instances` raise would then be reverted by every deploy. These two values - # are the expected shape, asserted before the candidate is created and again on the - # candidate itself, so a deploy that would change the gateway's Cloud SQL draw fails. - PUSH_MIN_INSTANCES: 1 - PUSH_MAX_INSTANCES: 2 - CONFIRMATION: ${{ inputs.confirmation }} - SOURCE_SHA: ${{ inputs.source_sha }} - steps: - - uses: actions/checkout@v4 - - - name: Require the explicit deploy confirmation - shell: bash - run: | - set -euo pipefail - test "${CONFIRMATION}" = DEPLOY_PUSH_GATEWAY - [[ "${SOURCE_SHA}" =~ ^[a-f0-9]{40}$ ]] - - # Keep the workflow and rollout lease on main; only the Docker build uses candidate code. - - name: Fetch the immutable gateway source - shell: bash - run: | - set -euo pipefail - git fetch --no-tags origin "${SOURCE_SHA}" - test "$(git rev-parse FETCH_HEAD)" = "${SOURCE_SHA}" - mkdir -p "${RUNNER_TEMP}/push-source" - git -C "${GITHUB_WORKSPACE}" archive "${SOURCE_SHA}" cloud \ - | tar -x -C "${RUNNER_TEMP}/push-source" - - - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT }} - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: docker/setup-buildx-action@v3 - - - name: Configure Docker auth - run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet - - # Building an image does not need the deployment lease. - - name: Build and publish the immutable gateway image - shell: bash - run: | - set -euo pipefail - image_tag="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${SOURCE_SHA}" - docker buildx build --push --platform linux/amd64 --provenance=false --metadata-file "${RUNNER_TEMP}/push-image.json" \ - -f "${RUNNER_TEMP}/push-source/cloud/apps/push/Dockerfile" \ - -t "${image_tag}" "${RUNNER_TEMP}/push-source/cloud" - digest="$(jq -er '."containerimage.digest"' "${RUNNER_TEMP}/push-image.json")" - [[ "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]] - echo "IMAGE=${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${digest}" \ - >> "${GITHUB_ENV}" - echo "IMAGE_DIGEST=${digest}" >> "${GITHUB_ENV}" - - # Refuse older images before they ever boot against production. - - name: Require image support for inert validation - shell: bash - run: | - set -euo pipefail - docker run --rm --network none --entrypoint node "${IMAGE}" --input-type=module -e ' - import { loadPushConfig } from "./apps/push/dist/config.js"; - const env = { ORCA_PUSH_PUBLIC_URL: "https://push.onorca.dev", ORCA_PUSH_MODE: "validation", ORCA_PUSH_FCM_PROJECT_ID: "onorca-cloud" }; - if (loadPushConfig(env).mode !== "validation") throw new Error("validation_mode_unsupported"); - let rejected = false; - try { loadPushConfig({ ...env, ORCA_PUSH_MODE: "invalid" }); } catch { rejected = true; } - if (!rejected) throw new Error("validation_mode_not_fail_closed"); - ' - - # Held across the deploy, not just a separate schema step: the gateway opens its pool and - # applies its schema while the new revision starts, so the revision is the schema step. - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-terraform-state - object: terraform/state/push-rollout/production.lock - - # Why: the candidate inherits the serving revision's scaling. A serving revision that has - # drifted below the floor would hand the candidate a cold start on every notification, and - # one that has drifted above the ceiling would hand it a larger Cloud SQL draw than the - # rollout lease was taken for. Refuse to inherit either rather than latch it. - - name: Record the serving revision and require its Terraform-owned scaling - shell: bash - run: | - set -euo pipefail - serving="$(gcloud run services describe "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -r '[.status.traffic[] | select((.percent // 0) > 0)] - | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" - test -n "${serving}" - revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')" - if test "${revisions}" != "${serving}"; then - echo 'Retire leftover revisions under the rollout lease before deploying; three pools are the limit.' >&2 - exit 1 - fi - floor="$(gcloud run revisions describe "${serving}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")" - if [[ "${floor:-0}" -lt "${PUSH_MIN_INSTANCES}" ]]; then - echo "serving revision ${serving} holds ${floor:-0} minimum instances," \ - "below ${PUSH_MIN_INSTANCES}; deploying would inherit and latch it." >&2 - echo "Restore the floor first: gcloud run services update ${SERVICE_NAME}" \ - "--region ${GCP_REGION} --min-instances=${PUSH_MIN_INSTANCES}" >&2 - exit 1 - fi - ceiling="$(gcloud run revisions describe "${serving}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")" - test "${ceiling}" = "${PUSH_MAX_INSTANCES}" - echo "serving revision ${serving} holds ${floor} minimum and ${ceiling} maximum instances" - echo "ROLLBACK_REVISION=${serving}" >> "${GITHUB_ENV}" - gcloud run revisions describe "${serving}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - > "${RUNNER_TEMP}/push-rollback-revision.json" - image="$(jq -er '.status.imageDigest' "${RUNNER_TEMP}/push-rollback-revision.json")" - [[ "${image}" =~ @sha256:[a-f0-9]{64}$ ]] - echo "ROLLBACK_IMAGE=${image}" >> "${GITHUB_ENV}" - jq -e 'all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE" or .value == "active")' \ - "${RUNNER_TEMP}/push-rollback-revision.json" > /dev/null - - # Validation has no schema writes, HTTP mutations, worker, or pruners; tags alone do not - # isolate background consumers from production. - - name: Deploy the candidate revision with no traffic - shell: bash - run: | - set -euo pipefail - tag="c${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - { - echo "VALIDATION_DEPLOY_ATTEMPTED=true" - echo "VALIDATION_REVISION=${SERVICE_NAME}-${tag}" - echo "VALIDATION_TAG=${tag}" - echo "CANDIDATE_TAG=${tag}" - echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}" - } >> "${GITHUB_ENV}" - gcloud run deploy "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --image "${IMAGE}" \ - --tag "${tag}" \ - --revision-suffix "${tag}" \ - --no-traffic \ - --update-env-vars ORCA_PUSH_MODE=validation \ - --quiet - candidate="$(gcloud run services describe "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -er --arg tag "${tag}" \ - '[.status.traffic[] | select(.tag == $tag)] - | if length == 1 then .[0] else error("tagged candidate is not unique") end')" - test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}" - echo "CANDIDATE_URL=$(jq -r '.url' <<< "${candidate}")" >> "${GITHUB_ENV}" - - # A tagged revision is directly addressable and sits outside the service-wide cap, so the - # candidate and the serving revision each draw up to the ceiling during the probe window. - # Successor creation later requires three revision pools; assert the inherited ceiling. - - name: Require the candidate to serve the exact image and inherited scaling - shell: bash - run: | - set -euo pipefail - served="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --format='value(spec.containers[0].image)')" - test "${served}" = "${IMAGE}" - test "${CANDIDATE_REVISION}" != "${ROLLBACK_REVISION}" - candidate_ceiling="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")" - test "${candidate_ceiling}" = "${PUSH_MAX_INSTANCES}" - - - name: Probe the candidate readiness endpoint - shell: bash - run: | - set -euo pipefail - [[ "${CANDIDATE_URL}" =~ ^https://[^/]+$ ]] - for attempt in $(seq 1 30); do - code="$(curl -sS -o "${RUNNER_TEMP}/push-ready.json" -w '%{http_code}' \ - --max-time 10 "${CANDIDATE_URL}/ready" || true)" - if test "${code}" = 200; then - jq -e . < "${RUNNER_TEMP}/push-ready.json" > /dev/null - curl --fail --silent --show-error --max-time 10 "${CANDIDATE_URL}/health" \ - | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "validation"' > /dev/null - echo "candidate ${CANDIDATE_REVISION} is ready after ${attempt} attempt(s)" - exit 0 - fi - echo "attempt ${attempt}: /ready returned ${code}" - sleep 5 - done - echo "candidate ${CANDIDATE_REVISION} never reported ready" >&2 - exit 1 - - # Why: a gateway that boots and answers /ready can still be unable to send. This proves the - # runtime account's FCM grant end to end without delivering anything: validate_only stops - # Google before any push, and the deliberately invalid token means a healthy credential - # answers INVALID_ARGUMENT. PERMISSION_DENIED is the failure this step exists to catch. - # - # Only the four verdicts below are conclusive. A 429, a 5xx, or a transport failure says - # nothing about the credential, so it is retried rather than treated as either answer; a - # denied credential still fails on the first attempt, without burning the retries. - - name: Prove the runtime identity can reach FCM - shell: bash - run: | - set -euo pipefail - token="$(gcloud auth print-access-token \ - --impersonate-service-account "${PUSH_RUNTIME_SERVICE_ACCOUNT}")" - test -n "${token}" - echo "::add-mask::${token}" - body='{"validate_only":true,"message":{"token":"orca-push-deploy-probe-invalid-token","notification":{"title":"Orca","body":"deploy probe"}}}' - for attempt in $(seq 1 5); do - code="$(curl -sS -o "${RUNNER_TEMP}/push-fcm.json" -w '%{http_code}' --max-time 20 \ - -X POST "https://fcm.googleapis.com/v1/projects/${GCP_PROJECT_ID}/messages:send" \ - -H "Authorization: Bearer ${token}" \ - -H 'Content-Type: application/json' \ - --data "${body}" || true)" - status="$(jq -r '.error.status // empty' < "${RUNNER_TEMP}/push-fcm.json" || true)" - echo "attempt ${attempt}: FCM validate-only send returned HTTP ${code} status ${status:-OK}" - if test "${status}" = PERMISSION_DENIED || test "${status}" = INVALID_ARGUMENT || - test "${code}" = 401 || test "${code}" = 403; then - break - fi - sleep 5 - done - if test "${status}" = PERMISSION_DENIED || test "${code}" = 401 || test "${code}" = 403; then - echo "the push runtime identity cannot send through FCM" >&2 - exit 1 - fi - test "${status}" = INVALID_ARGUMENT - - # Cloud Run requires a successor before the latest revision can be deleted. - - name: Retire inert validation and activate the verified image - shell: bash - run: | - set -euo pipefail - tag="a${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - { - echo "CANDIDATE_TAG=${tag}" - echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}" - echo "ACTIVATION_ATTEMPTED=true" - } >> "${GITHUB_ENV}" - # This is the production-effect boundary: schema, pruners and workers start here. - gcloud run deploy "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --image "${IMAGE}" --tag "${tag}" --revision-suffix "${tag}" \ - --remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet - gcloud run services update-traffic "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --remove-tags "${VALIDATION_TAG}" --quiet - gcloud run revisions delete "${VALIDATION_REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet - echo "VALIDATION_RETIRED=true" >> "${GITHUB_ENV}" - revision="$(gcloud run revisions describe "${SERVICE_NAME}-${tag}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)" - jq -e --arg image "${IMAGE}" --arg account "${PUSH_RUNTIME_SERVICE_ACCOUNT}" \ - --arg ceiling "${PUSH_MAX_INSTANCES}" --arg floor "${PUSH_MIN_INSTANCES}" \ - --slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" ' - def shape: del(.containers[0].image) | - .containers[0].env = ((.containers[0].env // []) | - map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name)); - .spec.containers[0].image == $image and .spec.serviceAccountName == $account and - all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and - (.spec | shape) == ($prior[0].spec | shape) and - .metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and - (.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber)' \ - <<< "${revision}" > /dev/null - candidate="$(gcloud run services describe "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -er --arg tag "${tag}" '[.status.traffic[] | select(.tag == $tag)] - | if length == 1 then .[0] else error("active candidate is not unique") end')" - test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}" - url="$(jq -er '.url' <<< "${candidate}")" - [[ "${url}" =~ ^https://[^/]+$ ]] - curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true' - curl --fail --silent --show-error --max-time 10 "${url}/health" \ - | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' - - - name: Shift all traffic to the verified candidate - shell: bash - run: | - set -euo pipefail - echo "TRAFFIC_SHIFT_ATTEMPTED=true" >> "${GITHUB_ENV}" - gcloud run services update-traffic "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --to-revisions "${CANDIDATE_REVISION}=100" \ - --quiet - serving="$(gcloud run services describe "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -r '[.status.traffic[] | select((.percent // 0) > 0)] - | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" - test "${serving}" = "${CANDIDATE_REVISION}" - echo "TRAFFIC_SHIFTED=true" >> "${GITHUB_ENV}" - - # Why: the summary is written before the origin check, not after it. Once traffic has - # moved, the rollback target is the single thing an operator needs, and a summary that only - # appeared on success would be missing in exactly the run that needs it. - - name: Publish the rollout summary - if: ${{ always() && env.CANDIDATE_REVISION != '' && env.ROLLBACK_REVISION != '' }} - shell: bash - run: | - set -euo pipefail - { - echo '### Push gateway rollout' - echo - echo "Source: ${SOURCE_SHA}" - echo - echo "Revision: \`${CANDIDATE_REVISION}\`" - echo - echo "Image: \`${IMAGE_DIGEST}\`" - echo "Known-good image: \`${ROLLBACK_IMAGE}\`" - echo - echo "Recovery: deploy \`${ROLLBACK_IMAGE}\` as a new revision with" \ - "\`--remove-env-vars ORCA_PUSH_MODE --no-traffic --tag --revision-suffix \`." - echo 'Verify its exact digest, configuration, readiness and active mode, then promote and check the public origin.' - echo 'Only then remove obsolete tags and delete rejected/previous revisions; never delete the latest revision.' - echo 'The previous revision is retired after public checks; retain this immutable image for recovery under the rollout lease.' - echo "Activation attempted: ${ACTIVATION_ATTEMPTED:-false}; traffic rollback cannot undo schema or deliveries." - } >> "${GITHUB_STEP_SUMMARY}" - - - name: Verify the public origin after the shift - shell: bash - run: | - set -euo pipefail - for attempt in $(seq 1 30); do - code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \ - "${PUSH_ORIGIN}/ready" || true)" - if test "${code}" = 200; then - curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \ - | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' > /dev/null - echo "ROLLOUT_VERIFIED=true" >> "${GITHUB_ENV}" - echo "${PUSH_ORIGIN} is ready after ${attempt} attempt(s)" - exit 0 - fi - echo "attempt ${attempt}: ${PUSH_ORIGIN}/ready returned ${code}" - sleep 5 - done - echo "${PUSH_ORIGIN} never reported ready after the shift" >&2 - exit 1 - - # Why: everything after the shift runs with production on the candidate. A failure there - # is not a failure to deploy, it is a live gateway that has to go back, so the traffic move - # is undone here rather than left to whoever reads the run. - - name: Roll traffic back to the previous revision - if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }} - shell: bash - run: | - set -euo pipefail - test -n "${ROLLBACK_REVISION:-}" - gcloud run services update-traffic "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --to-revisions "${ROLLBACK_REVISION}=100" \ - --quiet - serving="$(gcloud run services describe "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -r '[.status.traffic[] | select((.percent // 0) > 0)] - | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" - test "${serving}" = "${ROLLBACK_REVISION}" - echo "TRAFFIC_ROLLED_BACK=true" >> "${GITHUB_ENV}" - { - echo - echo '### Push gateway rolled back' - echo - echo "Traffic returned to \`${ROLLBACK_REVISION}\`; the candidate" \ - "\`${CANDIDATE_REVISION}\` no longer serves HTTP; deletion below must stop its workers." - } >> "${GITHUB_STEP_SUMMARY}" - - # Deleting a revision does not restore the service template that Terraform reconciles. - - name: Restore the known-good service template - if: ${{ (failure() || cancelled()) && env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }} - shell: bash - run: | - set -euo pipefail - test "${TRAFFIC_SHIFT_ATTEMPTED:-false}" != true || test "${TRAFFIC_ROLLED_BACK:-false}" = true - test -n "${ROLLBACK_IMAGE}" - # A partial activation may leave validation plus active; free one slot before recovery. - latest="$(gcloud run services describe "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --format='value(status.latestCreatedRevisionName)')" - test -n "${latest}" - if test "${VALIDATION_RETIRED:-false}" != true && test "${latest}" != "${VALIDATION_REVISION}"; then - existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --filter "metadata.name=${VALIDATION_REVISION}" --format='value(metadata.name)')" - if test -n "${existing}"; then - test "${existing}" = "${VALIDATION_REVISION}" - gcloud run services update-traffic "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --remove-tags "${VALIDATION_TAG}" --quiet - gcloud run revisions delete "${VALIDATION_REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet - fi - fi - tag="r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - echo "RECOVERY_TAG=${tag}" >> "${GITHUB_ENV}" - echo "TEMPLATE_RECOVERY_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}" - echo "Template recovery attempted: ${SERVICE_NAME}-${tag}, image ${ROLLBACK_IMAGE}." \ - >> "${GITHUB_STEP_SUMMARY}" - # Known-good schema/workers can run here even though HTTP stays on the old revision. - gcloud run deploy "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --image "${ROLLBACK_IMAGE}" --revision-suffix "${tag}" --tag "${tag}" \ - --remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet - gcloud run services describe "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - > "${RUNNER_TEMP}/push-recovered-service.json" - gcloud run revisions describe "${SERVICE_NAME}-${tag}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - > "${RUNNER_TEMP}/push-recovery-revision.json" - jq -e --arg image "${ROLLBACK_IMAGE}" --arg serving "${ROLLBACK_REVISION}" \ - --arg floor "${PUSH_MIN_INSTANCES}" --arg ceiling "${PUSH_MAX_INSTANCES}" \ - --slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" \ - --slurpfile recovered "${RUNNER_TEMP}/push-recovery-revision.json" ' - def shape: del(.containers[0].image) | - .containers[0].env = ((.containers[0].env // []) | - map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name)); - .spec.template.spec.containers[0].image == $image and - all(.spec.template.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and - $recovered[0].spec.containers[0].image == $image and - all($recovered[0].spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and - ($recovered[0].spec | shape) == ($prior[0].spec | shape) and - .spec.template.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and - (.spec.template.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber) and - ([.status.traffic[] | select((.percent // 0) > 0)] | - length == 1 and .[0].revisionName == $serving and .[0].percent == 100) - ' "${RUNNER_TEMP}/push-recovered-service.json" > /dev/null - echo "TEMPLATE_RESTORED=true" >> "${GITHUB_ENV}" - echo 'Known-good image and normal mode restored; previous revision still serves HTTP.' \ - >> "${GITHUB_STEP_SUMMARY}" - - - name: Promote and verify the known-good recovery revision - if: ${{ always() && env.TEMPLATE_RESTORED == 'true' }} - shell: bash - run: | - set -euo pipefail - url="$(jq -er --arg tag "${RECOVERY_TAG}" \ - '.status.traffic[] | select(.tag == $tag) | .url' "${RUNNER_TEMP}/push-recovered-service.json")" - [[ "${url}" =~ ^https://[^/]+$ ]] - curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true' - curl --fail --silent --show-error --max-time 10 "${url}/health" \ - | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' - gcloud run services update-traffic "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --to-revisions "${TEMPLATE_RECOVERY_REVISION}=100" --quiet - serving="$(gcloud run services describe "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \ - | jq -er '[.status.traffic[] | select((.percent // 0) > 0)] | - if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')" - test "${serving}" = "${TEMPLATE_RECOVERY_REVISION}" - curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/ready" | jq -e '.ok == true' - curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \ - | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' - echo "RECOVERY_VERIFIED=true" >> "${GITHUB_ENV}" - echo "Recovery revision ${TEMPLATE_RECOVERY_REVISION} now serves the known-good image." \ - >> "${GITHUB_STEP_SUMMARY}" - - - name: Delete the rejected candidate revision - if: ${{ always() && env.RECOVERY_VERIFIED == 'true' }} - shell: bash - run: | - set -euo pipefail - if test -z "${CANDIDATE_REVISION:-}"; then - echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}" - exit 0 - fi - if test -n "${CANDIDATE_TAG:-}"; then - gcloud run services update-traffic "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --remove-tags "${CANDIDATE_TAG}" \ - --quiet - echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}" - fi - existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \ - --filter "metadata.name=${CANDIDATE_REVISION}" --format='value(metadata.name)')" - if test -n "${existing}"; then - test "${existing}" = "${CANDIDATE_REVISION}" - gcloud run revisions delete "${CANDIDATE_REVISION}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet - fi - echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}" - echo "candidate revision ${CANDIDATE_REVISION} is absent" - - # Public checks commit the serving revision; cleanup failures must not roll it back. - - name: Retire previous consumers after public checks - if: ${{ always() && (env.ROLLOUT_VERIFIED == 'true' || (env.RECOVERY_VERIFIED == 'true' && env.CANDIDATE_DELETED == 'true')) }} - shell: bash - run: | - set -euo pipefail - serving="${CANDIDATE_REVISION}" - if test "${RECOVERY_VERIFIED:-false}" = true; then - serving="${TEMPLATE_RECOVERY_REVISION}" - fi - gcloud run services update-traffic "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --clear-tags --quiet - revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')" - while IFS= read -r revision; do - test -n "${revision}" || continue - test "${revision}" != "${serving}" || continue - case "${revision}" in - "${ROLLBACK_REVISION}"|"${VALIDATION_REVISION}"|"${CANDIDATE_REVISION}") ;; - *) echo "Unexpected revision ${revision}; manual retirement required." >&2; exit 1 ;; - esac - gcloud run revisions delete "${revision}" \ - --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet - done <<< "${revisions}" - echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}" - echo 'Obsolete revision resources retired; verify actual SQL session drain operationally.' \ - >> "${GITHUB_STEP_SUMMARY}" - - - name: Drop the candidate traffic tag - if: always() - shell: bash - run: | - set -euo pipefail - test -n "${CANDIDATE_TAG:-}" || exit 0 - gcloud run services update-traffic "${SERVICE_NAME}" \ - --project "${GCP_PROJECT_ID}" \ - --region "${GCP_REGION}" \ - --remove-tags "${CANDIDATE_TAG}" \ - --quiet diff --git a/.github/workflows/cloud-recover-relay-staging-c4-image.yml b/.github/workflows/cloud-recover-relay-staging-c4-image.yml deleted file mode 100644 index b0c7ea0aa9d..00000000000 --- a/.github/workflows/cloud-recover-relay-staging-c4-image.yml +++ /dev/null @@ -1,414 +0,0 @@ -name: Recover Relay Staging C4 Image - -on: - workflow_run: - workflows: [Prove Relay Staging Capacity] - types: [completed] - workflow_dispatch: - inputs: - confirmation: - description: Enter RECOVER_STAGING_ASIA_C4_IMAGE - required: true - type: string - -permissions: - actions: read - contents: read - id-token: write - -defaults: - run: - working-directory: cloud - -jobs: - gate: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event_name == 'workflow_dispatch' || (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion != 'success')) }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 5 - outputs: - recover: ${{ steps.trigger.outputs.recover }} - steps: - - name: Bind recovery to the exact failed C4 job - working-directory: . - id: trigger - env: - CONFIRMATION: ${{ inputs.confirmation }} - GH_TOKEN: ${{ github.token }} - SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} - SOURCE_RUN_EVENT: ${{ github.event.workflow_run.event }} - shell: bash - run: | - set -euo pipefail - if test "${GITHUB_EVENT_NAME}" = workflow_dispatch; then - test "${CONFIRMATION}" = RECOVER_STAGING_ASIA_C4_IMAGE - echo "recover=true" >> "${GITHUB_OUTPUT}" - exit 0 - fi - test "${SOURCE_RUN_EVENT}" = workflow_dispatch - [[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]] - jobs="$(gh api --paginate \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")" - count="$(jq -s '[.[].jobs[] | select(.name == "refresh-asia-c4-image" and - (.conclusion == "failure" or .conclusion == "cancelled" or - .conclusion == "timed_out"))] | length' <<< "${jobs}")" - if test "${count}" = 0; then - echo "recover=false" >> "${GITHUB_OUTPUT}" - exit 0 - fi - test "${count}" = 1 - echo "recover=true" >> "${GITHUB_OUTPUT}" - - recover: - needs: gate - if: ${{ needs.gate.outputs.recover == 'true' }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 90 - environment: staging - concurrency: - group: relay-staging-mutation - cancel-in-progress: false - env: - GCP_PROJECT_ID: onorca-cloud-staging - DIRECTOR_ORIGIN: https://relay-staging.onorca.dev - CELL_ORIGIN: https://c4.relay-staging.onorca.dev - TARGET_CELL_ID: staging-gce-c4 - TARGET_ZONE: asia-east2-a - MIG_NAME: orca-cloud-staging-relay-gce-c4 - PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7 - TARGET_IMAGE_DIGEST: sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563 - steps: - - uses: actions/checkout@v4 - - - id: auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - - - uses: google-github-actions/setup-gcloud@v2 - with: - skip_install: true - - - uses: ./.github/actions/cloud-sql-rollout-lease - with: - bucket: onorca-cloud-staging-terraform-state - object: terraform/state/cloud-sql-rollout/staging.lock - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.17.0 - terraform_wrapper: false - - - uses: actions/setup-node@v4 - with: - node-version: 24 - check-latest: true - - - name: Initialize the exact staging backend - run: node dev/scripts/infra.mjs init --env staging - - - id: preflight-auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Inspect the exact C4 recovery state - id: preflight - timeout-minutes: 3 - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.preflight-auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \ - --expected-generation '' --expected-membership-sha256 '' --attempt-id '' \ - --image-digest "${PREDECESSOR_IMAGE_DIGEST}")" - generation="$(jq -er '.generation' <<< "${inspect}")" - [[ "${generation}" =~ ^(0|[1-9][0-9]*)$ ]] - test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \ - migration-only - echo "selector_generation=${generation}" >> "${GITHUB_OUTPUT}" - if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \ - "${CELL_ORIGIN}/v1/admin/runtime-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' --data '{"v":1}')"; then - current_digest="$(jq -er '.imageDigest' <<< "${runtime}")" - [[ "${current_digest}" =~ ^sha256:[a-f0-9]{64}$ ]] - status="$(curl --fail-with-body --max-time 30 --request POST \ - "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" - jq -e '.draining | type == "boolean"' <<< "${runtime}" >/dev/null - { - echo "runtime_available=true" - echo "current_digest=${current_digest}" - echo "draining=$(jq -r '.draining' <<< "${runtime}")" - echo "ready=$(jq -r '.status.runtime.ready == true' <<< "${status}")" - } >> "${GITHUB_OUTPUT}" - else - echo "runtime_available=false" >> "${GITHUB_OUTPUT}" - fi - - - name: Classify both exact recovery end states - id: recovery-plan - timeout-minutes: 10 - shell: bash - run: | - set -euo pipefail - image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay" - predecessor_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}" - target_image="${image_repository}@${TARGET_IMAGE_DIGEST}" - served_digest="$(gcloud artifacts docker images describe "${predecessor_image}" \ - --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')" - test "${served_digest}" = "${PREDECESSOR_IMAGE_DIGEST}" - cells="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars \ - <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')" - test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].image' <<< "${cells}")" = \ - "${target_image}" - target_plan="${RUNNER_TEMP}/relay-c4-image-target.tfplan" - terraform -chdir=infra/terraform plan \ - -var-file=environments/staging.tfvars -lock-timeout=5m \ - '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \ - '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \ - -out="${target_plan}" - target_result="$(terraform -chdir=infra/terraform show -json "${target_plan}" \ - | node dev/scripts/validate-relay-capacity-plan.mjs \ - --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \ - --unobserved-bound 60 --image "${target_image}" \ - --rollback-image "${predecessor_image}")" - target_state="$(jq -r '[.changes,.changeKind] | join(":")' <<< "${target_result}")" - case "${target_state}" in - 0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;; - *) exit 1 ;; - esac - recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" \ - --arg image "${predecessor_image}" '.[$cell].image = $image' <<< "${cells}")" - jq -n --argjson cells "${recovery_cells}" \ - '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-recovery.tfvars.json" - plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan" - terraform -chdir=infra/terraform plan \ - -var-file=environments/staging.tfvars \ - -var-file="${RUNNER_TEMP}/relay-c4-recovery.tfvars.json" \ - -lock-timeout=5m \ - '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \ - '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \ - -out="${plan}" - result="$(terraform -chdir=infra/terraform show -json "${plan}" \ - | node dev/scripts/validate-relay-capacity-plan.mjs \ - --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \ - --unobserved-bound 60 --image "${predecessor_image}" \ - --rollback-image "${target_image}")" - changes="$(jq -r '.changes' <<< "${result}")" - change_kind="$(jq -r '.changeKind' <<< "${result}")" - case "${changes}:${change_kind}" in - 0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;; - *) exit 1 ;; - esac - mig="$(gcloud compute instance-groups managed describe "${MIG_NAME}" \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)" - mig_stable="$(jq -r '.status.isStable == true and .status.versionTarget.isReached == true' \ - <<< "${mig}")" - instances="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)" - instance_stable="$(jq -r 'length == 1 and .[0].instanceStatus == "RUNNING" and - .[0].currentAction == "NONE"' <<< "${instances}")" - action=rollback-predecessor - recovery_digest="${PREDECESSOR_IMAGE_DIGEST}" - if test "${{ steps.preflight.outputs.runtime_available }}" = true && \ - test "${{ steps.preflight.outputs.ready }}" = true && \ - test "${{ steps.preflight.outputs.draining }}" = false && \ - test "${mig_stable}" = true && test "${instance_stable}" = true; then - if test "${{ steps.preflight.outputs.current_digest }}" = "${TARGET_IMAGE_DIGEST}" && \ - test "${target_state}" = 0:none; then - action=verify-target - recovery_digest="${TARGET_IMAGE_DIGEST}" - elif test "${{ steps.preflight.outputs.current_digest }}" = \ - "${PREDECESSOR_IMAGE_DIGEST}" && test "${changes}:${change_kind}" = 0:none; then - action=verify-predecessor - fi - fi - { - echo "changes=${changes}" - echo "change_kind=${change_kind}" - echo "action=${action}" - echo "recovery_digest=${recovery_digest}" - } >> "${GITHUB_OUTPUT}" - - - id: fence-auth - if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Fence C4 before predecessor recovery - if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }} - timeout-minutes: 6 - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }} - shell: bash - run: | - set -euo pipefail - expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}" - current_digest="${{ steps.preflight.outputs.current_digest }}" - if test -n "${current_digest}" && [[ ",${expected_digests}," != *",${current_digest},"* ]]; then - expected_digests="${expected_digests},${current_digest}" - fi - if curl --silent --show-error --fail-with-body --max-time 30 --request POST \ - "${CELL_ORIGIN}/v1/admin/drain" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' --data '{"v":1,"graceMs":0}' \ - >/dev/null; then - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \ - --heartbeat fresh --admission migration-only --draining required \ - --activity quiescent \ - --expected-image-digests "${expected_digests}" \ - --timeout-ms 240000 - else - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \ - --admission migration-only --draining either --activity restart-safe \ - --timeout-ms 240000 - fi - - - name: Apply and stabilize the saved predecessor plan - id: apply - if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }} - timeout-minutes: 20 - env: - CHANGES: ${{ steps.recovery-plan.outputs.changes }} - CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }} - shell: bash - run: | - set -euo pipefail - plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan" - if test "${CHANGES}" != 0; then - terraform -chdir=infra/terraform apply -auto-approve "${plan}" - fi - gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 - echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}" - - - name: Restart only when the plan did not replace C4 - id: restore - if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }} - timeout-minutes: 20 - env: - APPLY_STABLE_AT_MS: ${{ steps.apply.outputs.stable_at_ms }} - CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }} - shell: bash - run: | - set -euo pipefail - if [[ "${CHANGE_KIND}" =~ ^(replacement|replacement-with-obsolete-template|manager-convergence)$ ]]; then - echo "stable_at_ms=${APPLY_STABLE_AT_MS}" >> "${GITHUB_OUTPUT}" - exit 0 - fi - instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \ - | jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and - .[0].currentAction == "NONE" then .[0].instance | split("/") | last - else error("C4 is not one stable running instance") end')" - gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \ - --instances "${instance}" --project "${GCP_PROJECT_ID}" \ - --zone "${TARGET_ZONE}" --quiet - gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 - echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}" - - - name: Require an empty selected-image recovery readback - timeout-minutes: 8 - env: - RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }} - shell: bash - run: | - set -euo pipefail - image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay" - recovery_image="${image_repository}@${RECOVERY_DIGEST}" - other_image="${image_repository}@${TARGET_IMAGE_DIGEST}" - if test "${RECOVERY_DIGEST}" = "${TARGET_IMAGE_DIGEST}"; then - other_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}" - fi - cells="$(terraform -chdir=infra/terraform console \ - -var-file=environments/staging.tfvars \ - <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')" - recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" --arg image "${recovery_image}" \ - '.[$cell].image = $image' <<< "${cells}")" - jq -n --argjson cells "${recovery_cells}" \ - '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-readback.tfvars.json" - readback="${RUNNER_TEMP}/relay-c4-image-recovery-readback.tfplan" - terraform -chdir=infra/terraform plan \ - -var-file=environments/staging.tfvars \ - -var-file="${RUNNER_TEMP}/relay-c4-readback.tfvars.json" \ - -lock-timeout=5m \ - '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \ - '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \ - -out="${readback}" - readback_result="$(terraform -chdir=infra/terraform show -json "${readback}" \ - | node dev/scripts/validate-relay-capacity-plan.mjs \ - --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \ - --unobserved-bound 60 --image "${recovery_image}" \ - --rollback-image "${other_image}")" - test "$(jq -r '.changes' <<< "${readback_result}")" = 0 - - - id: verify-auth - if: ${{ always() }} - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }} - token_format: id_token - id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain - id_token_include_email: true - - - name: Verify the recovered image and unchanged isolation - if: ${{ always() && steps.verify-auth.outcome == 'success' }} - timeout-minutes: 8 - env: - ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.verify-auth.outputs.id_token }} - SELECTOR_GENERATION: ${{ steps.preflight.outputs.selector_generation }} - STABLE_AT_MS: ${{ steps.restore.outputs.stable_at_ms }} - RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }} - shell: bash - run: | - set -euo pipefail - node dev/scripts/verify-relay-capacity-transition.mjs \ - --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ - --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \ - --heartbeat fresh --admission migration-only --draining forbidden \ - --activity quiescent --expected-image-digests "${RECOVERY_DIGEST}" \ - --timeout-ms 240000 - stable_at_ms="${STABLE_AT_MS:-0}" - for _ in $(seq 1 36); do - status="$(curl --fail-with-body --max-time 30 --request POST \ - "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" - if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \ - test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \ - -ge "${stable_at_ms}"; then break; fi - sleep 5 - done - test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true - test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \ - -ge "${stable_at_ms}" - result="$(node dev/scripts/operate-relay-asia-admission.mjs \ - --environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \ - --expected-generation "${SELECTOR_GENERATION}" \ - --expected-membership-sha256 '' --attempt-id '' \ - --image-digest "${RECOVERY_DIGEST}")" - test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \ - migration-only diff --git a/.github/workflows/cloud-requeue-relay-staging-c4-recovery.yml b/.github/workflows/cloud-requeue-relay-staging-c4-recovery.yml deleted file mode 100644 index d92086a620a..00000000000 --- a/.github/workflows/cloud-requeue-relay-staging-c4-recovery.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Requeue Relay Staging C4 Recovery - -on: - workflow_run: - workflows: [Recover Relay Staging C4 Image] - types: [completed] - -permissions: - actions: write - contents: read - -concurrency: - group: relay-staging-c4-recovery-requeue - cancel-in-progress: false - -defaults: - run: - working-directory: cloud - -jobs: - requeue: - if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion == 'cancelled') }} - runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 5 - steps: - - name: Requeue only a cancelled protected recovery job - working-directory: . - env: - GH_TOKEN: ${{ github.token }} - SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} - shell: bash - run: | - set -euo pipefail - [[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]] - jobs="$(gh api --paginate \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")" - count="$(jq -s '[.[].jobs[] | select(.name == "recover" and - .conclusion == "cancelled" and .started_at == null)] | length' <<< "${jobs}")" - if test "${count}" = 0; then exit 0; fi - test "${count}" = 1 - runs="$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/cloud-recover-relay-staging-c4-image.yml/runs?branch=main&per_page=100")" - active=0 - while IFS= read -r run_id; do - active_jobs="$(gh api --paginate \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?filter=latest")" - if jq -se ' - ([.[].jobs[] | select(.name == "gate" and .conclusion == "success")] | length) == 1 and - ([.[].jobs[] | select(.name == "recover" and .status != "completed")] | length) == 1 - ' <<< "${active_jobs}" >/dev/null; then - active=1 - break - fi - done < <(jq -r --arg source "${SOURCE_RUN_ID}" \ - '.workflow_runs[] | select((.id | tostring) != $source and - .status != "completed") | .id' <<< "${runs}") - if test "${active}" != 0; then exit 0; fi - gh workflow run cloud-recover-relay-staging-c4-image.yml \ - --repo "${GITHUB_REPOSITORY}" --ref main \ - -f confirmation=RECOVER_STAGING_ASIA_C4_IMAGE diff --git a/.github/workflows/cloud-verify.yml b/.github/workflows/cloud-verify.yml index 54c8b0397f5..68318fb3269 100644 --- a/.github/workflows/cloud-verify.yml +++ b/.github/workflows/cloud-verify.yml @@ -116,7 +116,7 @@ jobs: strategy: fail-fast: false matrix: - app: [relay, push, relay-fence-broker, cloud-sql-proxy] + app: [relay, push] env: APP: ${{ matrix.app }} TRIVY_IMAGE: aquasec/trivy:0.76.1@sha256:790fd66d1697395d8693d1acaa1ea702406a20f109b6fc7bac8db623eca6c06c @@ -126,20 +126,12 @@ jobs: - name: Build or resolve the runtime image run: | set -euo pipefail - if test "$APP" = cloud-sql-proxy; then - IMAGE="$(sed -n '/variable "relay_gce_cloud_sql_proxy_image"/,/^}/s/.*default *= "\([^"]*\)"/\1/p' infra/terraform/variables.tf)" - [[ "$IMAGE" =~ ^gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:[a-f0-9]{64}$ ]] - docker pull --platform linux/amd64 "$IMAGE" - else - IMAGE="orca-cloud-security:$APP" - docker build --platform linux/amd64 \ - --build-arg "ORCA_RELAY_FENCE_IMAGE_COMMIT=$GITHUB_SHA" \ - --file "apps/$APP/Dockerfile" --tag "$IMAGE" . - fi + IMAGE="orca-cloud-security:$APP" + docker build --platform linux/amd64 \ + --file "apps/$APP/Dockerfile" --tag "$IMAGE" . echo "IMAGE=$IMAGE" >> "$GITHUB_ENV" - name: Smoke test the nonroot Node runtime - if: matrix.app != 'cloud-sql-proxy' run: | docker run --rm --platform linux/amd64 --entrypoint sh "$IMAGE" -ec ' test "$(id -u)" -ne 0 @@ -150,23 +142,6 @@ jobs: node --input-type=module -e "await import(\"./apps/$1/dist/$module.js\")" ' sh "$APP" - - name: Smoke test broker tools and checksum fallback - if: matrix.app == 'relay-fence-broker' - run: | - docker run --rm --platform linux/amd64 --entrypoint sh "$IMAGE" -ec ' - gcloud version - printf relay-security > /tmp/hash-input - test "$(gcloud storage hash --format="value(crc32c_hash)" /tmp/hash-input)" = "6oayYQ==" - terraform version - cd infra/terraform - terraform init -backend=false -input=false -lockfile=readonly - terraform validate - ' - - - name: Smoke test the official SQL proxy binary - if: matrix.app == 'cloud-sql-proxy' - run: docker run --rm --platform linux/amd64 "$IMAGE" --version - - name: Scan every severity and retain the complete inventory run: | mkdir -p security-reports @@ -177,16 +152,12 @@ jobs: docker run --rm --volume "$PWD/security-reports:/reports" "$TRIVY_IMAGE" convert \ --format cyclonedx --output /reports/sbom.cdx.json /reports/vulnerabilities.json - - name: Reject available fixes except dated upstream proxy rebuilds + - name: Reject available fixes run: | jq '(.Results[]?.Vulnerabilities) |= ((. // []) | map(select(.FixedVersion != null and .FixedVersion != "")))' \ security-reports/vulnerabilities.json > security-reports/fixable.json - ignorefile='' - if test "$APP" = cloud-sql-proxy; then - ignorefile=/repo/security/cloud-sql-proxy.trivyignore.yaml - fi docker run --rm --volume "$PWD:/repo:ro" "$TRIVY_IMAGE" convert \ - --ignorefile "$ignorefile" --show-suppressed --exit-code 1 \ + --ignorefile '' --show-suppressed --exit-code 1 \ /repo/security-reports/fixable.json - uses: actions/upload-artifact@v4 @@ -234,38 +205,3 @@ jobs: - run: pnpm install --frozen-lockfile - run: pnpm test - - # Fork pull requests reach this job, so it never configures a backend, never plans, and never - # holds a credential. Only the relay root ships here; foundation and apps stay private. - terraform: - if: github.event_name != 'pull_request' || github.event.pull_request.draft != true - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.17.0 - - - run: terraform -chdir=infra/terraform fmt -check -recursive - - run: terraform -chdir=infra/terraform init -backend=false -input=false - - run: terraform -chdir=infra/terraform validate - - - name: Inventory provider binary vulnerabilities - env: - TRIVY_IMAGE: aquasec/trivy:0.76.1@sha256:790fd66d1697395d8693d1acaa1ea702406a20f109b6fc7bac8db623eca6c06c - run: | - mkdir -p security-reports - docker run --rm --volume "$PWD:/repo" "$TRIVY_IMAGE" rootfs \ - --scanners vuln --ignorefile '' --format json \ - --output /repo/security-reports/providers.json /repo/infra/terraform/.terraform/providers - docker run --rm --volume "$PWD/security-reports:/reports" "$TRIVY_IMAGE" convert \ - --format cyclonedx --output /reports/providers.cdx.json /reports/providers.json - docker run --rm --volume "$PWD/security-reports:/reports" "$TRIVY_IMAGE" convert /reports/providers.json - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: cloud-terraform-provider-security - path: cloud/security-reports/ - retention-days: 30 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 06b8e406710..e057c0712a3 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1386,7 +1386,7 @@ jobs: run: | $files = node config/scripts/pr-package-test-paths.mjs if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - pnpm exec vitest run --config config/vitest.config.ts $files + node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts @files # Why its own step: the CPU burners that make taskkill slow would starve # every other test file sharing the runner. diff --git a/cloud/README.md b/cloud/README.md index 3c7a53d85ff..f1114655842 100644 --- a/cloud/README.md +++ b/cloud/README.md @@ -25,13 +25,9 @@ the validation required before rolling out the updated images and host pins. mutation lease, the Terraform checkout, and the narrow Compute mutation used when a registered target is superseded. The workflow that calls it holds read and invoke rights only, never those mutation permissions. -- `apps/relay-ops`: the relay operations console and the incident monitor - behind `pnpm ops:relay`, `pnpm incident:relay`, and - `pnpm incident:relay-preflight`. - `apps/push` and `packages/push-contract`: the mobile push gateway that holds the APNs key and sends to phones through APNs and FCM, and its wire contract. - It is deployed and operated from here but is not part of the relay data path; - see [docs/push-gateway.md](docs/push-gateway.md). + Deployment and operations are owned by the private `stablyai/orca-cloud` repository. ## Mobile push gateway @@ -58,53 +54,21 @@ and local development. Configure it with `ORCA_PUSH_PUBLIC_URL`, `ORCA_PUSH_FCM_ `ORCA_PUSH_APNS_KEY_ID`, `ORCA_PUSH_APPLE_TEAM_ID`, all three or none), and optionally `ORCA_PUSH_APNS_TOPIC`. The FCM credential comes from the runtime service account, so no key material is configured for Android. See -[push gateway operations](docs/push-gateway.md) for deployment and recovery. +[push gateway operations](https://github.com/stablyai/orca-cloud/blob/main/cloud/docs/push-gateway.md) for deployment and recovery. Logging is aggregate counters only. Tokens, notification titles, notification bodies, and full host fingerprints never reach a log line. -## Infrastructure and operations +## Deployment configuration -- `infra/terraform`: the relay Terraform root. It owns the cells, the director, - the shared Cloud SQL instance, DNS, observability, and every GitHub Workload - Identity provider the relay workflows authenticate through. `backend/` holds - the per-environment backend configuration and `environments/` the tfvars. - Drive it through `pnpm infra:init`, `pnpm infra:plan`, and `pnpm infra:apply`. -- `dev/scripts`: the deploy, capacity, admission, rehome, monitoring, and load - scripts the workflows call, plus the contract tests that pin each workflow - and Terraform surface. Run them with `pnpm test`. -- `dev/contracts` and `dev/fixtures`: the checked-in data those contract tests - read, including the Terraform root partition. -- `docs/`: the relay runbooks, capacity-testing guide, incident-monitor - reference, the workflow variable reference in `docs/relay-workflows.md`, and - the push gateway runbook in `docs/push-gateway.md`. +The relay runtime and wire contracts are public. Production and staging Terraform, +backend configuration, operator tools, runbooks, broker image assembly, and deployment +workflows live in the private `stablyai/orca-cloud` repository. That repository pins +an immutable public source revision and combines it with its deployment configuration. -## Workflows - -The 25 `.github/workflows/cloud-*.yml` workflows are the deploy and operate -surface: publish and deploy the director, roll GCE cell capacity, operate Asia -admission and regional rehoming, prove staging capacity, monitor production, -power staging up and down, and deploy the mobile push gateway. -`.github/actions/cloud-sql-rollout-lease` is the compare-and-swap lease that -serializes rollouts against the shared Cloud SQL instance. Push reuses that -action with its own lease object and deployment concurrency group. - -Every one of them is inert. Each top-level job is gated on -`vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'`, a repository variable that is -unset here, so the two scheduled triggers and every manual dispatch skip -without running a step. Only the repository owner, holding the GCP identities -these workflows authenticate as, can turn them on. - -`Cloud Verify` is not gated. It builds, typechecks, lints, tests, secret-scans, -and validates the relay Terraform on every change under `cloud/`, and it runs -on fork pull requests, so it configures no backend and holds no credential. - -## What is not here - -The `terraform-foundation` and `terraform-apps` roots and the API and auth -services live in the private `stablyai/orca-cloud` repository. Scripts and -tests that spanned both trees were narrowed to the relay side rather than -carrying a dangling reference. +`Cloud Verify` here builds and tests public runtime code, audits dependencies, scans +relay and push images, and checks for secrets without cloud credentials. Private CI +also validates Terraform, scans the SQL proxy and broker, and runs deployment contracts. ## Local development diff --git a/cloud/apps/push/src/push-deploy-preflight.test.ts b/cloud/apps/push/src/push-deploy-preflight.test.ts deleted file mode 100644 index 4652ef8edaa..00000000000 --- a/cloud/apps/push/src/push-deploy-preflight.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { readFileSync } from 'node:fs' -import { expect, it } from 'vitest' -import { loadPushConfig } from './config.js' - -it('runs the deployment image preflight against the actual config loader', () => { - const workflow = readFileSync( - new URL('../../../../.github/workflows/cloud-push-deploy.yml', import.meta.url), - 'utf8' - ) - const step = workflow.split('- name: Require image support for inert validation')[1]! - const script = step.match(/--input-type=module -e '([\s\S]*?)'/)?.[1] - expect(script).toBeDefined() - const run = new Function('loadPushConfig', script!.replace(/import .*?;/, '')) - expect(() => run(loadPushConfig)).not.toThrow() - expect(() => run(() => ({ mode: 'active' }))).toThrow('validation_mode_unsupported') - expect(() => run(() => ({ mode: 'validation' }))).toThrow('validation_mode_not_fail_closed') -}) diff --git a/cloud/apps/relay-fence-broker/Dockerfile b/cloud/apps/relay-fence-broker/Dockerfile deleted file mode 100644 index 892803dacd3..00000000000 --- a/cloud/apps/relay-fence-broker/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -FROM node:24.22.1-alpine@sha256:feb76650b7195319268c59cb2d419c732d6c499b13bf5296f16ab59eb1f0a4f3 AS build -WORKDIR /workspace -ENV PATH="/opt/pnpm/node_modules/.bin:$PATH" -COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./ -RUN npm install --prefix /opt/pnpm --ignore-scripts --no-audit --no-fund "$(node -p 'require("./package.json").packageManager.split("+")[0]')" \ - && npm audit signatures --prefix /opt/pnpm -COPY apps/relay-fence-broker/package.json apps/relay-fence-broker/package.json -RUN pnpm install --frozen-lockfile -COPY apps/relay-fence-broker apps/relay-fence-broker -RUN pnpm --filter @orca-cloud/relay-fence-broker build - -FROM hashicorp/terraform:1.17.0@sha256:4d1b496a256f2989bccdb04b9104a4ede97234b10fba90259e173e905ec4fb58 AS terraform - -FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:590.0.0-alpine@sha256:2d6788b06641d02cff063b337dd01a66b71b813bd22fb1fa1b9ee2e7c0ef0743 -RUN apk upgrade --no-cache && apk add --no-cache libstdc++ -ARG ORCA_RELAY_FENCE_IMAGE_COMMIT -RUN test "$(printf '%s' "${ORCA_RELAY_FENCE_IMAGE_COMMIT}" | grep -E '^[a-f0-9]{40}$')" -ENV NODE_ENV=production -ENV PORT=8080 -ENV ORCA_RELAY_FENCE_IMAGE_COMMIT=${ORCA_RELAY_FENCE_IMAGE_COMMIT} -ENV IAC_TOOL=terraform -ENV CLOUDSDK_STORAGE_USE_GCLOUD_CRC32C=false -WORKDIR /workspace -COPY --from=build /usr/local /usr/local -COPY --from=build /opt/pnpm /opt/pnpm -ENV PATH="/opt/pnpm/node_modules/.bin:$PATH" -COPY --from=terraform /bin/terraform /usr/local/bin/terraform -COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ -COPY apps/relay-fence-broker/package.json apps/relay-fence-broker/package.json -COPY --from=build /workspace/apps/relay-fence-broker/dist apps/relay-fence-broker/dist -COPY dev/scripts dev/scripts -COPY infra/terraform infra/terraform -COPY apps/relay-fence-broker/requirements.txt /tmp/broker-requirements.txt -RUN /google-cloud-sdk/platform/bundledpythonunixmusl/bin/python3 -m pip install \ - --no-cache-dir --no-deps --require-hashes --upgrade \ - --target /google-cloud-sdk/lib/third_party -r /tmp/broker-requirements.txt \ - && rm -rf /google-cloud-sdk/platform/bq /google-cloud-sdk/platform/gsutil \ - /google-cloud-sdk/platform/bundledpythonunixmusl/lib/python3.14/site-packages/pip* \ - /google-cloud-sdk/platform/bundledpythonunixmusl/lib/python3.14/site-packages/wheel* \ - && rm -f /google-cloud-sdk/bin/gcloud-crc32c /google-cloud-sdk/bin/bq \ - /google-cloud-sdk/bin/gsutil /google-cloud-sdk/platform/bundledpythonunixmusl/bin/pip* \ - /google-cloud-sdk/platform/bundledpythonunixmusl/bin/wheel /tmp/broker-requirements.txt -RUN pnpm install --prod --frozen-lockfile --filter @orca-cloud/relay-fence-broker... \ - && rm -rf /opt/pnpm /usr/local/lib/node_modules /opt/yarn-* /root/.cache /root/.local \ - && rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack \ - /usr/local/bin/pnpm /usr/local/bin/pnpx /usr/local/bin/yarn /usr/local/bin/yarnpkg \ - && adduser -D -u 10001 broker \ - && chown -R broker:broker /workspace -USER broker -EXPOSE 8080 -CMD ["node", "apps/relay-fence-broker/dist/index.js"] diff --git a/cloud/apps/relay-ops/README.md b/cloud/apps/relay-ops/README.md deleted file mode 100644 index 7dd16307dc0..00000000000 --- a/cloud/apps/relay-ops/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Orca Relay Operations - -A private, aggregate dashboard for the Orca Relay control and data planes. It reads local `gcloud` and `gh` credentials on the server; credentials and per-user Relay state never enter the browser. One cached `gcloud auth print-access-token` refresh feeds concurrent read-only Google APIs so the collector does not stampede the local credential store. - -## Run locally - -Prerequisites: - -- Node 24 and pnpm 10 -- `gcloud` authenticated for `onorca-cloud` and `onorca-cloud-staging` -- `gh` authenticated with read access to `stablyai/orca-cloud` - -From the repository root: - -```sh -pnpm install -pnpm ops:relay -``` - -Open . The server binds only to loopback and refreshes aggregate data every minute. Production and staging are read-only by default. - -The cost panel is a labeled planning estimate. There is currently no Cloud Billing export in either project, so the dashboard cannot claim exact billed spend. GCP Billing remains authoritative. - -## Share through Tailscale - -Keep the dashboard bound to loopback and let Tailscale provide identity, TLS, and tailnet ACL enforcement: - -```sh -tailscale serve --bg http://127.0.0.1:2455 -tailscale serve status -``` - -Share the HTTPS URL printed by `tailscale serve status` with the team. Limit access to the intended operator group in the tailnet ACL. Do not use a public funnel. Stop sharing with: - -```sh -tailscale serve reset -``` - -For a persistent host, run `pnpm --filter @orca-cloud/relay-ops build` and supervise `pnpm --filter @orca-cloud/relay-ops start` with the host's normal process manager. The process needs the same non-interactive `gcloud` and `gh` identities. - -## Optional staging controls - -Controls are intentionally local-only and disabled unless explicitly enabled: - -```sh -RELAY_OPS_ENABLE_STAGING_CONTROLS=1 pnpm ops:relay -``` - -Even in this mode the service never changes GCP directly. It dispatches `.github/workflows/power-relay-staging.yml`, preserves the workflow's typed `WAKE_STAGING` / `SLEEP_STAGING` confirmation, and always wakes only configured-admission cells. Requests require the loopback origin and a per-process CSRF token, so controls stay unavailable through the Tailscale view. - -## Data and security boundaries - -- Browser payloads contain aggregate Monitoring points, resource health, immutable image digests, alert-policy metadata, and workflow metadata. -- Account IDs, host IDs, device IDs, pairing state, assignment rows, bearer tokens, service-account tokens, startup scripts, secret values, and individual Relay-admin state are excluded. -- Sleeping staging is inventory-only. Viewing it does not probe or cold-start Cloud Run services and cannot resize empty MIGs. -- Partial GCP or GitHub failures degrade the affected panel and produce a sanitized warning. -- Missing cell inventory renders as `Unknown`, never `Sleeping`. After one successful read, transient credential or collector failures retain the last good snapshot and mark it stale. -- Responses use `no-store`, a restrictive CSP, frame denial, and no-referrer headers. - -## Verification - -```sh -pnpm --filter @orca-cloud/relay-ops test -pnpm --filter @orca-cloud/relay-ops typecheck -pnpm --filter @orca-cloud/relay-ops build -``` diff --git a/cloud/apps/relay-ops/package.json b/cloud/apps/relay-ops/package.json deleted file mode 100644 index d580eecea80..00000000000 --- a/cloud/apps/relay-ops/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@orca-cloud/relay-ops", - "private": true, - "version": "0.0.0", - "type": "module", - "main": "dist/index.js", - "scripts": { - "build": "pnpm clean && tsc -p tsconfig.build.json && node -e \"require('fs').cpSync('public','dist/public',{recursive:true})\"", - "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"", - "dev": "tsx watch src/index.ts", - "incident:monitor": "tsx src/incident-monitor-cli.ts", - "incident:preflight": "tsx src/incident-live-preflight-cli.ts", - "incident:pre-drain-sample": "tsx src/pre-drain-sample-cli.ts", - "lint": "tsc -p tsconfig.json --noEmit", - "start": "node dist/index.js", - "test": "vitest run", - "typecheck": "tsc -p tsconfig.json --noEmit" - }, - "dependencies": { - "@hono/node-server": "^2.1.5", - "hono": "^4.13.13", - "zod": "^4.6.7" - }, - "devDependencies": { - "@types/node": "^24.19.2", - "tsx": "^4.23.15", - "typescript": "^7.0.2", - "vitest": "^5.0.5" - } -} diff --git a/cloud/apps/relay-ops/public/app.js b/cloud/apps/relay-ops/public/app.js deleted file mode 100644 index 83c072662fb..00000000000 --- a/cloud/apps/relay-ops/public/app.js +++ /dev/null @@ -1,273 +0,0 @@ -const state = { environment: 'production', window: 360, snapshot: null, config: null, loading: false } - -const $ = (selector) => document.querySelector(selector) -const all = (selector) => [...document.querySelectorAll(selector)] -const escapeHtml = (value) => String(value).replace(/[&<>'"]/g, (character) => ({ - '&': '&', '<': '<', '>': '>', "'": ''', '"': '"' -})[character]) - -function formatNumber(value, maximumFractionDigits = 0) { - if (value === null || value === undefined) return '—' - return new Intl.NumberFormat('en-US', { notation: value >= 10_000 ? 'compact' : 'standard', maximumFractionDigits }).format(value) -} - -function formatBytes(value) { - if (value === null || value === undefined) return '—' - const units = ['B', 'KB', 'MB', 'GB', 'TB'] - let amount = value - let unit = 0 - while (Math.abs(amount) >= 1000 && unit < units.length - 1) { amount /= 1000; unit += 1 } - return `${formatNumber(amount, amount < 10 ? 1 : 0)} ${units[unit]}` -} - -function formatMetric(metric, value) { - if (value === null || value === undefined) return '—' - if (metric.unit === 'bytes') return formatBytes(value) - if (metric.unit === 'milliseconds') return `${formatNumber(value, 1)} ms` - return formatNumber(value, value < 10 ? 1 : 0) -} - -function timeAgo(value) { - const seconds = Math.max(0, Math.round((Date.now() - Date.parse(value)) / 1000)) - if (seconds < 60) return `${seconds}s ago` - if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago` - if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago` - return `${Math.floor(seconds / 86400)}d ago` -} - -function shortImage(value) { - const digest = value?.match(/sha256:([a-f0-9]{64})/)?.[1] - if (digest) return digest.slice(0, 10) - const tag = value?.split(':').at(-1) - return tag?.slice(0, 16) ?? '—' -} - -function badge(label, healthy) { - return `${escapeHtml(label)}` -} - -function renderSummary(snapshot) { - const { summary } = snapshot - const utilization = summary.poweredCapacity === null - ? null - : summary.poweredCapacity > 0 - ? summary.observedConnections / summary.poweredCapacity * 100 - : 0 - const items = [ - ['Observed connections', formatNumber(summary.observedConnections, 1), 'Latest 1-minute aggregate mean'], - ['Active relay sessions', formatNumber(summary.observedSplices, 1), `${formatNumber(summary.observedControls, 1)} desktop-control mean`], - ['Healthy cells', `${summary.activeCells ?? '—'} / ${summary.totalCells}`, summary.poweredCapacity === null ? 'Cell inventory unavailable' : `${formatNumber(summary.poweredCapacity)} powered request units`], - ['Capacity signal', utilization === null ? '—' : `${formatNumber(utilization, 2)}%`, `${formatNumber(summary.configuredCapacity)} configured admission units`] - ] - $('#summary').innerHTML = items.map(([label, value, caption]) => ` -
-

${escapeHtml(label)}

-
${escapeHtml(value)}
-
${escapeHtml(caption)}
-
`).join('') -} - -function cellState(cell) { - if (cell.targetSize === null) return ['Unknown', null] - if (cell.targetSize === 0) return ['Sleeping', null] - if (cell.backendHealth === 'healthy' && cell.endpoint.ready) return ['Healthy', true] - if (cell.stable && cell.backendHealth === 'empty') return ['Starting', null] - return ['Attention', false] -} - -function renderTopology(snapshot) { - const director = snapshot.resources.director - const sleeping = snapshot.resources.cells.every((cell) => cell.targetSize === 0) - && snapshot.resources.sql?.activationPolicy === 'NEVER' - const directorHealthy = director?.ready && snapshot.resources.directorEndpoint.health - $('#topology-meta').textContent = `${snapshot.environment.project} · ${snapshot.environment.region}` - const cellNodes = snapshot.resources.cells.map((cell) => { - const [label, healthy] = cellState(cell) - return `
- ${escapeHtml(cell.hostname.toUpperCase())} ${badge(label, healthy)} - ${escapeHtml(cell.region)} · ${escapeHtml(cell.zone)} · ${formatNumber(cell.capacityRequests)} units -
` - }).join('') - $('#topology').innerHTML = ` -
Desktop + phoneEncrypted Relay traffic
- -
Director ${badge(sleeping ? 'Sleeping' : directorHealthy ? 'Ready' : 'Attention', sleeping ? null : Boolean(directorHealthy))}${escapeHtml(snapshot.environment.directorOrigin)}
- -
${cellNodes}
` -} - -function chartPaths(points, width = 400, height = 112) { - if (points.length === 0) return null - const values = points.map((point) => point.value) - const minimum = Math.min(0, ...values) - const maximum = Math.max(...values) - const spread = maximum - minimum || 1 - const coordinates = points.map((point, index) => { - const x = points.length === 1 ? width : index / (points.length - 1) * width - const y = height - ((point.value - minimum) / spread * (height - 10) + 5) - return [x, y] - }) - const line = coordinates.map(([x, y], index) => `${index === 0 ? 'M' : 'L'}${x.toFixed(1)},${y.toFixed(1)}`).join(' ') - const area = `${line} L${width},${height} L0,${height} Z` - return { line, area } -} - -function renderCharts(snapshot) { - const names = ['controls', 'splices', 'assignment_5xx', 'postgres_retries', 'auth_failures', 'event_loop_ms_p99'] - $('#charts').innerHTML = names.map((name) => { - const metric = snapshot.monitoring.metrics[name] - const paths = chartPaths(metric.points) - const graph = paths ? ` - - - ` : '
No samples in this window
' - return `

${escapeHtml(metric.label)}

${escapeHtml(formatMetric(metric, metric.latest))}
${escapeHtml(metric.unit)}
${graph}
` - }).join('') -} - -function renderCells(snapshot) { - const controlsByCell = snapshot.monitoring.metrics.controls.latestByCell - const splicesByCell = snapshot.monitoring.metrics.splices.latestByCell - $('#cells').innerHTML = snapshot.resources.cells.map((cell) => { - const [label, healthy] = cellState(cell) - const observed = (controlsByCell[cell.cellId] ?? 0) + (splicesByCell[cell.cellId] ?? 0) - return ` - ${escapeHtml(cell.hostname.toUpperCase())}
${escapeHtml(cell.region)} · ${escapeHtml(cell.zone)}
- ${badge(label, healthy)}
MIG ${cell.runningInstances ?? '—'}/${cell.targetSize ?? '—'}
- ${formatNumber(observed, 1)}
1-minute mean
- ${formatNumber(cell.capacityRequests)}
DB pool ${formatNumber(cell.databasePoolMax)} · ${cell.configuredAdmission ? 'configured' : 'candidate only'}
- ${escapeHtml(shortImage(cell.imageDigest))} - ` - }).join('') -} - -function serviceRow(name, healthy, detail, suffix = '') { - return `
${escapeHtml(name)}
${escapeHtml(detail)}
${badge(suffix || (healthy ? 'Ready' : 'Attention'), healthy)}
` -} - -function renderServices(snapshot) { - const { resources } = snapshot - const sleeping = resources.cells.every((cell) => cell.targetSize === 0) - && resources.sql?.activationPolicy === 'NEVER' - const certDays = resources.certificate?.expireTime - ? Math.floor((Date.parse(resources.certificate.expireTime) - Date.now()) / 86400000) - : null - $('#services').innerHTML = [ - serviceRow('Director', sleeping ? null : Boolean(resources.director?.ready && resources.directorEndpoint.health), `${resources.director?.revision ?? 'Revision unavailable'} · ${shortImage(resources.director?.image)}`, sleeping ? 'Sleeping' : ''), - serviceRow('Authentication', sleeping ? null : Boolean(resources.auth?.ready && resources.authEndpoint.health), `${resources.auth?.revision ?? 'Revision unavailable'} · ${shortImage(resources.auth?.image)}`, sleeping ? 'Sleeping' : ''), - serviceRow('Cloud SQL', resources.sql?.state === 'RUNNABLE' || resources.sql?.activationPolicy === 'NEVER', `${resources.sql?.tier ?? 'unknown'} · ${resources.sql?.activationPolicy ?? 'unknown'}`, resources.sql?.state ?? 'Unknown'), - serviceRow('Wildcard TLS', resources.certificate?.state === 'ACTIVE', resources.certificate?.domains.join(', ') ?? 'Certificate unavailable', certDays === null ? 'Unknown' : `${certDays}d`) - ].join('') -} - -function renderAlerts(snapshot) { - const policies = snapshot.monitoring.alertPolicies - $('#alerts').innerHTML = policies.length ? policies.map((policy) => `
${escapeHtml(policy.displayName.replace('Orca Relay: ', ''))}
Cloud Monitoring policy
${badge(policy.enabled ? 'Enabled' : 'Disabled', policy.enabled)}
`).join('') : '

No Relay alert policies returned.

' -} - -function renderWorkflows(snapshot) { - $('#workflows').innerHTML = snapshot.workflows.length ? snapshot.workflows.slice(0, 6).map((run) => { - const healthy = run.conclusion === 'success' - const stateLabel = run.status === 'completed' ? (run.conclusion ?? 'completed') : run.status - return `
${escapeHtml(run.name)}
${escapeHtml(run.headSha)} · ${timeAgo(run.updatedAt)}
${badge(stateLabel, healthy)}` - }).join('') : '

Workflow history unavailable.

' -} - -function renderCost(snapshot) { - const cost = snapshot.cost - $('#cost').innerHTML = `
$${formatNumber(cost.monthlyUsd)}/ month
-

Modeled range $${formatNumber(cost.rangeUsd[0])}–$${formatNumber(cost.rangeUsd[1])}. Not billed cost.

-
${cost.lines.map((line) => `
${escapeHtml(line.label)}$${formatNumber(line.monthlyUsd, 2)}
`).join('')}
-

Exact billing: ${escapeHtml(cost.actualBilling.reason)}
${escapeHtml(cost.caveats[1])}

` -} - -function renderWarnings(snapshot) { - const warning = $('#warnings') - warning.classList.toggle('hidden', snapshot.warnings.length === 0) - warning.textContent = snapshot.warnings.length ? `Partial data: ${snapshot.warnings.join(' ')}` : '' -} - -function render(snapshot) { - state.snapshot = snapshot - $('#environment-label').textContent = `${snapshot.environment.label} · ${snapshot.environment.region}` - $('#freshness').textContent = snapshot.stale - ? `Last good update ${timeAgo(snapshot.generatedAt)} · refresh degraded` - : `Updated ${timeAgo(snapshot.generatedAt)}` - $('#freshness-dot').className = snapshot.stale ? 'status-dot neutral' : 'status-dot healthy' - renderWarnings(snapshot) - renderSummary(snapshot) - renderTopology(snapshot) - renderCharts(snapshot) - renderCells(snapshot) - renderServices(snapshot) - renderAlerts(snapshot) - renderWorkflows(snapshot) - renderCost(snapshot) - $('#power-panel').classList.toggle('hidden', !(state.environment === 'staging' && state.config?.stagingControlsEnabled)) - $('#compute-link').href = snapshot.environment.consoleLinks.compute - $('#alert-link').href = snapshot.environment.consoleLinks.alerts -} - -async function loadSnapshot() { - if (state.loading) return - state.loading = true - $('#refresh').disabled = true - $('#error').classList.add('hidden') - $('#freshness-dot').className = 'status-dot neutral' - $('#freshness').textContent = 'Refreshing current state…' - try { - const response = await fetch(`/api/snapshot?environment=${state.environment}&window=${state.window}`) - const body = await response.json() - if (!response.ok) throw new Error(body.error ?? 'Snapshot failed') - render(body) - } catch (error) { - $('#freshness-dot').className = 'status-dot unhealthy' - $('#freshness').textContent = 'Refresh failed' - $('#error').textContent = error instanceof Error ? error.message : 'Relay operations data is unavailable.' - $('#error').classList.remove('hidden') - } finally { - state.loading = false - $('#refresh').disabled = false - } -} - -async function dispatchPower(mode) { - const confirmation = mode === 'wake' ? 'WAKE_STAGING' : mode === 'sleep' ? 'SLEEP_STAGING' : '' - if (confirmation) { - const entered = window.prompt(`Type ${confirmation} to dispatch the guarded workflow.`) ?? '' - if (entered !== confirmation) return - } - all('[data-power]').forEach((button) => { button.disabled = true }) - $('#power-result').textContent = 'Dispatching…' - try { - const response = await fetch('/api/staging/power', { - method: 'POST', - headers: { 'content-type': 'application/json', 'x-csrf-token': state.config.csrfToken }, - body: JSON.stringify({ mode, confirmation }) - }) - const body = await response.json() - if (!response.ok) throw new Error(body.error) - $('#power-result').textContent = 'Workflow accepted. Refresh after it completes.' - } catch (error) { - $('#power-result').textContent = error instanceof Error ? error.message : 'Dispatch failed.' - } finally { - all('[data-power]').forEach((button) => { button.disabled = false }) - } -} - -all('[data-environment]').forEach((button) => button.addEventListener('click', () => { - state.environment = button.dataset.environment - all('[data-environment]').forEach((candidate) => candidate.setAttribute('aria-pressed', String(candidate === button))) - loadSnapshot() -})) -$('#window').addEventListener('change', (event) => { state.window = Number(event.target.value); loadSnapshot() }) -$('#refresh').addEventListener('click', loadSnapshot) -all('[data-power]').forEach((button) => button.addEventListener('click', () => dispatchPower(button.dataset.power))) - -async function start() { - try { state.config = await fetch('/api/config').then((response) => response.json()) } catch { state.config = {} } - await loadSnapshot() - window.setInterval(loadSnapshot, 60_000) -} - -start() diff --git a/cloud/apps/relay-ops/public/index.html b/cloud/apps/relay-ops/public/index.html deleted file mode 100644 index c49c3bee579..00000000000 --- a/cloud/apps/relay-ops/public/index.html +++ /dev/null @@ -1,115 +0,0 @@ - - - - - - - Orca Relay Operations - - - - - -
-
- -
- Orca Relay - Operations -
-
-
-
- - -
- - -
-
- -
-
-
-

Production · us-central1

-

Relay data plane

-

Private, aggregate operations view. No account, host, device, or pairing identifiers.

-
-
- - Loading current state… -
-
- - - - -
-
-
-
-
-
- -
-
-

Topology

Request path

- -
-
-
- -

Traffic

Signals in selected window

-
- -
-
-

Compute

Relay cells

Open in GCP ↗
-

Configured admission is shown below. Live director admission and heartbeat state stays unavailable because this dashboard has no Relay-admin identity.

-
- - -
CellStateObservedCapacityImage
-
-
-

Control plane

Services

-
-
-
- -
- -
-

Delivery

Recent workflows

-
-
-
-

Planning

Monthly run-rate

-
-
-
- - -
-
Orca Relay Operations · Server-side credentials · Read-only by default
- - diff --git a/cloud/apps/relay-ops/public/styles.css b/cloud/apps/relay-ops/public/styles.css deleted file mode 100644 index 17a974c3d9a..00000000000 --- a/cloud/apps/relay-ops/public/styles.css +++ /dev/null @@ -1,165 +0,0 @@ -:root { - --radius: 10px; - --background: #fff; - --foreground: #0a0a0a; - --card: #fff; - --primary: #171717; - --primary-foreground: #fafafa; - --secondary: #f5f5f5; - --muted: #f5f5f5; - --muted-foreground: #737373; - --accent: #f5f5f5; - --border: #e5e5e5; - --ring: #a1a1a1; - --destructive: #e40014; - --success: #15803d; - --warning: #895503; - --font-mono: 'SF Mono', SFMono-Regular, ui-monospace, Menlo, Consolas, monospace; -} -@media (prefers-color-scheme: dark) { - :root { - --background: #0a0a0a; - --foreground: #fafafa; - --card: #171717; - --primary: #e5e5e5; - --primary-foreground: #171717; - --secondary: #262626; - --muted: #262626; - --muted-foreground: #a1a1a1; - --accent: #262626; - --border: rgb(255 255 255 / 0.07); - --ring: #737373; - --destructive: #ff6568; - --success: #4ade80; - --warning: #fbbf24; - } -} -* { box-sizing: border-box; } -body { - margin: 0; - background: var(--background); - color: var(--foreground); - font-family: 'Geist', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; - font-size: 14px; - letter-spacing: .01em; -} -button, select { font: inherit; } -button:focus-visible, select:focus-visible { outline: 2px solid var(--ring); outline-offset: 2px; } -.topbar { - height: 64px; - border-bottom: 1px solid var(--border); - padding: 0 max(24px, calc((100vw - 1440px) / 2)); - display: flex; - align-items: center; - justify-content: space-between; - position: sticky; - top: 0; - background: color-mix(in srgb, var(--background) 92%, transparent); - backdrop-filter: blur(16px); - z-index: 10; -} -.brand { display: flex; align-items: center; gap: 10px; } -.brand-mark { - width: 30px; height: 30px; border-radius: 9px; background: var(--primary); - color: var(--primary-foreground); display: grid; place-items: center; font-weight: 700; -} -.brand div { display: flex; align-items: baseline; gap: 7px; } -.brand span:last-child { color: var(--muted-foreground); font-size: 12px; } -.toolbar { display: flex; align-items: center; gap: 8px; } -.segmented { display: flex; padding: 3px; gap: 2px; border-radius: 8px; background: var(--secondary); } -.segmented button { border: 0; background: transparent; color: var(--muted-foreground); padding: 5px 10px; border-radius: 6px; cursor: pointer; } -.segmented button[aria-pressed="true"] { background: var(--card); color: var(--foreground); box-shadow: 0 1px 2px rgb(0 0 0 / .08); } -select, .button { height: 32px; border: 1px solid var(--border); border-radius: 8px; background: var(--card); color: var(--foreground); padding: 0 10px; } -.button { cursor: pointer; font-weight: 500; } -.button:hover { background: var(--accent); } -.button:disabled { opacity: .5; cursor: wait; } -main { max-width: 1440px; margin: 0 auto; padding: 42px 24px 72px; } -.page-heading { display: flex; justify-content: space-between; align-items: end; gap: 24px; margin-bottom: 28px; } -h1, h2, p { margin: 0; } -h1 { font-size: 28px; line-height: 1.2; letter-spacing: -.025em; margin-top: 5px; } -h2 { font-size: 16px; line-height: 1.25; letter-spacing: -.01em; margin-top: 3px; } -.lede { color: var(--muted-foreground); margin-top: 8px; max-width: 680px; } -.eyebrow { color: var(--muted-foreground); font-size: 11px; line-height: 1; font-weight: 600; text-transform: uppercase; letter-spacing: .05em; } -.freshness { color: var(--muted-foreground); display: flex; align-items: center; gap: 8px; font-size: 12px; white-space: nowrap; } -.status-dot { width: 7px; height: 7px; border-radius: 999px; background: var(--ring); display: inline-block; } -.status-dot.healthy { background: var(--success); } -.status-dot.unhealthy { background: var(--destructive); } -.summary-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; margin-bottom: 12px; } -.metric-card, .panel { border: 1px solid var(--border); border-radius: 14px; background: var(--card); } -.metric-card { padding: 18px; min-height: 116px; } -.metric-card .value { font-size: 29px; font-weight: 600; letter-spacing: -.035em; margin-top: 16px; } -.metric-card .caption { color: var(--muted-foreground); font-size: 12px; margin-top: 3px; } -.skeleton { background: linear-gradient(90deg, var(--card), var(--muted), var(--card)); background-size: 200% 100%; animation: shimmer 1.6s infinite; } -@keyframes shimmer { to { background-position: -200% 0; } } -.panel { padding: 18px; } -.topology-panel { margin-bottom: 36px; } -.panel-heading, .section-heading { display: flex; align-items: center; justify-content: space-between; gap: 16px; margin-bottom: 18px; } -.section-heading { margin-top: 34px; } -.topology { display: grid; grid-template-columns: 1fr 48px 1fr 48px 2fr; align-items: stretch; gap: 8px; } -.topology-node { border: 1px solid var(--border); background: var(--background); border-radius: 10px; padding: 14px; min-width: 0; } -.topology-node strong { display: block; font-size: 13px; } -.topology-node span { display: block; color: var(--muted-foreground); font-size: 12px; margin-top: 4px; overflow: hidden; text-overflow: ellipsis; } -.topology-cells { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 8px; } -.connector { display: grid; place-items: center; color: var(--muted-foreground); } -.connector::before { content: ''; width: 100%; height: 1px; background: var(--border); } -.chart-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-bottom: 36px; } -.chart-card { min-height: 220px; } -.chart-head { display: flex; justify-content: space-between; align-items: start; } -.chart-value { font-size: 22px; font-weight: 600; letter-spacing: -.03em; } -.chart-unit { color: var(--muted-foreground); font-size: 11px; } -.chart { width: 100%; height: 122px; margin-top: 18px; overflow: visible; } -.chart path.area { fill: color-mix(in srgb, var(--foreground) 5%, transparent); } -.chart path.line { fill: none; stroke: var(--foreground); stroke-width: 1.5; vector-effect: non-scaling-stroke; } -.chart line { stroke: var(--border); stroke-width: 1; } -.chart-empty { color: var(--muted-foreground); height: 120px; display: grid; place-items: center; font-size: 12px; } -.two-column { display: grid; grid-template-columns: 1.7fr 1fr; gap: 12px; margin-bottom: 12px; } -.three-column { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-bottom: 12px; } -.two-column > *, .three-column > *, .chart-grid > * { min-width: 0; } -.table-wrap { overflow-x: auto; } -table { width: 100%; border-collapse: collapse; font-size: 12px; } -th { color: var(--muted-foreground); font-weight: 500; text-align: left; padding: 0 10px 10px; } -td { padding: 12px 10px; border-top: 1px solid var(--border); vertical-align: middle; } -td:first-child, th:first-child { padding-left: 0; } -td:last-child, th:last-child { padding-right: 0; } -.mono { font-family: var(--font-mono); font-size: 11px; } -.badge { display: inline-flex; align-items: center; border: 1px solid var(--border); border-radius: 999px; padding: 2px 7px; font-size: 11px; color: var(--muted-foreground); } -.badge.healthy { color: var(--success); border-color: color-mix(in srgb, var(--success) 25%, var(--border)); } -.badge.unhealthy { color: var(--destructive); border-color: color-mix(in srgb, var(--destructive) 25%, var(--border)); } -.service-list, .list { display: grid; } -.service-row, .list-row { padding: 11px 0; border-top: 1px solid var(--border); display: flex; align-items: center; justify-content: space-between; gap: 12px; min-width: 0; } -.service-row:first-child, .list-row:first-child { border-top: 0; padding-top: 0; } -.service-row:last-child, .list-row:last-child { padding-bottom: 0; } -.row-title { font-size: 12px; font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } -.row-caption, .meta, .muted { color: var(--muted-foreground); font-size: 11px; } -.panel-note { color: var(--muted-foreground); font-size: 11px; line-height: 1.45; margin: -8px 0 14px; } -.panel-link:hover { color: var(--foreground); text-decoration: underline; } -.row-caption { margin-top: 3px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } -a { color: inherit; text-decoration: none; } -a:hover .row-title { text-decoration: underline; } -.cost-total { display: flex; align-items: baseline; gap: 7px; margin-bottom: 12px; } -.cost-total strong { font-size: 28px; letter-spacing: -.035em; } -.cost-line { display: flex; justify-content: space-between; gap: 10px; padding: 7px 0; border-top: 1px solid var(--border); font-size: 12px; } -.cost-note { color: var(--muted-foreground); font-size: 11px; line-height: 1.5; margin-top: 12px; } -.banner { border: 1px solid var(--border); border-radius: 10px; padding: 11px 13px; margin-bottom: 12px; font-size: 12px; } -.banner.error { color: var(--destructive); } -.banner.warning { color: var(--warning); } -.hidden { display: none !important; } -.power-actions { display: flex; align-items: center; gap: 8px; margin-top: 14px; flex-wrap: wrap; } -footer { border-top: 1px solid var(--border); padding: 24px; color: var(--muted-foreground); font-size: 11px; text-align: center; } -@media (max-width: 1050px) { - .summary-grid, .chart-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } - .three-column { grid-template-columns: 1fr; } - .topology { grid-template-columns: 1fr; } - .connector { height: 20px; } - .connector::before { height: 100%; width: 1px; } -} -@media (max-width: 760px) { - .topbar { height: auto; min-height: 64px; padding: 12px 16px; align-items: stretch; flex-direction: column; gap: 12px; } - .brand div { display: grid; gap: 0; } - .toolbar { flex-wrap: wrap; justify-content: flex-start; } - main { padding: 28px 16px 56px; } - .page-heading { align-items: flex-start; flex-direction: column; } - .summary-grid, .chart-grid, .two-column { grid-template-columns: 1fr; } - .topology-cells { grid-template-columns: 1fr; } - table { min-width: 600px; } -} diff --git a/cloud/apps/relay-ops/src/cost-model.test.ts b/cloud/apps/relay-ops/src/cost-model.test.ts deleted file mode 100644 index 9ee6f6adf9d..00000000000 --- a/cloud/apps/relay-ops/src/cost-model.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { buildCostModel } from './cost-model.js' -import { - RELAY_OPS_ENVIRONMENTS, - relayOpsCellsFromTerraform, - type RelayOpsCellConfig -} from './environment-config.js' -import type { ResourceInventory } from './resource-inventory.js' - -const regionalCellsSource = ` -relay_gce_cells = { - "staging-gce-c3" = { - hostname = "c3" - zone = "us-central1-a" - machine_type = "e2-standard-2" - capacity_requests = 4000 - } - "staging-gce-c4" = { - hostname = "c4" - region = "asia-east2" - zone = "asia-east2-a" - machine_type = "e2-standard-4" - capacity_requests = 6000 - database_pool_max = 10 - initially_enabled = false - } -} -` - -function inventory( - targetSize: number, - activationPolicy: string, - cells: RelayOpsCellConfig[] = RELAY_OPS_ENVIRONMENTS.staging.cells -): ResourceInventory { - return { - director: null, - auth: null, - sql: { state: targetSize ? 'RUNNABLE' : 'STOPPED', activationPolicy, tier: 'db-custom-1-3840', availabilityType: 'ZONAL', databaseVersion: 'POSTGRES_17' }, - certificate: null, - directorEndpoint: { health: null, ready: null, latencyMs: null }, - authEndpoint: { health: null, ready: null, latencyMs: null }, - cells: cells.map((cell) => ({ - ...cell, migName: `mig-${cell.hostname}`, targetSize, runningInstances: targetSize, - stable: true, template: 'template', imageDigest: null, - backendHealth: targetSize ? 'healthy' : 'empty', - endpoint: { health: null, ready: null, latencyMs: null } - })), - warnings: [] - } -} - -describe('buildCostModel', () => { - it('shows the sleeping staging floor without VM or SQL compute', () => { - const result = buildCostModel(RELAY_OPS_ENVIRONMENTS.staging, inventory(0, 'NEVER')) - expect(result.kind).toBe('planning-estimate') - expect(result.monthlyUsd).toBe(34) - expect(result.lines.find((line) => line.label === 'Relay cell VMs')?.monthlyUsd).toBe(0) - expect(result.actualBilling.available).toBe(false) - expect(result.caveats[0]).toContain('not the Cloud Billing invoice') - }) - - it('prices durable machine inventory and network floors by region', () => { - const cells = relayOpsCellsFromTerraform({ - environment: 'staging', - domain: 'relay-staging.onorca.dev', - source: regionalCellsSource - }) - const environment = { ...RELAY_OPS_ENVIRONMENTS.staging, cells } - const result = buildCostModel(environment, inventory(1, 'NEVER', cells)) - const machines = result.lines.find((line) => line.label === 'Relay cell VMs') - const network = result.lines.find((line) => line.label === 'Load balancer and network floor') - - expect(machines).toEqual({ - label: 'Relay cell VMs', - monthlyUsd: 185.79, - basis: '2 configured VM cells at regional machine rates × 730 hours' - }) - - expect(network).toEqual({ - label: 'Load balancer and network floor', - monthlyUsd: 29, - basis: 'shared HTTPS foundation plus NAT floor in 2 configured regions' - }) - }) -}) diff --git a/cloud/apps/relay-ops/src/cost-model.ts b/cloud/apps/relay-ops/src/cost-model.ts deleted file mode 100644 index 39e062597a8..00000000000 --- a/cloud/apps/relay-ops/src/cost-model.ts +++ /dev/null @@ -1,115 +0,0 @@ -import type { - RelayOpsEnvironment, - RelayOpsMachineType, - RelayOpsRegion -} from './environment-config.js' -import type { ResourceInventory } from './resource-inventory.js' - -export type CostLine = { - label: string - monthlyUsd: number - basis: string -} - -export type CostModel = { - kind: 'planning-estimate' - monthlyUsd: number - rangeUsd: [number, number] - actualBilling: { available: false; reason: string } - lines: CostLine[] - caveats: string[] -} - -const HOURS_PER_MONTH = 730 -const MACHINE_HOURLY_USD: Record< - RelayOpsRegion, - Record -> = { - 'us-central1': { - 'e2-standard-2': 0.06701142, - 'e2-standard-4': 0.13402284 - }, - 'asia-east2': { - 'e2-standard-2': 0.0938, - 'e2-standard-4': 0.1875 - } -} -const SHARED_NETWORK_FLOOR_USD = 19 -const REGIONAL_NAT_FLOOR_USD = 5 - -function round(value: number): number { - return Math.round(value * 100) / 100 -} - -export function buildCostModel( - environment: RelayOpsEnvironment, - resources: ResourceInventory -): CostModel { - const runningCells = resources.cells.filter((cell) => (cell.targetSize ?? 0) > 0) - const runningCellCount = runningCells.reduce((sum, cell) => sum + (cell.targetSize ?? 0), 0) - const compute = runningCells.reduce( - (sum, cell) => - sum + (cell.targetSize ?? 0) * MACHINE_HOURLY_USD[cell.region][cell.machineType], - 0 - ) * HOURS_PER_MONTH - const disks = runningCellCount * 30 * 0.1 - const sqlRunning = resources.sql?.activationPolicy === 'ALWAYS' - const sql = sqlRunning ? (environment.id === 'production' ? 105 : 52) : 0 - const cloudRunMinimums = - (resources.director?.minInstances ?? 0) + (resources.auth?.minInstances ?? 0) - const cloudRun = cloudRunMinimums * 10 - const configuredRegions = new Set( - (resources.cells.length > 0 ? resources.cells : environment.cells).map((cell) => cell.region) - ) - const networkFoundation = - SHARED_NETWORK_FLOOR_USD + configuredRegions.size * REGIONAL_NAT_FLOOR_USD - const observability = environment.id === 'production' ? 12 : 5 - const lines: CostLine[] = [ - { - label: 'Relay cell VMs', - monthlyUsd: round(compute), - basis: `${runningCellCount} configured VM cells at regional machine rates × 730 hours` - }, - { - label: 'Cell boot disks', - monthlyUsd: round(disks), - basis: `${runningCellCount} × 30 GB balanced persistent disk` - }, - { - label: 'Cloud SQL', - monthlyUsd: sql, - basis: sqlRunning ? `${resources.sql?.tier ?? 'configured tier'} active` : 'stopped' - }, - { - label: 'Cloud Run minimums', - monthlyUsd: cloudRun, - basis: `${cloudRunMinimums} configured minimum instances` - }, - { - label: 'Load balancer and network floor', - monthlyUsd: networkFoundation, - basis: `shared HTTPS foundation plus NAT floor in ${configuredRegions.size} configured region${configuredRegions.size === 1 ? '' : 's'}` - }, - { - label: 'Logs and monitoring allowance', - monthlyUsd: observability, - basis: 'planning allowance; varies with traffic and retention' - } - ] - const monthlyUsd = round(lines.reduce((sum, line) => sum + line.monthlyUsd, 0)) - return { - kind: 'planning-estimate', - monthlyUsd, - rangeUsd: [round(monthlyUsd * 0.85), round(monthlyUsd * 1.3)], - actualBilling: { - available: false, - reason: 'Cloud Billing export is not configured in either Relay project.' - }, - lines, - caveats: [ - 'This is a modeled run-rate, not the Cloud Billing invoice.', - 'Network egress, actual request traffic, credits, discounts, taxes, and free tiers are excluded.', - 'Use the GCP Billing report for authoritative spend and forecasts.' - ] - } -} diff --git a/cloud/apps/relay-ops/src/dashboard-snapshot.test.ts b/cloud/apps/relay-ops/src/dashboard-snapshot.test.ts deleted file mode 100644 index 247b4b53ed0..00000000000 --- a/cloud/apps/relay-ops/src/dashboard-snapshot.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { DashboardSnapshotCache } from './dashboard-snapshot.js' -import type { DashboardSnapshot } from './dashboard-snapshot.js' -import type { GcloudClient } from './gcloud-client.js' - -const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) } - -function snapshot(kind: 'good' | 'unavailable'): DashboardSnapshot { - const good = kind === 'good' - return { - generatedAt: '2026-07-15T12:00:00.000Z', - resources: { - director: good ? {} : null, - auth: good ? {} : null, - sql: good ? {} : null, - cells: [{ targetSize: good ? 1 : null }] - }, - monitoring: { - warnings: good - ? [] - : ['Cloud Monitoring credentials are unavailable. Run gcloud auth login.'] - }, - summary: { observedConnections: good ? 7 : 0 }, - warnings: good ? [] : ['Google Cloud credentials are unavailable. Run gcloud auth login.'], - stale: false, - staleReason: null - } as unknown as DashboardSnapshot -} - -describe('DashboardSnapshotCache', () => { - it('keeps the last good view when a later credential refresh fails', async () => { - let calls = 0 - const cache = new DashboardSnapshotCache(gcloud, 0, async () => { - calls += 1 - return snapshot(calls === 1 ? 'good' : 'unavailable') - }) - - const first = await cache.read('production', 30) - const second = await cache.read('production', 31) - - expect(first.stale).toBe(false) - expect(second.stale).toBe(true) - expect(second.summary.observedConnections).toBe(7) - expect(second.staleReason).toContain('credentials') - }) - - it('keeps the last good view when a later collector throws', async () => { - let calls = 0 - const cache = new DashboardSnapshotCache(gcloud, 0, async () => { - calls += 1 - if (calls > 1) throw new Error('sensitive collector context') - return snapshot('good') - }) - - await cache.read('production', 30) - const second = await cache.read('production', 31) - - expect(second.stale).toBe(true) - expect(second.summary.observedConnections).toBe(7) - expect(JSON.stringify(second)).not.toContain('sensitive collector context') - }) -}) diff --git a/cloud/apps/relay-ops/src/dashboard-snapshot.ts b/cloud/apps/relay-ops/src/dashboard-snapshot.ts deleted file mode 100644 index 0d6f0c1151b..00000000000 --- a/cloud/apps/relay-ops/src/dashboard-snapshot.ts +++ /dev/null @@ -1,167 +0,0 @@ -import type { RelayOpsEnvironmentId } from './environment-config.js' -import { relayOpsEnvironment } from './environment-config.js' -import type { GcloudClient } from './gcloud-client.js' -import { readRelayWorkflowRuns } from './github-runs.js' -import { readMonitoringSnapshot } from './monitoring-snapshot.js' -import { readResourceInventory } from './resource-inventory.js' -import { buildCostModel } from './cost-model.js' - -export type DashboardSnapshot = Awaited> - -export async function buildDashboardSnapshot( - environmentId: RelayOpsEnvironmentId, - gcloud: GcloudClient, - options: { windowMinutes?: number; fetchImpl?: typeof fetch; now?: Date } = {} -) { - const generatedAt = (options.now ?? new Date()).toISOString() - const environment = relayOpsEnvironment(environmentId) - const [monitoringResult, resourceResult, workflowResult] = await Promise.allSettled([ - readMonitoringSnapshot(environment, gcloud, { - ...(options.now === undefined ? {} : { now: options.now }), - ...(options.windowMinutes === undefined ? {} : { windowMinutes: options.windowMinutes }), - ...(options.fetchImpl === undefined ? {} : { fetchImpl: options.fetchImpl }) - }), - readResourceInventory(environment, gcloud, options.fetchImpl), - readRelayWorkflowRuns() - ]) - if (monitoringResult.status === 'rejected' || resourceResult.status === 'rejected') { - const failed = [ - monitoringResult.status === 'rejected' ? 'Monitoring snapshot' : null, - resourceResult.status === 'rejected' ? 'Resource inventory' : null - ].filter(Boolean) - throw new Error(`${failed.join(' and ')} unavailable`) - } - const resources = resourceResult.value - const monitoring = monitoringResult.value - const warnings = [...resources.warnings, ...monitoring.warnings] - const poweredDigests = new Set( - resources.cells.filter((cell) => (cell.targetSize ?? 0) > 0).map((cell) => cell.imageDigest) - ) - if (poweredDigests.has(null)) warnings.push('A powered cell image digest is unavailable.') - if (poweredDigests.size > 1) warnings.push('Powered cells are not serving one immutable digest.') - const expectedCertificateDomain = `*.${new URL(environment.cells[0]!.origin).hostname - .split('.').slice(1).join('.')}` - if (resources.certificate && !resources.certificate.domains.includes(expectedCertificateDomain)) { - warnings.push('The Relay certificate domain does not match the configured cell domain.') - } - if (workflowResult.status === 'rejected') warnings.push('GitHub workflow history is unavailable.') - const observedConnections = monitoring.metrics.total_connections.latest ?? 0 - const observedControls = monitoring.metrics.controls.latest ?? 0 - const observedSplices = monitoring.metrics.splices.latest ?? 0 - const configuredCapacity = environment.cells - .filter((cell) => cell.configuredAdmission) - .reduce((sum, cell) => sum + cell.capacityRequests, 0) - const cellInventoryAvailable = resources.cells.every((cell) => cell.targetSize !== null) - const poweredCapacity = cellInventoryAvailable - ? resources.cells - .filter((cell) => (cell.targetSize ?? 0) > 0) - .reduce((sum, cell) => sum + cell.capacityRequests, 0) - : null - return { - schemaVersion: 1, - generatedAt, - environment: { - id: environment.id, - label: environment.label, - project: environment.project, - region: environment.region, - directorOrigin: environment.directorOrigin, - authOrigin: environment.authOrigin, - consoleLinks: { - project: `https://console.cloud.google.com/home/dashboard?project=${environment.project}`, - alerts: `https://console.cloud.google.com/monitoring/alerting?project=${environment.project}`, - compute: `https://console.cloud.google.com/compute/instanceGroups/list?project=${environment.project}` - } - }, - summary: { - observedConnections, - observedControls, - observedSplices, - configuredCapacity, - poweredCapacity, - activeCells: cellInventoryAvailable - ? resources.cells.filter( - (cell) => (cell.targetSize ?? 0) > 0 && cell.backendHealth === 'healthy' - ).length - : null, - totalCells: resources.cells.length - }, - resources, - monitoring, - workflows: workflowResult.status === 'fulfilled' ? workflowResult.value : [], - cost: buildCostModel(environment, resources), - warnings, - stale: false, - staleReason: null as string | null - } -} - -type SnapshotCacheEntry = { snapshot: DashboardSnapshot; expiresAt: number } -type SnapshotBuilder = ( - environment: RelayOpsEnvironmentId, - gcloud: GcloudClient, - options: { windowMinutes?: number } -) => Promise - -export class DashboardSnapshotCache { - private readonly entries = new Map() - private readonly pending = new Map>() - private readonly lastGood = new Map() - - constructor( - private readonly gcloud: GcloudClient, - private readonly ttlMs = 30_000, - private readonly builder: SnapshotBuilder = buildDashboardSnapshot - ) {} - - async read(environment: RelayOpsEnvironmentId, windowMinutes: number): Promise { - const key = `${environment}:${windowMinutes}` - const cached = this.entries.get(key) - if (cached && cached.expiresAt > Date.now()) return cached.snapshot - const existing = this.pending.get(key) - if (existing) return await existing - const request = this.builder(environment, this.gcloud, { windowMinutes }) - .then((snapshot) => { - const coreInventoryUnavailable = - snapshot.resources.director === null && - snapshot.resources.auth === null && - snapshot.resources.sql === null && - snapshot.resources.cells.every((cell) => cell.targetSize === null) - const monitoringCredentialsUnavailable = snapshot.monitoring.warnings.some( - (warning) => warning.includes('credentials are unavailable') - ) - const lastGood = this.lastGood.get(environment) - const result = (coreInventoryUnavailable || monitoringCredentialsUnavailable) && lastGood - ? { - ...lastGood, - stale: true, - staleReason: 'Local Google Cloud credentials are temporarily unavailable.', - warnings: [...new Set([...lastGood.warnings, ...snapshot.warnings])] - } - : snapshot - if (!coreInventoryUnavailable && !monitoringCredentialsUnavailable) { - this.lastGood.set(environment, snapshot) - } - this.entries.set(key, { snapshot: result, expiresAt: Date.now() + this.ttlMs }) - return result - }) - .catch((error: unknown) => { - const lastGood = this.lastGood.get(environment) - if (!lastGood) throw error - const stale = { - ...lastGood, - stale: true, - staleReason: 'The latest operations refresh failed; showing the last good snapshot.', - warnings: [...new Set([ - ...lastGood.warnings, - 'The latest operations refresh failed before a complete snapshot was available.' - ])] - } - this.entries.set(key, { snapshot: stale, expiresAt: Date.now() + this.ttlMs }) - return stale - }) - .finally(() => this.pending.delete(key)) - this.pending.set(key, request) - return await request - } -} diff --git a/cloud/apps/relay-ops/src/environment-config.test.ts b/cloud/apps/relay-ops/src/environment-config.test.ts deleted file mode 100644 index f3522a300cd..00000000000 --- a/cloud/apps/relay-ops/src/environment-config.test.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { readFileSync } from 'node:fs' -import { fileURLToPath } from 'node:url' -import { describe, expect, it } from 'vitest' -import { - RELAY_OPS_ENVIRONMENTS, - relayOpsCellsFromTerraform -} from './environment-config.js' - -const durableUsCell = ` - "production-gce-c26" = { - hostname = "c26" - zone = "us-central1-a" - machine_type = "e2-standard-4" - capacity_requests = 4000 - initially_enabled = false - } -` - -const durableAsiaCells = ` - "production-gce-c27" = { - hostname = "c27" - region = "asia-east2" - zone = "asia-east2-a" - machine_type = "e2-standard-4" - capacity_requests = 6000 - database_pool_max = 10 - initially_enabled = false - } - "production-gce-c28" = { - hostname = "c28" - region = "asia-east2" - zone = "asia-east2-b" - machine_type = "e2-standard-4" - capacity_requests = 6000 - database_pool_max = 10 - initially_enabled = false - } - "production-gce-c29" = { - hostname = "c29" - region = "asia-east2" - zone = "asia-east2-c" - machine_type = "e2-standard-4" - capacity_requests = 6000 - database_pool_max = 10 - initially_enabled = false - } -` - -const durableAsiaSource = ` -relay_gce_cells = {${durableUsCell}${durableAsiaCells}} -` - -const durableUsOnlySource = ` -relay_gce_cells = {${durableUsCell}} -` - -// Why: relay-ops sat at 18 cells for four days after C19-C22 shipped, which threw -// `selector membership must contain every configured cell exactly once` and blocked -// every production mutation. Reading Terraform here makes that drift fail the build. -function terraformCells(environment: 'production' | 'staging'): Array<{ - cellId: string - region: string - zone: string - machineType: string - capacityRequests: number - databasePoolMax: number - connectionHardCap: number | null -}> { - const tfvars = readFileSync( - fileURLToPath(new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url)), - 'utf8' - ) - const block = /relay_gce_cells\s*=\s*\{([\s\S]*)\n\}/.exec(tfvars)?.[1] ?? '' - return [...block.matchAll(/"([a-z]+-gce-c\d+)"\s*=\s*\{([\s\S]*?)\n {2}\}/g)] - .map((match) => ({ - cellId: match[1] ?? '', - region: /region\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? 'us-central1', - zone: /zone\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '', - machineType: /machine_type\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '', - capacityRequests: Number(/capacity_requests\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1]), - databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10), - connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))( - /connection_hard_cap\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] - ) - })) - .sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId)) -} - -const cellOrdinal = (cellId: string): number => Number(/c(\d+)$/.exec(cellId)?.[1] ?? 0) - -describe('relay operations environment config', () => { - it.each(['production', 'staging'] as const)( - 'matches the %s cells Terraform actually provisions', - (environment) => { - const expected = terraformCells(environment) - expect(expected.length).toBeGreaterThan(0) - expect( - RELAY_OPS_ENVIRONMENTS[environment].cells.map((cell) => ({ - cellId: cell.cellId, - region: cell.region, - zone: cell.zone, - machineType: cell.machineType, - capacityRequests: cell.capacityRequests, - databasePoolMax: cell.databasePoolMax, - connectionHardCap: cell.connectionHardCap - })) - ).toEqual(expected) - } - ) - - it('derives hostname and origin from the cell ordinal', () => { - const cells = RELAY_OPS_ENVIRONMENTS.production.cells - expect(cells.at(-1)).toMatchObject({ - hostname: `c${cells.length}`, - origin: `https://c${cells.length}.relay.onorca.dev` - }) - }) - - it('inventories Asia cells only when they exist in durable Terraform', () => { - const cells = relayOpsCellsFromTerraform({ - environment: 'production', - domain: 'relay.onorca.dev', - source: durableAsiaSource - }) - - expect(cells.slice(1)).toEqual([ - { - cellId: 'production-gce-c27', hostname: 'c27', origin: 'https://c27.relay.onorca.dev', - region: 'asia-east2', zone: 'asia-east2-a', machineType: 'e2-standard-4', - capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null, - configuredAdmission: false - }, - { - cellId: 'production-gce-c28', hostname: 'c28', origin: 'https://c28.relay.onorca.dev', - region: 'asia-east2', zone: 'asia-east2-b', machineType: 'e2-standard-4', - capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null, - configuredAdmission: false - }, - { - cellId: 'production-gce-c29', hostname: 'c29', origin: 'https://c29.relay.onorca.dev', - region: 'asia-east2', zone: 'asia-east2-c', machineType: 'e2-standard-4', - capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null, - configuredAdmission: false - } - ]) - - const usOnlyCells = relayOpsCellsFromTerraform({ - environment: 'production', - domain: 'relay.onorca.dev', - source: durableUsOnlySource - }) - expect(usOnlyCells).toHaveLength(1) - expect(usOnlyCells.some((cell) => cell.region === 'asia-east2')).toBe(false) - expect(usOnlyCells.some((cell) => cell.cellId === 'production-gce-c27')).toBe(false) - }) -}) diff --git a/cloud/apps/relay-ops/src/environment-config.ts b/cloud/apps/relay-ops/src/environment-config.ts deleted file mode 100644 index efd890367c8..00000000000 --- a/cloud/apps/relay-ops/src/environment-config.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { readFileSync } from 'node:fs' -import { z } from 'zod' - -export type RelayOpsEnvironmentId = 'production' | 'staging' -export type RelayOpsRegion = 'us-central1' | 'asia-east2' -export type RelayOpsMachineType = 'e2-standard-2' | 'e2-standard-4' - -export type RelayOpsCellConfig = { - cellId: string - hostname: string - origin: string - region: RelayOpsRegion - zone: string - machineType: RelayOpsMachineType - capacityRequests: number - databasePoolMax: number - // The cell's rendered connectionHardCap; null when Terraform leaves it unset. - connectionHardCap: number | null - configuredAdmission: boolean -} - -export type RelayOpsEnvironment = { - id: RelayOpsEnvironmentId - label: string - project: string - region: string - directorOrigin: string - authOrigin: string - directorService: string - authService: string - sqlInstance: string - migPrefix: string - certificateName: string - cells: RelayOpsCellConfig[] -} - -const RegionSchema = z.enum(['us-central1', 'asia-east2']) -const MachineTypeSchema = z.enum(['e2-standard-2', 'e2-standard-4']) -const EnvironmentSchema = z.enum(['production', 'staging']) - -function cellOrdinal(cellId: string): number { - return Number(/c(\d+)$/.exec(cellId)?.[1] ?? 0) -} - -function required(body: string, pattern: RegExp, label: string): string { - const value = pattern.exec(body)?.[1] - if (!value) throw new Error(`Relay Ops could not read ${label} from durable Terraform config`) - return value -} - -export function relayOpsCellsFromTerraform(input: { - environment: RelayOpsEnvironmentId - domain: string - source: string -}): RelayOpsCellConfig[] { - const block = /relay_gce_cells\s*=\s*\{([\s\S]*)\n\}/.exec(input.source)?.[1] - if (!block) throw new Error('Relay Ops could not read durable Relay cells') - return [...block.matchAll(/"([a-z]+-gce-c\d+)"\s*=\s*\{([\s\S]*?)\n {2}\}/g)] - .map((match) => { - const cellId = match[1]! - const body = match[2]! - const hostname = required(body, /\bhostname\s*=\s*"([^"]+)"/, `${cellId} hostname`) - const configuredAdmission = /\binitially_enabled\s*=\s*(true|false)/.exec(body)?.[1] - return { - cellId, - hostname, - origin: `https://${hostname}.${input.domain}`, - region: RegionSchema.parse( - /\bregion\s*=\s*"([^"]+)"/.exec(body)?.[1] ?? 'us-central1' - ), - zone: required(body, /\bzone\s*=\s*"([^"]+)"/, `${cellId} zone`), - machineType: MachineTypeSchema.parse( - required(body, /\bmachine_type\s*=\s*"([^"]+)"/, `${cellId} machine type`) - ), - capacityRequests: Number( - required(body, /\bcapacity_requests\s*=\s*(\d+)/, `${cellId} capacity`) - ), - databasePoolMax: Number(/\bdatabase_pool_max\s*=\s*(\d+)/.exec(body)?.[1] ?? 10), - connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))( - /\bconnection_hard_cap\s*=\s*(\d+)/.exec(body)?.[1] - ), - configuredAdmission: configuredAdmission === undefined || configuredAdmission === 'true' - } - }) - .sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId)) -} - -function durableCells(environment: RelayOpsEnvironmentId, domain: string): RelayOpsCellConfig[] { - const source = readFileSync( - // Repository root; infra/terraform moves with this tree, so the relative depth holds. - new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url), - 'utf8' - ) - return relayOpsCellsFromTerraform({ environment, domain, source }) -} - -export const RELAY_OPS_ENVIRONMENTS: Record = { - production: { - id: 'production', - label: 'Production', - project: 'onorca-cloud', - region: 'us-central1', - directorOrigin: 'https://relay.onorca.dev', - authOrigin: 'https://login.onorca.dev', - directorService: 'orca-cloud-relay', - authService: 'orca-cloud-auth', - sqlInstance: 'orca-cloud-auth-db', - migPrefix: 'orca-cloud-relay-gce-', - certificateName: 'orca-cloud-relay-gce', - cells: durableCells('production', 'relay.onorca.dev') - }, - staging: { - id: 'staging', - label: 'Staging', - project: 'onorca-cloud-staging', - region: 'us-central1', - directorOrigin: 'https://relay-staging.onorca.dev', - authOrigin: 'https://auth-staging.onorca.dev', - directorService: 'orca-cloud-relay-staging', - authService: 'orca-cloud-auth-staging', - sqlInstance: 'orca-cloud-staging-auth-db', - migPrefix: 'orca-cloud-staging-relay-gce-', - certificateName: 'orca-cloud-staging-relay-gce', - cells: durableCells('staging', 'relay-staging.onorca.dev') - } -} - -export function relayOpsEnvironment(value: unknown): RelayOpsEnvironment { - return RELAY_OPS_ENVIRONMENTS[EnvironmentSchema.parse(value)] -} diff --git a/cloud/apps/relay-ops/src/gcloud-client.test.ts b/cloud/apps/relay-ops/src/gcloud-client.test.ts deleted file mode 100644 index 3347c0cb367..00000000000 --- a/cloud/apps/relay-ops/src/gcloud-client.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { createGcloudClient } from './gcloud-client.js' - -describe('createGcloudClient', () => { - it('shares and caches one credential refresh across concurrent readers', async () => { - let calls = 0 - const token = 'a'.repeat(40) - const client = createGcloudClient(async () => { - calls += 1 - await Promise.resolve() - return token - }) - - const values = await Promise.all([ - client.accessToken(), - client.accessToken(), - client.accessToken() - ]) - - expect(values).toEqual([token, token, token]) - expect(await client.accessToken()).toBe(token) - expect(calls).toBe(1) - }) - - it('caches bounded identity tokens by audience', async () => { - const commands: string[][] = [] - const token = 'aaa.bbb.ccc' - const client = createGcloudClient(async (args) => { - commands.push(args) - return token - }) - await expect(client.identityToken?.('https://relay.example/admin')).resolves.toBe(token) - await expect(client.identityToken?.('https://relay.example/admin')).resolves.toBe(token) - expect(commands).toEqual([ - [ - 'auth', - 'print-identity-token', - '--audiences=https://relay.example/admin', - '--include-email' - ] - ]) - }) -}) diff --git a/cloud/apps/relay-ops/src/gcloud-client.ts b/cloud/apps/relay-ops/src/gcloud-client.ts deleted file mode 100644 index b565242ec0e..00000000000 --- a/cloud/apps/relay-ops/src/gcloud-client.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { execFile } from 'node:child_process' -import { promisify } from 'node:util' - -const execFileAsync = promisify(execFile) -const TOKEN_PATTERN = /^[A-Za-z0-9._~+\/-]{32,8192}$/ - -export type GcloudClient = { - accessToken(): Promise - identityToken?(audience: string): Promise -} - -export class GcloudCommandError extends Error { - constructor(readonly operation: string) { - super(`${operation} is unavailable`) - } -} - -async function runGcloud(args: string[]): Promise { - try { - const result = await execFileAsync('gcloud', args, { - encoding: 'utf8', - timeout: 90_000, - maxBuffer: 8 * 1024 * 1024, - env: { - ...process.env, - CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK: '1', - CLOUDSDK_CORE_DISABLE_PROMPTS: '1', - CLOUDSDK_CORE_DISABLE_USAGE_REPORTING: '1' - } - }) - return result.stdout.trim() - } catch { - // Gcloud stderr can echo command context; keep dashboard errors intentionally non-sensitive. - throw new GcloudCommandError(`gcloud ${args.slice(0, 3).join(' ')}`) - } -} - -export function createGcloudClient( - tokenCommand: (args: string[]) => Promise = runGcloud -): GcloudClient { - let cachedToken: { value: string; expiresAt: number } | null = null - const identityTokens = new Map() - let pendingToken: Promise | null = null - return { - async accessToken(): Promise { - if (cachedToken && cachedToken.expiresAt > Date.now()) return cachedToken.value - if (pendingToken) return await pendingToken - // One refresh avoids concurrent gcloud processes contending on the local credential store. - pendingToken = tokenCommand(['auth', 'print-access-token']) - .then((token) => { - if (!TOKEN_PATTERN.test(token)) throw new GcloudCommandError('gcloud access token') - cachedToken = { value: token, expiresAt: Date.now() + 5 * 60_000 } - return token - }) - .finally(() => { pendingToken = null }) - return await pendingToken - }, - async identityToken(audience: string): Promise { - const cached = identityTokens.get(audience) - if (cached && cached.expiresAt > Date.now()) return cached.value - const token = await tokenCommand([ - 'auth', - 'print-identity-token', - `--audiences=${audience}`, - '--include-email' - ]) - if ( - token.length > 8_192 || - !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token) - ) { - throw new GcloudCommandError('gcloud identity token') - } - identityTokens.set(audience, { value: token, expiresAt: Date.now() + 5 * 60_000 }) - return token - } - } -} diff --git a/cloud/apps/relay-ops/src/github-runs.ts b/cloud/apps/relay-ops/src/github-runs.ts deleted file mode 100644 index 47ef8469fa9..00000000000 --- a/cloud/apps/relay-ops/src/github-runs.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { execFile } from 'node:child_process' -import { promisify } from 'node:util' -import { z } from 'zod' -import { relayRepositoryApiPath } from './relay-repository.js' - -const execFileAsync = promisify(execFile) - -const WorkflowRunSchema = z.object({ - id: z.number().int().positive(), - name: z.string(), - event: z.string(), - status: z.string(), - conclusion: z.string().nullable(), - head_sha: z.string(), - html_url: z.string().url(), - created_at: z.string(), - updated_at: z.string() -}) - -const WorkflowRunsSchema = z.object({ - workflow_runs: z.array(WorkflowRunSchema) -}) - -export type RelayWorkflowRun = { - id: number - name: string - status: string - conclusion: string | null - headSha: string - url: string - createdAt: string - updatedAt: string -} - -const RELAY_WORKFLOW_PATTERN = /(Relay|Auth|Power)/i - -export async function readRelayWorkflowRuns(): Promise { - let stdout: string - try { - const result = await execFileAsync( - 'gh', - [ - 'api', - '--method', - 'GET', - relayRepositoryApiPath('actions/runs'), - '-f', - 'per_page=100' - ], - { encoding: 'utf8', timeout: 30_000, maxBuffer: 4 * 1024 * 1024 } - ) - stdout = result.stdout - } catch { - throw new Error('GitHub workflow history is unavailable') - } - const runs = WorkflowRunsSchema.parse(JSON.parse(stdout) as unknown).workflow_runs - const counts = new Map() - return runs - .filter((run) => { - if (!RELAY_WORKFLOW_PATTERN.test(run.name)) return false - const count = counts.get(run.name) ?? 0 - if (count >= 2) return false - counts.set(run.name, count + 1) - return true - }) - .slice(0, 12) - .map((run) => ({ - id: run.id, - name: run.name, - status: run.status, - conclusion: run.conclusion, - headSha: run.head_sha.slice(0, 8), - url: run.html_url, - createdAt: run.created_at, - updatedAt: run.updated_at - })) -} diff --git a/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts b/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts deleted file mode 100644 index a382d0d0910..00000000000 --- a/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts +++ /dev/null @@ -1,874 +0,0 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, describe, expect, it, vi } from 'vitest' -import { - livePreflightGcloud, - runIncidentLivePreflight -} from './incident-live-preflight-cli.js' -import { - INCIDENT_MONITOR_THRESHOLDS, - type IncidentSample -} from './incident-monitor.js' -import { relayOpsEnvironment } from './environment-config.js' -import type { AdmissionSelector } from './incident-selector.js' - -const directories: string[] = [] -const now = Date.parse('2026-07-28T12:00:00.000Z') -const selector = { - generation: 1, - membership: { - existingOnly: ['production-gce-c1'], - migrationOnly: [], - general: [] - } -} - -function stateFile( - migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition' = 'strict', - overrides: Record = {} -): string { - const directory = mkdtempSync(join(tmpdir(), 'relay-live-preflight-')) - directories.push(directory) - const path = join(directory, 'state.json') - const expectedSelector = migrationPolicy === 'capacity-transition' - ? { - generation: 1, - membership: { - existingOnly: [], - migrationOnly: [], - general: ['production-gce-c1'] - } - } - : selector - writeFileSync(path, JSON.stringify({ - schemaVersion: 4, - environment: 'production', - expectedSelector, - migrationPolicy, - recoverySourceCellId: - migrationPolicy === 'recover-forward' ? 'production-gce-c1' : null, - capacityCellId: - migrationPolicy === 'capacity-transition' ? 'production-gce-c1' : null, - preDrainDryRun: true, - startedAt: new Date(now - 17 * 60_000).toISOString(), - windowStartedAt: new Date(now - 16 * 60_000).toISOString(), - durationMinutes: 15, - intervalMs: 60_000, - sampleCount: 16, - lastSampleAt: new Date(now - 60_007).toISOString(), - frozenAt: null, - completedAt: new Date(now - 60_000).toISOString(), - ...overrides - })) - return path -} - -// Every configured production cell, in the lexicographic order -// normalizeSelectorMembership canonicalises to. Derived from the same durable -// Terraform config the override path reads, so a new cell cannot strand these. -const configuredCellIds = relayOpsEnvironment('production').cells.map( - (cell) => cell.cellId -) -const canonicalCellIds = [...configuredCellIds].sort() -const canonicalMembership = { - existingOnly: canonicalCellIds, - migrationOnly: [], - general: [] -} - -// What the director reports: a normalised selector, never an echo of what the -// caller expected. An order-sensitive comparison only holds if the override path -// canonicalises its own input the same way. -function canonicalSample(generation = 1): IncidentSample { - const next = sample() - next.selector = { generation, membership: canonicalMembership } - return next -} - -function membershipFile( - membership: Record = canonicalMembership -): string { - const directory = mkdtempSync(join(tmpdir(), 'relay-live-preflight-selector-')) - directories.push(directory) - const path = join(directory, 'selector.json') - writeFileSync(path, JSON.stringify(membership)) - return path -} - -function sample(): IncidentSample { - const observedAt = new Date(now).toISOString() - const signal = (value: number) => ({ value, observedAt }) - return { - collectedAt: observedAt, - selector, - expectedSelector: selector, - cells: [{ - cellId: 'production-gce-c1', - region: 'us-central1', - runtimeKnown: true, - powered: true, - expectedAdmissionState: 'existing-only' - }], - sources: { - 'active-probe': { - observedAt, - signals: { - 'director.health': signal(1), - 'director.ready': signal(1), - 'director.latency_ms': signal(1), - 'auth.health': signal(1), - 'auth.ready': signal(1), - 'auth.latency_ms': signal(1), - 'cell.production-gce-c1.health': signal(1), - 'cell.production-gce-c1.ready': signal(1), - 'cell.production-gce-c1.latency_ms': signal(1) - } - }, - 'cloud-monitoring': { - observedAt, - signals: { - 'cloud_sql.cpu': signal(0.1), - 'cloud_sql.memory': signal(0.1), - 'cloud_sql.backends': signal(1), - 'cloud_sql.lock_waits': signal(0), - 'cloud_sql.deadlocks': signal(0), - 'director.instances': signal(5), - 'director.cpu': signal(0.1), - 'director.memory': signal(0.1), - 'director.concurrency': signal(1), - 'director.errors': signal(0), - 'auth.errors': signal(0) - } - }, - 'relay-logs': { - observedAt, - signals: { - 'relay.pool_waiting': signal(0), - 'relay.pool_wait_ms': signal(0), - 'relay.postgres_retries': signal(0), - 'relay.postgres_retry_exhausted': signal(0), - 'cell.production-gce-c1.connections': signal(1), - 'cell.production-gce-c1.queued_bytes': signal(0) - } - }, - 'director-admin': { - observedAt, - signals: { - 'cell.production-gce-c1.admission_state': signal(0), - 'cell.production-gce-c1.heartbeat_fresh': signal(1), - 'cell.production-gce-c1.heartbeat_age_ms': signal(1), - 'cell.production-gce-c1.migration_blocked': signal(0), - 'cell.production-gce-c1.migration_target_inactive': signal(0) - } - } - } - } -} - -afterEach(() => { - for (const directory of directories.splice(0)) { - rmSync(directory, { recursive: true, force: true }) - } -}) - -describe('relay incident live preflight', () => { - it('accepts the package-manager argument separator', async () => { - await expect(runIncidentLivePreflight( - ['--', '--state-file', stateFile()], - { now: () => now, collect: async () => sample() } - )).resolves.toBeUndefined() - }) - - it('accepts one complete fresh green sample', async () => { - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { now: () => now, collect: async () => sample() } - )).resolves.toBeUndefined() - }) - - it('rejects monitor evidence beyond the 35-minute lineage bound', async () => { - const path = stateFile('strict', { - startedAt: new Date(now - 36 * 60_000 - 1).toISOString() - }) - await expect(runIncidentLivePreflight( - ['--state-file', path], - { now: () => now, collect: async () => sample() } - )).rejects.toThrow('monitor evidence is incomplete or stale') - }) - - it('scales the evidence age bound by same-cap wave index', async () => { - const agedState = (ageMs: number) => stateFile('strict', { - startedAt: new Date(now - ageMs - 17 * 60_000).toISOString(), - windowStartedAt: new Date(now - ageMs - 16 * 60_000).toISOString(), - lastSampleAt: new Date(now - ageMs - 7).toISOString(), - completedAt: new Date(now - ageMs).toISOString() - }) - const deps = { now: () => now, collect: async () => sample() } - // One predecessor cell roll (~16 min) exceeds wave 0 but fits wave 1. - const oneRollOld = agedState(17 * 60_000) - await expect(runIncidentLivePreflight( - ['--state-file', oneRollOld], deps - )).rejects.toThrow('monitor evidence is incomplete or stale') - await expect(runIncidentLivePreflight( - ['--state-file', oneRollOld, '--wave-index', '0'], deps - )).rejects.toThrow('monitor evidence is incomplete or stale') - await expect(runIncidentLivePreflight( - ['--state-file', oneRollOld, '--wave-index', '1'], deps - )).resolves.toBeUndefined() - // Wave 0 edges: the 10-minute bound covers same-cap job start-up latency. - await expect(runIncidentLivePreflight( - ['--state-file', agedState(10 * 60_000), '--wave-index', '0'], deps - )).resolves.toBeUndefined() - await expect(runIncidentLivePreflight( - ['--state-file', agedState(10 * 60_000 + 1), '--wave-index', '0'], deps - )).rejects.toThrow('monitor evidence is incomplete or stale') - // Both edges of one predecessor job timeout: 10min + 75min exactly. - await expect(runIncidentLivePreflight( - ['--state-file', agedState(85 * 60_000), '--wave-index', '1'], deps - )).resolves.toBeUndefined() - await expect(runIncidentLivePreflight( - ['--state-file', agedState(85 * 60_000 + 1), '--wave-index', '1'], deps - )).rejects.toThrow('monitor evidence is incomplete or stale') - await expect(runIncidentLivePreflight( - ['--state-file', agedState(160 * 60_000), '--wave-index', '2'], deps - )).resolves.toBeUndefined() - await expect(runIncidentLivePreflight( - ['--state-file', agedState(160 * 60_000 + 1), '--wave-index', '2'], deps - )).rejects.toThrow('monitor evidence is incomplete or stale') - await expect(runIncidentLivePreflight( - ['--state-file', agedState(235 * 60_000), '--wave-index', '3'], deps - )).resolves.toBeUndefined() - await expect(runIncidentLivePreflight( - ['--state-file', agedState(235 * 60_000 + 1), '--wave-index', '3'], deps - )).rejects.toThrow('monitor evidence is incomplete or stale') - // The last cell of a ten-cell same-cap batch: 10min + 9 * 75min exactly. - await expect(runIncidentLivePreflight( - ['--state-file', agedState(685 * 60_000), '--wave-index', '9'], deps - )).resolves.toBeUndefined() - await expect(runIncidentLivePreflight( - ['--state-file', agedState(685 * 60_000 + 1), '--wave-index', '9'], deps - )).rejects.toThrow('monitor evidence is incomplete or stale') - // The wave index is a strict single-use 0-9 argument. - await expect(runIncidentLivePreflight( - ['--state-file', stateFile(), '--wave-index', '10'], deps - )).rejects.toThrow('usage:') - await expect(runIncidentLivePreflight( - ['--state-file', stateFile(), '--wave-index', ''], deps - )).rejects.toThrow('usage:') - await expect(runIncidentLivePreflight( - ['--state-file', stateFile(), '--wave-index', '1', '--wave-index', '1'], - deps - )).rejects.toThrow('usage:') - }) - - it('expects the wave-adjusted live selector generation', async () => { - const agedPath = stateFile('strict', { - startedAt: new Date(now - 34 * 60_000).toISOString(), - windowStartedAt: new Date(now - 33 * 60_000).toISOString(), - lastSampleAt: new Date(now - 17 * 60_000 - 7).toISOString(), - completedAt: new Date(now - 17 * 60_000).toISOString() - }) - const liveAt = (generation: number) => - async (expectedSelector: AdmissionSelector) => ({ - ...sample(), - selector: { ...selector, generation }, - expectedSelector - }) - // One predecessor roll advanced the live selector by exactly 2. - await expect(runIncidentLivePreflight( - ['--state-file', agedPath, '--wave-index', '1'], - { now: () => now, collect: liveAt(selector.generation + 2) } - )).resolves.toBeUndefined() - // The sealed pre-roll generation must no longer satisfy wave 1. - await expect(runIncidentLivePreflight( - ['--state-file', agedPath, '--wave-index', '1'], - { now: () => now, collect: liveAt(selector.generation) } - )).rejects.toThrow('director-admin/selector_mismatch') - // Wave 0 still expects the sealed generation itself. - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { now: () => now, collect: liveAt(selector.generation) } - )).resolves.toBeUndefined() - }) - - it('fails closed on a live threshold breach', async () => { - const unhealthy = sample() - unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9 - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { now: () => now, collect: async () => unhealthy } - )).rejects.toThrow('cloud-monitoring/threshold_max') - }) - - // Why: a frozen wave has to name what froze it without re-reading the sample. - it('names the signal and its numbers in the failure message', async () => { - const slowCell = sample() - slowCell.sources['active-probe']!.signals[ - 'cell.production-gce-c1.latency_ms' - ]!.value = 2_568 - // A cell probe now re-samples before it fails a wave, so the wait is injected; - // this cell stays slow on every sample and still names what stopped it. - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { now: () => now, collect: async () => slowCell, wait: async () => {} } - )).rejects.toThrow( - 'relay live preflight failed: active-probe/threshold_max cell.production-gce-c1.latency_ms observed=2568 threshold=2000' - ) - - // A failure with no signal keeps the source/code token and drops the rest. - const stale = sample() - stale.sources['active-probe']!.observedAt = new Date(now - 60_001).toISOString() - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { now: () => now, collect: async () => stale } - )).rejects.toThrow( - 'relay live preflight failed: active-probe/source_stale observed=60001 threshold=60000' - ) - }) - - // Why: this one sample decides a mutating wave, so an Asia cell's ~30 s - // "no healthy upstream" window could still fail a wave here even after the - // 15-minute gate learned to ride it out. - describe('cell probe tolerance', () => { - const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - - // Serves `badSamples` unhealthy cell readings, then healthy ones. - const downThen = (badSamples: number) => { - let index = 0 - return async () => { - const next = sample() - if (index++ < badSamples) { - next.sources['active-probe']!.signals['cell.production-gce-c1.health']! - .value = 0 - next.sources['active-probe']!.signals['cell.production-gce-c1.ready']! - .value = 0 - } - return next - } - } - - it('re-samples through a probe outage within the tolerance', async () => { - const waits: number[] = [] - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { - now: () => now, - collect: downThen(tolerance), - wait: async (ms) => { - waits.push(ms) - } - } - )).resolves.toBeUndefined() - expect(waits).toHaveLength(tolerance) - }) - - it('fails the wave once the probe outage outlasts the tolerance', async () => { - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { - now: () => now, - collect: downThen(tolerance + 1), - wait: async () => {} - } - )).rejects.toThrow('active-probe/threshold_equal cell.production-gce-c1.health') - }) - - it('does not re-sample a director probe failure', async () => { - let samples = 0 - const down = async () => { - samples++ - const next = sample() - next.sources['active-probe']!.signals['director.health']!.value = 0 - return next - } - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { now: () => now, collect: down, wait: async () => {} } - )).rejects.toThrow('active-probe/threshold_equal director.health') - expect(samples).toBe(1) - }) - - it('does not re-sample a non-probe threshold failure', async () => { - let samples = 0 - const hot = async () => { - samples++ - const next = sample() - next.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9 - return next - } - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { now: () => now, collect: hot, wait: async () => {} } - )).rejects.toThrow('cloud-monitoring/threshold_max') - expect(samples).toBe(1) - }) - }) - - // Why: on 2026-09-17 a canary wave died because one director admin read - // returned 404 for a 2 s Cloud SQL pool timeout. Collecting the sample is not - // a health verdict, so a thrown collector spends an attempt instead. - describe('collector failures', () => { - it('re-samples after a thrown collector and then passes', async () => { - const waits: number[] = [] - let attempts = 0 - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { - now: () => now, - collect: async () => { - attempts++ - if (attempts === 1) throw new Error('Relay admin telemetry returned 404') - return sample() - }, - wait: async (ms) => { - waits.push(ms) - } - } - )).resolves.toBeUndefined() - expect(attempts).toBe(2) - expect(waits).toEqual([15_000]) - }) - - it('fails the wave when every attempt throws, naming the collector', async () => { - let attempts = 0 - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { - now: () => now, - collect: async () => { - attempts++ - throw new Error('Relay admin telemetry returned 404') - }, - wait: async () => {} - } - )).rejects.toThrow( - 'relay live preflight failed: collector: Relay admin telemetry returned 404' - ) - expect(attempts).toBe(1 + INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples) - }) - - it('does not re-sample a collector failure past the evidence-age budget', async () => { - const agedPath = stateFile('strict', { - startedAt: new Date(now - 26 * 60_000).toISOString(), - windowStartedAt: new Date(now - 25 * 60_000).toISOString(), - lastSampleAt: new Date(now - 10 * 60_000 + 7).toISOString(), - completedAt: new Date(now - 10 * 60_000 + 14).toISOString() - }) - let attempts = 0 - await expect(runIncidentLivePreflight( - ['--state-file', agedPath], - { - now: () => now, - collect: async () => { - attempts++ - throw new Error('Relay admin telemetry returned 404') - }, - wait: async () => {} - } - )).rejects.toThrow('relay live preflight failed: collector:') - expect(attempts).toBe(1) - }) - }) - - it('enforces the signed migration policy', async () => { - const inactiveTarget = sample() - inactiveTarget.sources['director-admin']!.signals[ - 'cell.production-gce-c1.migration_target_inactive' - ]!.value = 30 - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { now: () => now, collect: async () => inactiveTarget } - )).rejects.toThrow('director-admin/threshold_max') - await expect(runIncidentLivePreflight( - ['--state-file', stateFile('recover-forward')], - { now: () => now, collect: async () => inactiveTarget } - )).resolves.toBeUndefined() - inactiveTarget.sources['director-admin']!.signals[ - 'cell.production-gce-c1.migration_blocked' - ]!.value = 1 - await expect(runIncidentLivePreflight( - ['--state-file', stateFile('recover-forward')], - { now: () => now, collect: async () => inactiveTarget } - )).rejects.toThrow('director-admin/threshold_max') - }) - - it('binds capacity-transition evidence to its general cell', async () => { - const capacitySample = sample() - const capacitySelector = { - generation: 1, - membership: { - existingOnly: [], - migrationOnly: [], - general: ['production-gce-c1'] - } - } - capacitySample.selector = capacitySelector - capacitySample.expectedSelector = capacitySelector - capacitySample.cells[0]!.expectedAdmissionState = 'general' - capacitySample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.admission_state' - ]!.value = 2 - await expect(runIncidentLivePreflight( - ['--state-file', stateFile('capacity-transition')], - { now: () => now, collect: async () => capacitySample } - )).resolves.toBeUndefined() - capacitySample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.migration_target_inactive' - ]!.value = 1 - await expect(runIncidentLivePreflight( - ['--state-file', stateFile('capacity-transition')], - { now: () => now, collect: async () => capacitySample } - )).rejects.toThrow('director-admin/threshold_max') - }) - - it('rejects stale live evidence', async () => { - const stale = sample() - stale.sources['active-probe']!.observedAt = new Date(now - 60_001).toISOString() - await expect(runIncidentLivePreflight( - ['--state-file', stateFile()], - { now: () => now, collect: async () => stale } - )).rejects.toThrow('active-probe/source_stale') - }) - - it('retries freshness-only failures when explicitly requested', async () => { - const stale = sample() - stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt = - new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString() - const missing = sample() - delete missing.sources['relay-logs'] - const collect = vi.fn() - .mockResolvedValueOnce(stale) - .mockResolvedValueOnce(missing) - .mockResolvedValueOnce(sample()) - const wait = vi.fn(async () => undefined) - await expect(runIncidentLivePreflight( - ['--state-file', stateFile(), '--retry-freshness'], - { now: () => now, collect, wait } - )).resolves.toBeUndefined() - expect(collect).toHaveBeenCalledTimes(3) - expect(wait).toHaveBeenCalledTimes(2) - expect(wait).toHaveBeenNthCalledWith(1, 15_000) - expect(wait).toHaveBeenNthCalledWith(2, 15_000) - }) - - it('retries a first-wave stale sample and passes on the fresh one', async () => { - const stale = sample() - stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt = - new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString() - const collect = vi.fn().mockResolvedValueOnce(stale).mockResolvedValueOnce(sample()) - const wait = vi.fn(async () => undefined) - await expect(runIncidentLivePreflight( - ['--state-file', stateFile(), '--wave-index', '0', '--retry-freshness'], - { now: () => now, collect, wait } - )).resolves.toBeUndefined() - expect(collect).toHaveBeenCalledTimes(2) - expect(wait).toHaveBeenCalledOnce() - }) - - it('stops retrying when the next wait would exceed the evidence-age bound', async () => { - const completedAt = now - 590_000 - const stale = sample() - stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString() - const collect = vi.fn(async () => stale) - const wait = vi.fn(async () => undefined) - await expect(runIncidentLivePreflight( - ['--state-file', stateFile('strict', { - startedAt: new Date(completedAt - 17 * 60_000).toISOString(), - windowStartedAt: new Date(completedAt - 16 * 60_000).toISOString(), - lastSampleAt: new Date(completedAt - 30_000).toISOString(), - completedAt: new Date(completedAt).toISOString() - }), '--retry-freshness'], - { now: () => now, collect, wait } - )).rejects.toThrow('cloud-monitoring/source_stale') - expect(collect).toHaveBeenCalledOnce() - expect(wait).not.toHaveBeenCalled() - }) - - it('does not retry a threshold failure', async () => { - const unhealthy = sample() - unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9 - unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt = - new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString() - const collect = vi.fn(async () => unhealthy) - const wait = vi.fn(async () => undefined) - await expect(runIncidentLivePreflight( - ['--state-file', stateFile(), '--retry-freshness'], - { now: () => now, collect, wait } - )).rejects.toThrow('cloud-monitoring/threshold_max') - expect(collect).toHaveBeenCalledOnce() - expect(wait).not.toHaveBeenCalled() - }) - - it('fails closed after the bounded freshness retry window', async () => { - const stale = sample() - stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString() - const collect = vi.fn(async () => stale) - const wait = vi.fn(async () => undefined) - await expect(runIncidentLivePreflight( - ['--state-file', stateFile(), '--retry-freshness'], - { now: () => now, collect, wait } - )).rejects.toThrow('cloud-monitoring/source_stale') - expect(collect).toHaveBeenCalledTimes(5) - expect(wait).toHaveBeenCalledTimes(4) - }) - - - // Why: a same-cap wave has no sealed 15-minute evidence, so this live recheck - // and the pre-drain sample after it are what stand between the dispatch and a - // mutation. It must judge the fleet exactly as it does with evidence, and it - // must never accept a half-specified selector. - describe('dispatch selector without monitor state', () => { - const overrideArgs = (extra: string[] = [], membership = canonicalMembership) => [ - '--no-monitor-state', - '--expected-selector-generation', '1', - '--selector-membership-file', membershipFile(membership), - ...extra - ] - - // The director's reading, plus whatever the override path decided to expect. - const liveCollect = ( - mutate: (next: IncidentSample) => IncidentSample = (next) => next - ) => async (expected: AdmissionSelector) => { - const next = canonicalSample(expected.generation) - next.expectedSelector = expected - return mutate(next) - } - - it('accepts one complete fresh green sample with no sealed evidence', async () => { - await expect(runIncidentLivePreflight( - overrideArgs(), - { now: () => now, collect: liveCollect() } - )).resolves.toBeUndefined() - }) - - // Why: the live selector is normalised and the comparison is an ordered - // stringify, so an operator's unsorted membership must canonicalise here or - // every override wave reads as selector drift on a healthy fleet. - it('canonicalises an unsorted operator membership', async () => { - const shuffled = { - existingOnly: [...canonicalCellIds].reverse(), - migrationOnly: [], - general: [] - } - expect(shuffled.existingOnly).not.toEqual(canonicalCellIds) - const seen: AdmissionSelector[] = [] - await expect(runIncidentLivePreflight( - overrideArgs([], shuffled), - { - now: () => now, - collect: async (expected) => { - seen.push(expected) - const next = canonicalSample(expected.generation) - next.expectedSelector = expected - return next - } - } - )).resolves.toBeUndefined() - expect(seen[0]!.membership.existingOnly).toEqual(canonicalCellIds) - }) - - // Why: normalising is also what enforces every configured cell exactly once, - // which a bare schema parse would have dropped. - it('rejects a membership that is not every configured cell exactly once', async () => { - const duplicated = { - existingOnly: [...canonicalCellIds, canonicalCellIds[0] as string], - migrationOnly: [], - general: [] - } - await expect(runIncidentLivePreflight( - overrideArgs([], duplicated), - { now: () => now, collect: liveCollect() } - )).rejects.toThrow('every configured cell exactly once') - const missing = { - existingOnly: canonicalCellIds.slice(1), - migrationOnly: [], - general: [] - } - await expect(runIncidentLivePreflight( - overrideArgs([], missing), - { now: () => now, collect: liveCollect() } - )).rejects.toThrow('every configured cell exactly once') - const unknown = { - existingOnly: [...canonicalCellIds.slice(1), 'production-gce-c999'], - migrationOnly: [], - general: [] - } - await expect(runIncidentLivePreflight( - overrideArgs([], unknown), - { now: () => now, collect: liveCollect() } - )).rejects.toThrow('every configured cell exactly once') - }) - - it('fails closed on a live threshold breach', async () => { - await expect(runIncidentLivePreflight( - overrideArgs(), - { - now: () => now, - collect: liveCollect((next) => { - next.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.99 - return next - }), - wait: async () => {} - } - )).rejects.toThrow('cloud-monitoring/threshold_max cloud_sql.cpu') - }) - - it('fails closed on a live selector mismatch', async () => { - await expect(runIncidentLivePreflight( - overrideArgs(), - { - now: () => now, - collect: liveCollect((next) => { - next.selector = { ...next.selector, generation: 7 } - return next - }), - wait: async () => {} - } - )).rejects.toThrow('selector_mismatch') - }) - - it('expects the wave-adjusted live selector generation', async () => { - const seen: AdmissionSelector[] = [] - await expect(runIncidentLivePreflight( - overrideArgs(['--wave-index', '2']), - { - now: () => now, - collect: async (expected) => { - seen.push(expected) - const next = canonicalSample(expected.generation) - next.expectedSelector = expected - return next - } - } - )).resolves.toBeUndefined() - expect(seen[0]!.generation).toBe(5) - }) - - it('offsets by the wave delta the cell class declares', async () => { - const generationFor = async (args: string[]) => { - const seen: AdmissionSelector[] = [] - await expect(runIncidentLivePreflight(args, { - now: () => now, - collect: async (expected) => { - seen.push(expected) - const next = canonicalSample(expected.generation) - next.expectedSelector = expected - return next - } - })).resolves.toBeUndefined() - return seen[0]!.generation - } - // A migration-only cell's wave isolates and restores nothing, so no predecessor moved it. - expect(await generationFor( - overrideArgs(['--wave-index', '2', '--selector-wave-delta', '0']) - )).toBe(1) - expect(await generationFor( - overrideArgs(['--wave-index', '2', '--selector-wave-delta', '2']) - )).toBe(5) - }) - - it('rejects a selector wave delta no cell class produces', async () => { - for (const delta of ['1', '3', '4', '', '-0', '02']) { - await expect(runIncidentLivePreflight( - overrideArgs(['--selector-wave-delta', delta]), - { now: () => now } - )).rejects.toThrow('usage:') - } - await expect(runIncidentLivePreflight( - overrideArgs(['--selector-wave-delta', '0', '--selector-wave-delta', '0']), - { now: () => now } - )).rejects.toThrow('usage:') - }) - - it('pins the strictest migration policy', async () => { - // An inactive migration target is tolerable only under recover-forward, - // and an override cannot elect that policy, so this must still fail. - await expect(runIncidentLivePreflight( - overrideArgs(), - { - now: () => now, - collect: liveCollect((next) => { - next.sources['director-admin']!.signals[ - 'cell.production-gce-c1.migration_target_inactive' - ]!.value = 30 - return next - }), - wait: async () => {} - } - )).rejects.toThrow('director-admin/threshold_max') - }) - - it('rejects a half-specified override', async () => { - const cases: string[][] = [ - ['--no-monitor-state'], - ['--no-monitor-state', '--expected-selector-generation', '1'], - ['--no-monitor-state', '--selector-membership-file', membershipFile()], - // Mixing the two sources would let a caller pass sealed evidence it - // never wants read. - [ - '--no-monitor-state', - '--expected-selector-generation', '1', - '--selector-membership-file', membershipFile(), - '--state-file', stateFile() - ], - // Override arguments without the flag must not be silently ignored. - ['--state-file', stateFile(), '--expected-selector-generation', '1'], - ['--no-monitor-state', '--no-monitor-state'], - ['--expected-selector-generation', '1'] - ] - for (const args of cases) { - await expect(runIncidentLivePreflight( - args, - { now: () => now, collect: liveCollect() } - )).rejects.toThrow('usage:') - } - }) - - it('rejects an unknown option and a negative generation', async () => { - await expect(runIncidentLivePreflight( - overrideArgs(['--skip-everything']), - { now: () => now, collect: liveCollect() } - )).rejects.toThrow('usage:') - await expect(runIncidentLivePreflight( - [ - '--no-monitor-state', - '--expected-selector-generation', '-1', - '--selector-membership-file', membershipFile() - ], - { now: () => now, collect: liveCollect() } - )).rejects.toThrow() - }) - - it('re-samples a tolerable failure and then passes', async () => { - let samples = 0 - await expect(runIncidentLivePreflight( - overrideArgs(), - { - now: () => now, - collect: liveCollect((next) => { - samples++ - if (samples === 1) { - next.sources['cloud-monitoring']!.signals['director.instances']!.value = 2 - } - return next - }), - wait: async () => {} - } - )).resolves.toBeUndefined() - expect(samples).toBe(2) - }) - }) - - it('uses the supplied admin token without minting through gcloud', async () => { - const identityToken = vi.fn(async () => 'minted.token.value') - const gcloud = livePreflightGcloud( - { accessToken: async () => 'access-token', identityToken }, - { ORCA_RELAY_ADMIN_ID_TOKEN: 'supplied.token.value' } - ) - await expect(gcloud.identityToken!('audience')).resolves.toBe( - 'supplied.token.value' - ) - expect(identityToken).not.toHaveBeenCalled() - }) -}) diff --git a/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts b/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts deleted file mode 100644 index 4160d93ef06..00000000000 --- a/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts +++ /dev/null @@ -1,360 +0,0 @@ -import { readFile } from 'node:fs/promises' -import { resolve } from 'node:path' -import { pathToFileURL } from 'node:url' -import { z } from 'zod' -import { relayOpsEnvironment } from './environment-config.js' -import { createGcloudClient } from './gcloud-client.js' -import { suppliedIdentityToken } from './incident-monitor-cli.js' -import { - AdmissionSelectorSchema, - normalizeSelectorMembership, - SelectorMembershipSchema, - type AdmissionSelector -} from './incident-selector.js' -import { - describeIncidentFailure, - evaluateIncidentSample, - FRESHNESS_FAILURE_CODES, - INCIDENT_MONITOR_THRESHOLDS, - preDrainDryRunPassed, - toleratedStreakKey, - type IncidentSample -} from './incident-monitor.js' -import { createIncidentSampleCollector } from './incident-monitor-sources.js' - -const FRESHNESS_RETRY_ATTEMPTS = 5 -const FRESHNESS_RETRY_INTERVAL_MS = 15_000 -// 10 min, not 5: a wave job reaches this check ~5 min after the monitor completes -// (runner queue ~2 min, gate job ~80 s, checkout ~60 s); on 2026-09-17 a green -// gate died at 302 s. The live samples below hold every wave to now. -const MONITOR_EVIDENCE_MAX_AGE_MS = 10 * 60_000 -// Matches the wave job timeout-minutes; bounds each predecessor wave. -const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000 -// Widest any wave chain declares (same-cap's cell_1..cell_10); each job workflow -// pins its own narrower range. -const WAVE_INDEX_PATTERN = /^[0-9]$/ -// 2 for a general cell's isolate-and-restore wave, 0 for a migration-only cell's no-op pair. -const SELECTOR_WAVE_DELTA_PATTERN = /^[02]$/ - -export function livePreflightGcloud( - gcloud: ReturnType, - environment: NodeJS.ProcessEnv = process.env -): ReturnType { - const token = suppliedIdentityToken(environment.ORCA_RELAY_ADMIN_ID_TOKEN) - return token ? { ...gcloud, identityToken: async () => token } : gcloud -} - -const PreflightStateSchema = z.object({ - schemaVersion: z.literal(4), - environment: z.literal('production'), - expectedSelector: AdmissionSelectorSchema, - migrationPolicy: z.enum(['strict', 'recover-forward', 'capacity-transition']), - recoverySourceCellId: z.string().nullable(), - capacityCellId: z.string().nullable(), - preDrainDryRun: z.literal(true), - startedAt: z.string(), - windowStartedAt: z.string(), - durationMinutes: z.literal(15), - intervalMs: z.literal(60_000), - sampleCount: z.number().int().min(16), - lastSampleAt: z.string(), - frozenAt: z.null(), - completedAt: z.string() -}).superRefine((state, context) => { - const validRecovery = - state.migrationPolicy === 'recover-forward' && - state.capacityCellId === null && - state.recoverySourceCellId !== null && - state.expectedSelector.membership.existingOnly.includes( - state.recoverySourceCellId - ) - const validStrict = - state.migrationPolicy === 'strict' && - state.recoverySourceCellId === null && - state.capacityCellId === null - const validCapacity = - state.migrationPolicy === 'capacity-transition' && - state.recoverySourceCellId === null && - state.capacityCellId !== null && - state.expectedSelector.membership.general.includes(state.capacityCellId) - if (!validRecovery && !validStrict && !validCapacity) { - context.addIssue({ - code: 'custom', - message: 'relay live preflight migration policy is invalid' - }) - } -}) - -const PREFLIGHT_USAGE = - 'usage: --state-file [--wave-index <0-3>]' + - ' [--selector-wave-delta <0|2>] [--retry-freshness]' + - ' | --no-monitor-state --expected-selector-generation ' + - ' --selector-membership-file [--wave-index <0-3>]' + - ' [--selector-wave-delta <0|2>]' - -const VALUE_OPTIONS = new Set([ - '--state-file', - '--wave-index', - '--selector-wave-delta', - '--expected-selector-generation', - '--selector-membership-file' -]) -const FLAG_OPTIONS = new Set(['--retry-freshness', '--no-monitor-state']) - -// Rejects an unknown option and a repeated one, so a typo can never silently -// widen what this check accepts. -export function parsePreflightArgs(argv: string[]): { - options: Map - flags: Set -} { - const args = argv[0] === '--' ? argv.slice(1) : argv - const options = new Map() - const flags = new Set() - for (let index = 0; index < args.length; index += 1) { - const arg = args[index] as string - if (FLAG_OPTIONS.has(arg)) { - if (flags.has(arg)) throw new Error(PREFLIGHT_USAGE) - flags.add(arg) - continue - } - const value = args[index + 1] - if (!VALUE_OPTIONS.has(arg) || options.has(arg) || !value) { - throw new Error(PREFLIGHT_USAGE) - } - options.set(arg, value) - index += 1 - } - return { options, flags } -} - -// What the live recheck measures the fleet against. Either source supplies the -// exact same fields; only where they come from and how they can go stale differs. -type PreflightPlan = { - environment: 'production' - expectedSelector: AdmissionSelector - migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition' - recoverySourceCellId: string | null - capacityCellId: string | null - // The instant the live samples age from. Monitor evidence ages from the moment - // the 15-minute window closed; a dispatch plan has no evidence to age, so its - // retry budget starts when this process does. - evidenceAnchorMs: number -} - -async function monitorEvidencePreflightPlan( - options: Map, - nowMs: number, - waveIndex: string -): Promise { - const stateFile = options.get('--state-file') - if ( - !stateFile || - options.has('--expected-selector-generation') || - options.has('--selector-membership-file') - ) throw new Error(PREFLIGHT_USAGE) - const state = PreflightStateSchema.parse( - JSON.parse(await readFile(resolve(stateFile), 'utf8')) - ) - const completedAt = Date.parse(state.completedAt) - const windowStartedAt = Date.parse(state.windowStartedAt) - const lastSampleAt = Date.parse(state.lastSampleAt) - const evidenceAgeMs = nowMs - completedAt - // Later waves start after sequential predecessor cell rolls, so the - // freshness bound grows by one cell-job timeout per predecessor; the live - // samples collected below still hold every wave to current health. - const maxEvidenceAgeMs = - MONITOR_EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS - if ( - !preDrainDryRunPassed(state) || - !Number.isFinite(windowStartedAt) || - completedAt - windowStartedAt < 15 * 60_000 || - !Number.isFinite(lastSampleAt) || - lastSampleAt > completedAt || - completedAt - lastSampleAt > state.intervalMs || - !Number.isFinite(completedAt) || - evidenceAgeMs < 0 || - evidenceAgeMs > maxEvidenceAgeMs - ) { - throw new Error('relay live preflight monitor evidence is incomplete or stale') - } - return { - environment: state.environment, - expectedSelector: state.expectedSelector, - migrationPolicy: state.migrationPolicy, - recoverySourceCellId: state.recoverySourceCellId, - capacityCellId: state.capacityCellId, - evidenceAnchorMs: completedAt - } -} - -// The expected selector exactly as a same-cap dispatch states it, canonicalised as the monitor CLI -// does when it seals evidence. The live selector read from the director is normalised too and the -// comparison is an ordered stringify, so unsorted operator input would read as selector drift on a -// healthy fleet; normalising is also what enforces every configured cell exactly once. -export async function readDispatchSelector( - generation: string, - membershipFile: string -): Promise { - const membership = normalizeSelectorMembership( - SelectorMembershipSchema.parse( - JSON.parse(await readFile(resolve(membershipFile), 'utf8')) - ), - new Set(relayOpsEnvironment('production').cells.map((cell) => cell.cellId)) - ) - return AdmissionSelectorSchema.parse({ generation: Number(generation), membership }) -} - -// Each predecessor same-cap apply wave reversibly isolates and restores its cell with membership -// unchanged (rollback is single-cell, so it never reaches a later wave), so the live selector -// comparison must expect the wave-adjusted generation. A general cell advances it by 2; a -// migration-only cell is already isolated and stays that way, so its wave advances it by 0. A wave -// is never mixed, so one delta covers every predecessor. -export function waveAdjustedSelector( - selector: AdmissionSelector, - waveIndex: string, - selectorWaveDelta: string -): AdmissionSelector { - if (!WAVE_INDEX_PATTERN.test(waveIndex) || !SELECTOR_WAVE_DELTA_PATTERN.test(selectorWaveDelta)) { - throw new Error('relay wave index or selector wave delta is invalid') - } - return { - ...selector, - generation: selector.generation + Number(selectorWaveDelta) * Number(waveIndex) - } -} - -// No monitor evidence: the expected selector comes straight from the dispatch inputs, which the -// rehome inspect after this verifies against the live director. The policy is pinned to strict -- -// the only one the same-cap rollout ever verifies -- and every threshold is unchanged. -async function dispatchPreflightPlan( - options: Map, - nowMs: number -): Promise { - const generation = options.get('--expected-selector-generation') - const membershipFile = options.get('--selector-membership-file') - if (!generation || !membershipFile || options.has('--state-file')) { - throw new Error(PREFLIGHT_USAGE) - } - return { - environment: 'production', - expectedSelector: await readDispatchSelector(generation, membershipFile), - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - evidenceAnchorMs: nowMs - } -} - -export async function runIncidentLivePreflight( - argv: string[], - dependencies: { - now?: () => number - wait?: (ms: number) => Promise - collect?: (expectedSelector: AdmissionSelector) => Promise - gcloud?: ReturnType - environment?: NodeJS.ProcessEnv - } = {} -): Promise { - const parsed = parsePreflightArgs(argv) - const waveIndex = parsed.options.get('--wave-index') ?? '0' - if (!WAVE_INDEX_PATTERN.test(waveIndex)) throw new Error(PREFLIGHT_USAGE) - const selectorWaveDelta = parsed.options.get('--selector-wave-delta') ?? '2' - if (!SELECTOR_WAVE_DELTA_PATTERN.test(selectorWaveDelta)) throw new Error(PREFLIGHT_USAGE) - const now = dependencies.now ?? Date.now - const plan = parsed.flags.has('--no-monitor-state') - ? await dispatchPreflightPlan(parsed.options, now()) - : await monitorEvidencePreflightPlan(parsed.options, now(), waveIndex) - const maxEvidenceAgeMs = - MONITOR_EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS - const gcloud = livePreflightGcloud( - dependencies.gcloud ?? createGcloudClient(), - dependencies.environment - ) - const collectOptions = { - environment: plan.environment, - expectedSelector: waveAdjustedSelector(plan.expectedSelector, waveIndex, selectorWaveDelta), - ...(dependencies.now ? { now: dependencies.now } : {}) - } - const injected = dependencies.collect - const collect = injected - ? () => injected(collectOptions.expectedSelector) - : createIncidentSampleCollector(gcloud, collectOptions) - const wait = dependencies.wait ?? ((ms: number) => new Promise((resolveWait) => { - setTimeout(resolveWait, ms) - })) - const freshnessAttempts = parsed.flags.has('--retry-freshness') ? FRESHNESS_RETRY_ATTEMPTS : 1 - // Why: this single sample decides a mutating wave, so an Asia cell's ~30 s - // "no healthy upstream" window, or a one-minute director instance-replacement - // dip, could fail a wave here even after the 15-minute gate learned to ride it - // out. Hold the two to the same tolerance. Unlike the freshness retry this - // needs no flag, because a tolerated breach is never the operator's call to - // waive. - const cellProbeAttempts = 1 + INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - const attempts = Math.max(freshnessAttempts, cellProbeAttempts) - let freshnessRetries = freshnessAttempts - 1 - let cellProbeRetries = cellProbeAttempts - 1 - // Waiting must never carry the mutation past the same evidence-age bound the - // entry check enforces, so the wave budget also caps the retry window. - const budgetExhausted = (): boolean => - now() + FRESHNESS_RETRY_INTERVAL_MS - plan.evidenceAnchorMs > maxEvidenceAgeMs - for (let attempt = 1; attempt <= attempts; attempt++) { - // A director admin read can fail on its own (its handler maps a Cloud SQL - // pool timeout onto 404), which says nothing about relay health; spend an - // attempt on it rather than failing the wave on one unlucky sample. - let sample: IncidentSample - try { - sample = await collect() - } catch (error) { - const message = error instanceof Error ? error.message : 'sample collection failed' - if (attempt === attempts || budgetExhausted()) { - throw new Error(`relay live preflight failed: collector: ${message}`) - } - console.warn( - `relay live preflight re-sampling after collector failure (${attempt}/${attempts - 1})` - ) - await wait(FRESHNESS_RETRY_INTERVAL_MS) - continue - } - const evaluation = evaluateIncidentSample( - sample, - now(), - plan.migrationPolicy, - plan.recoverySourceCellId, - plan.capacityCellId - ) - if (evaluation.status === 'green') return - const freshnessFailures = evaluation.failures.filter((failure) => - FRESHNESS_FAILURE_CODES.has(failure.code) - ) - // Tolerated readings only (per-cell probes and the director instance count). - // The director and auth health probes are absent here on purpose and fail - // the wave on their first bad sample. - const cellProbeFailures = evaluation.failures.filter((failure) => - !FRESHNESS_FAILURE_CODES.has(failure.code) && toleratedStreakKey(failure) !== null - ) - const retryable = - freshnessFailures.length + cellProbeFailures.length === evaluation.failures.length && - (freshnessFailures.length === 0 || freshnessRetries > 0) && - (cellProbeFailures.length === 0 || cellProbeRetries > 0) - if (!retryable || attempt === attempts || budgetExhausted()) { - throw new Error( - `relay live preflight failed: ${evaluation.failures - .map(describeIncidentFailure) - .join(',')}` - ) - } - if (freshnessFailures.length > 0) freshnessRetries-- - if (cellProbeFailures.length > 0) cellProbeRetries-- - console.warn( - `relay live preflight re-sampling after tolerable failure (${attempt}/${attempts - 1})` - ) - await wait(FRESHNESS_RETRY_INTERVAL_MS) - } -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - runIncidentLivePreflight(process.argv.slice(2)).catch((error: unknown) => { - console.error(error instanceof Error ? error.message : 'relay live preflight failed') - process.exitCode = 1 - }) -} diff --git a/cloud/apps/relay-ops/src/incident-monitor-cli.test.ts b/cloud/apps/relay-ops/src/incident-monitor-cli.test.ts deleted file mode 100644 index 31e5a131d35..00000000000 --- a/cloud/apps/relay-ops/src/incident-monitor-cli.test.ts +++ /dev/null @@ -1,455 +0,0 @@ -import { mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' -import { - parseIncidentMonitorArguments, - runIncidentMonitorCli -} from './incident-monitor-cli.js' -import type { IncidentSample } from './incident-monitor.js' -import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js' - -const directories: string[] = [] -const startedAt = Date.parse('2026-07-28T00:00:00.000Z') -const productionCells = RELAY_OPS_ENVIRONMENTS.production.cells.map((cell) => cell.cellId) -const selector = { - generation: 1, - membership: { - existingOnly: productionCells.slice(1), - migrationOnly: [], - general: [productionCells[0]!] - } -} -const selectorArguments = [ - '--expected-selector-generation', - '1', - '--expected-existing-only-cells', - productionCells.slice(1).join(','), - '--expected-migration-only-cells', - 'none', - '--expected-general-cells', - productionCells[0]! -] -const signal = (value: number, at: number) => ({ - value, - observedAt: new Date(at).toISOString() -}) - -function sample(at: number): IncidentSample { - const observedAt = new Date(at).toISOString() - const cellId = 'production-gce-c1' - return { - collectedAt: observedAt, - selector, - expectedSelector: selector, - cells: [{ - cellId, - region: 'us-central1', - runtimeKnown: true, - powered: true, - expectedAdmissionState: 'general' - }], - sources: { - 'active-probe': { - observedAt, - signals: { - 'director.health': signal(1, at), - 'director.ready': signal(1, at), - 'director.latency_ms': signal(1, at), - 'auth.health': signal(1, at), - 'auth.ready': signal(1, at), - 'auth.latency_ms': signal(1, at), - [`cell.${cellId}.health`]: signal(1, at), - [`cell.${cellId}.ready`]: signal(1, at), - [`cell.${cellId}.latency_ms`]: signal(1, at) - } - }, - 'cloud-monitoring': { - observedAt, - signals: { - 'cloud_sql.cpu': signal(0.1, at), - 'cloud_sql.memory': signal(0.1, at), - 'cloud_sql.backends': signal(1, at), - 'cloud_sql.lock_waits': signal(0, at), - 'cloud_sql.deadlocks': signal(0, at), - 'director.instances': signal(5, at), - 'director.cpu': signal(0.1, at), - 'director.memory': signal(0.1, at), - 'director.concurrency': signal(1, at), - 'director.errors': signal(0, at), - 'auth.errors': signal(0, at) - } - }, - 'relay-logs': { - observedAt, - signals: { - 'relay.pool_waiting': signal(0, at), - 'relay.pool_wait_ms': signal(0, at), - 'relay.postgres_retries': signal(0, at), - 'relay.postgres_retry_exhausted': signal(0, at), - [`cell.${cellId}.connections`]: signal(1, at), - [`cell.${cellId}.queued_bytes`]: signal(0, at) - } - }, - 'director-admin': { - observedAt, - signals: { - [`cell.${cellId}.admission_state`]: signal(2, at), - [`cell.${cellId}.heartbeat_fresh`]: signal(1, at), - [`cell.${cellId}.heartbeat_age_ms`]: signal(1, at), - [`cell.${cellId}.migration_blocked`]: signal(0, at), - [`cell.${cellId}.migration_target_inactive`]: signal(0, at) - } - } - } - } -} - -afterEach(() => { - for (const directory of directories.splice(0)) { - rmSync(directory, { recursive: true, force: true }) - } -}) - -describe('incident monitor CLI', () => { - it('requires an exact selector and rejects invalid membership', () => { - expect(() => parseIncidentMonitorArguments([])).toThrow( - '--expected-selector-generation' - ) - expect(() => - parseIncidentMonitorArguments([ - '--incident-id', - 'incident-1', - '--expected-selector-generation', - '1', - '--expected-existing-only-cells', - productionCells.slice(1).join(','), - '--expected-migration-only-cells', - 'none', - '--expected-general-cells', - 'production-gce-c99' - ]) - ).toThrow('every configured cell exactly once') - expect(() => - parseIncidentMonitorArguments([ - '--incident-id', - 'incident-1', - ...selectorArguments, - '--interval-seconds', - '61' - ]) - ).toThrow('between 1 and 60') - expect(() => - parseIncidentMonitorArguments([ - '--incident-id', - 'incident-1', - ...selectorArguments, - '--duration-minutes', - '14' - ]) - ).toThrow('between 15 and 90') - expect(() => - parseIncidentMonitorArguments([ - '--incident-id', - 'incident-1', - '--expected-selector-generation', - '0', - '--expected-existing-only-cells', - productionCells.slice(1).join(','), - '--expected-migration-only-cells', - productionCells[0]!, - '--expected-general-cells', - 'none' - ]) - ).toThrow('generation 0 cannot represent migration-only') - expect(() => - parseIncidentMonitorArguments([ - '--incident-id', - 'incident-1', - ...selectorArguments, - '--migration-policy', - 'recover-forward', - '--pre-drain-dry-run' - ]) - ).toThrow('requires an existing-only --recovery-source-cell-id') - expect(() => - parseIncidentMonitorArguments([ - '--incident-id', - 'incident-1', - ...selectorArguments, - '--migration-policy', - 'capacity-transition', - '--pre-drain-dry-run' - ]) - ).toThrow('requires a general --capacity-cell-id') - expect( - parseIncidentMonitorArguments([ - '--incident-id', - 'incident-1', - ...selectorArguments, - '--migration-policy', - 'capacity-transition', - '--capacity-cell-id', - productionCells[0]!, - '--pre-drain-dry-run' - ]) - ).toMatchObject({ - migrationPolicy: 'capacity-transition', - capacityCellId: productionCells[0]!, - recoverySourceCellId: null - }) - }) - - it('writes private durable checkpoints for a green pre-drain dry run', async () => { - const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-')) - directories.push(directory) - let now = startedAt - const output: string[] = [] - const code = await runIncidentMonitorCli( - [ - '--incident-id', - 'incident-1', - ...selectorArguments, - '--pre-drain-dry-run', - '--output-directory', - directory - ], - { - cwd: directory, - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => sample(now), - writeOutput: (value) => output.push(value) - } - ) - expect(code).toBe(0) - const statePath = join(directory, 'incident-1.state.json') - const summaryPath = join(directory, 'incident-1.summaries.jsonl') - const markdownPath = join(directory, 'incident-1.summary.md') - expect(statSync(statePath).mode & 0o077).toBe(0) - expect(statSync(summaryPath).mode & 0o077).toBe(0) - expect(statSync(markdownPath).mode & 0o077).toBe(0) - const summaries = readFileSync(summaryPath, 'utf8') - .trim() - .split('\n') - .map((line) => JSON.parse(line)) - expect(summaries.map((entry) => entry.checkpointMinute)).toEqual([0, 5, 15]) - expect(output.join('')).not.toContain('token') - expect(readFileSync(markdownPath, 'utf8')).toContain( - '| 0 | 15 | green | 16 | none |' - ) - expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({ - completedAt: new Date(startedAt + 15 * 60_000).toISOString(), - frozenAt: null, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - sampleCount: 16 - }) - }) - - it('runs a recovery dry run without masking blocked migrations', async () => { - const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-')) - directories.push(directory) - let now = startedAt - const recoverySelector = { - generation: 1, - membership: { - existingOnly: ['production-gce-c1'], - migrationOnly: [], - general: productionCells.slice(1) - } - } - const recoverySample = (): IncidentSample => { - const current = sample(now) - current.selector = recoverySelector - current.expectedSelector = recoverySelector - current.cells[0]!.expectedAdmissionState = 'existing-only' - current.sources['director-admin']!.signals[ - 'cell.production-gce-c1.admission_state' - ] = signal(0, now) - current.sources['director-admin']!.signals[ - 'cell.production-gce-c1.migration_target_inactive' - ] = signal(30, now) - return current - } - const args = [ - '--incident-id', - 'incident-1', - '--expected-selector-generation', - '1', - '--expected-existing-only-cells', - 'production-gce-c1', - '--expected-migration-only-cells', - 'none', - '--expected-general-cells', - productionCells.slice(1).join(','), - '--pre-drain-dry-run', - '--migration-policy', - 'recover-forward', - '--recovery-source-cell-id', - 'production-gce-c1', - '--output-directory', - directory - ] - const dependencies = { - cwd: directory, - now: () => now, - wait: async (ms: number) => { - now += ms - }, - collect: async () => recoverySample(), - writeOutput: () => {} - } - await expect(runIncidentMonitorCli(args, dependencies)).resolves.toBe(0) - expect( - JSON.parse(readFileSync(join(directory, 'incident-1.state.json'), 'utf8')) - ).toMatchObject({ - migrationPolicy: 'recover-forward', - recoverySourceCellId: 'production-gce-c1', - capacityCellId: null, - frozenAt: null - }) - }) - - it('fails a frozen pre-drain gate after its first sample', async () => { - const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-')) - directories.push(directory) - let now = startedAt - let collections = 0 - let waits = 0 - const code = await runIncidentMonitorCli( - [ - '--incident-id', - 'incident-1', - ...selectorArguments, - '--pre-drain-dry-run', - '--output-directory', - directory - ], - { - cwd: directory, - now: () => now, - wait: async (ms) => { - waits++ - now += ms - }, - collect: async () => { - collections++ - const unhealthy = sample(now) - unhealthy.sources['relay-logs']!.signals['relay.pool_waiting'] = - signal(801, now) - return unhealthy - }, - writeOutput: () => {} - } - ) - expect(code).toBe(2) - expect(collections).toBe(1) - expect(waits).toBe(0) - expect( - JSON.parse(readFileSync(join(directory, 'incident-1.state.json'), 'utf8')) - ).toMatchObject({ - completedAt: new Date(startedAt).toISOString(), - frozenAt: new Date(startedAt).toISOString(), - sampleCount: 1 - }) - }) - - it('requires --restart and preserves a latched freeze', async () => { - const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-')) - directories.push(directory) - let now = startedAt - const args = [ - '--incident-id', - 'incident-1', - ...selectorArguments, - '--duration-minutes', - '15', - '--output-directory', - directory - ] - await runIncidentMonitorCli(args, { - cwd: directory, - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - const unhealthy = sample(now) - unhealthy.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(801, now) - return unhealthy - }, - writeOutput: () => {} - }) - await expect( - runIncidentMonitorCli(args, { - cwd: directory, - now: () => now, - collect: async () => sample(now), - writeOutput: () => {} - }) - ).rejects.toThrow('pass --restart') - const changedAdmission = [...args] - changedAdmission[changedAdmission.indexOf('--expected-selector-generation') + 1] = '2' - await expect( - runIncidentMonitorCli([...changedAdmission, '--restart'], { - cwd: directory, - now: () => now, - collect: async () => sample(now), - writeOutput: () => {} - }) - ).rejects.toThrow('do not match') - await expect( - runIncidentMonitorCli([...args, '--restart'], { - cwd: directory, - now: () => now, - collect: async () => sample(now), - writeOutput: () => {} - }) - ).resolves.toBe(2) - }) - - it('resumes a gracefully segmented monitor without resetting continuity', async () => { - const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-')) - directories.push(directory) - let now = startedAt - const args = [ - '--incident-id', - 'incident-1', - ...selectorArguments, - '--duration-minutes', - '15', - '--output-directory', - directory - ] - const dependencies = { - cwd: directory, - now: () => now, - wait: async (ms: number) => { - now += ms - }, - collect: async () => sample(now), - writeOutput: () => {} - } - await expect( - runIncidentMonitorCli([...args, '--max-samples-this-run', '2'], dependencies) - ).resolves.toBe(0) - const statePath = join(directory, 'incident-1.state.json') - expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({ - completedAt: null, - sampleCount: 2, - lastSampleAt: new Date(startedAt + 60_000).toISOString() - }) - await expect( - runIncidentMonitorCli([...args, '--restart'], dependencies) - ).resolves.toBe(0) - expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({ - completedAt: new Date(startedAt + 15 * 60_000).toISOString(), - windowSequence: 0, - sampleCount: 17 - }) - }) -}) diff --git a/cloud/apps/relay-ops/src/incident-monitor-cli.ts b/cloud/apps/relay-ops/src/incident-monitor-cli.ts deleted file mode 100644 index 0f24459c9fd..00000000000 --- a/cloud/apps/relay-ops/src/incident-monitor-cli.ts +++ /dev/null @@ -1,510 +0,0 @@ -import { randomUUID } from 'node:crypto' -import { - appendFile, - chmod, - mkdir, - open, - readFile, - rename, - stat, - writeFile -} from 'node:fs/promises' -import { dirname, resolve } from 'node:path' -import { pathToFileURL } from 'node:url' -import { z } from 'zod' -import { relayOpsEnvironment } from './environment-config.js' -import { createGcloudClient } from './gcloud-client.js' -import { - AdmissionSelectorSchema, - normalizeSelectorMembership, - type AdmissionSelector -} from './incident-selector.js' -import { - initialIncidentMonitorState, - preDrainDryRunPassed, - runIncidentMonitor, - type IncidentCheckpoint, - type IncidentSample, - type IncidentMonitorState -} from './incident-monitor.js' -import { createIncidentSampleCollector } from './incident-monitor-sources.js' - -const StateSchema = z.object({ - schemaVersion: z.literal(4), - incidentId: z.string(), - environment: z.enum(['production', 'staging']), - expectedSelector: AdmissionSelectorSchema, - preDrainDryRun: z.boolean(), - migrationPolicy: z.enum(['strict', 'recover-forward', 'capacity-transition']), - recoverySourceCellId: z.string().nullable(), - capacityCellId: z.string().nullable(), - startedAt: z.string(), - windowStartedAt: z.string().nullable(), - windowSequence: z.number().int().nonnegative(), - durationMinutes: z.number().int(), - intervalMs: z.number().int(), - nextCheckpointIndex: z.number().int().nonnegative(), - sampleCount: z.number().int().nonnegative(), - totalSampleCount: z.number().int().nonnegative(), - lastSampleAt: z.string().nullable(), - continuityEvents: z.array(z.object({ - recordedAt: z.string(), - windowSequence: z.number().int().nonnegative(), - // Pre-2026-09-05 state files predate tolerated freshness gaps. - tolerated: z.boolean().default(false), - failures: z.array(z.object({ - code: z.string(), - source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']), - signal: z.string().optional(), - observed: z.number().optional(), - threshold: z.number().optional() - })) - })), - frozenAt: z.string().nullable(), - failures: z.array(z.object({ - code: z.string(), - source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']), - signal: z.string().optional(), - observed: z.number().optional(), - threshold: z.number().optional() - })), - // Pre-2026-09-17 state files predate cell-probe tolerance; a resumed run that - // carries no streak is one that also restarts its window, so it earns nothing. - probeStreaks: z.record(z.string(), z.number().int().nonnegative()).default({}), - toleratedProbeEvents: z.array(z.object({ - recordedAt: z.string(), - windowSequence: z.number().int().nonnegative(), - failures: z.array(z.object({ - code: z.string(), - source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']), - signal: z.string().optional(), - observed: z.number().optional(), - threshold: z.number().optional() - })) - })).default([]), - completedAt: z.string().nullable() -}) - -type CliOptions = { - environment: 'production' | 'staging' - incidentId: string - durationMinutes: number - intervalMs: number - expectedSelector: AdmissionSelector - stateFile: string - summaryFile: string - markdownFile: string - restart: boolean - preDrainDryRun: boolean - migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition' - recoverySourceCellId: string | null - capacityCellId: string | null - maxSamplesThisRun: number | null -} - -function parsePositiveInteger(value: string | undefined, name: string): number { - const parsed = Number(value) - if (!Number.isSafeInteger(parsed) || parsed <= 0) { - throw new Error(`${name} must be a positive integer`) - } - return parsed -} - -export function parseIncidentMonitorArguments(argv: string[], cwd = process.cwd()): CliOptions { - const flags = new Set(['restart', 'pre-drain-dry-run']) - const valueArguments = new Set([ - 'duration-minutes', - 'environment', - 'expected-selector-generation', - 'expected-existing-only-cells', - 'expected-migration-only-cells', - 'expected-general-cells', - 'incident-id', - 'interval-seconds', - 'max-samples-this-run', - 'migration-policy', - 'output-directory', - 'recovery-source-cell-id', - 'capacity-cell-id' - ]) - const values: Record = {} - const enabledFlags = new Set() - for (let index = 0; index < argv.length; index++) { - const argument = argv[index] - if (!argument?.startsWith('--')) throw new Error(`invalid argument ${argument ?? ''}`) - const name = argument.slice(2) - if (flags.has(name)) { - enabledFlags.add(name) - continue - } - if (!valueArguments.has(name)) throw new Error(`unknown argument --${name}`) - const value = argv[++index] - if (!value || value.startsWith('--')) throw new Error(`missing --${name} value`) - values[name] = value - } - const environment = z.enum(['production', 'staging']).parse( - values.environment ?? 'production' - ) - const incidentId = values['incident-id'] ?? randomUUID() - if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{7,127}$/.test(incidentId)) { - throw new Error('--incident-id must be 8-128 safe characters') - } - const selectorGeneration = Number(values['expected-selector-generation']) - if (!Number.isSafeInteger(selectorGeneration) || selectorGeneration < 0) { - throw new Error('--expected-selector-generation must be a nonnegative integer') - } - const configuredCells = new Set( - relayOpsEnvironment(environment).cells.map((cell) => cell.cellId) - ) - const cellList = (name: string): string[] => { - const value = values[name] - if (value === undefined) throw new Error(`--${name} is required; use none for an empty set`) - return value === 'none' ? [] : value.split(',').map((cellId) => cellId.trim()) - } - const expectedSelector = { - generation: selectorGeneration, - membership: normalizeSelectorMembership( - { - existingOnly: cellList('expected-existing-only-cells'), - migrationOnly: cellList('expected-migration-only-cells'), - general: cellList('expected-general-cells') - }, - configuredCells - ) - } - if (selectorGeneration === 0 && expectedSelector.membership.migrationOnly.length > 0) { - throw new Error('generation 0 cannot represent migration-only admission') - } - const preDrainDryRun = enabledFlags.has('pre-drain-dry-run') - const migrationPolicy = z.enum([ - 'strict', - 'recover-forward', - 'capacity-transition' - ]).parse( - values['migration-policy'] ?? 'strict' - ) - const recoverySourceCellId = - values['recovery-source-cell-id'] === undefined || - values['recovery-source-cell-id'] === 'none' - ? null - : values['recovery-source-cell-id'] - const capacityCellId = - values['capacity-cell-id'] === undefined || values['capacity-cell-id'] === 'none' - ? null - : values['capacity-cell-id'] - const durationMinutes = parsePositiveInteger( - values['duration-minutes'] ?? (preDrainDryRun ? '15' : '90'), - '--duration-minutes' - ) - const intervalMs = - parsePositiveInteger(values['interval-seconds'] ?? '60', '--interval-seconds') * 1_000 - if (durationMinutes < 15 || durationMinutes > 90) { - throw new Error('--duration-minutes must be between 15 and 90') - } - if (intervalMs > 60_000) { - throw new Error('--interval-seconds must be between 1 and 60') - } - if (preDrainDryRun && durationMinutes !== 15) { - throw new Error('--pre-drain-dry-run requires --duration-minutes 15') - } - if (migrationPolicy !== 'strict' && !preDrainDryRun) { - throw new Error(`--migration-policy ${migrationPolicy} requires --pre-drain-dry-run`) - } - if ( - migrationPolicy === 'recover-forward' && - ( - recoverySourceCellId === null || - !configuredCells.has(recoverySourceCellId) || - !expectedSelector.membership.existingOnly.includes(recoverySourceCellId) - ) - ) { - throw new Error( - '--migration-policy recover-forward requires an existing-only --recovery-source-cell-id' - ) - } - if (migrationPolicy !== 'recover-forward' && recoverySourceCellId !== null) { - throw new Error('--recovery-source-cell-id requires --migration-policy recover-forward') - } - if ( - migrationPolicy === 'capacity-transition' && - ( - capacityCellId === null || - !configuredCells.has(capacityCellId) || - !expectedSelector.membership.general.includes(capacityCellId) - ) - ) { - throw new Error( - '--migration-policy capacity-transition requires a general --capacity-cell-id' - ) - } - if (migrationPolicy !== 'capacity-transition' && capacityCellId !== null) { - throw new Error('--capacity-cell-id requires --migration-policy capacity-transition') - } - const directory = resolve(cwd, values['output-directory'] ?? '.relay-incidents') - const maxSamplesThisRun = values['max-samples-this-run'] - ? parsePositiveInteger(values['max-samples-this-run'], '--max-samples-this-run') - : null - return { - environment, - incidentId, - durationMinutes, - intervalMs, - expectedSelector, - stateFile: resolve(directory, `${incidentId}.state.json`), - summaryFile: resolve(directory, `${incidentId}.summaries.jsonl`), - markdownFile: resolve(directory, `${incidentId}.summary.md`), - restart: enabledFlags.has('restart'), - preDrainDryRun, - migrationPolicy, - recoverySourceCellId, - capacityCellId, - maxSamplesThisRun - } -} - -async function fileExists(path: string): Promise { - try { - await stat(path) - return true - } catch { - return false - } -} - -async function syncFile(path: string): Promise { - const handle = await open(path, 'r') - try { - await handle.sync() - } finally { - await handle.close() - } -} - -async function persistState(path: string, state: IncidentMonitorState): Promise { - await mkdir(dirname(path), { recursive: true, mode: 0o700 }) - await chmod(dirname(path), 0o700) - const temporaryPath = `${path}.tmp` - await writeFile(temporaryPath, `${JSON.stringify(state)}\n`, { mode: 0o600 }) - await chmod(temporaryPath, 0o600) - await syncFile(temporaryPath) - await rename(temporaryPath, path) - await syncFile(path) -} - -async function appendCheckpoint(path: string, checkpoint: IncidentCheckpoint): Promise { - await mkdir(dirname(path), { recursive: true, mode: 0o700 }) - await chmod(dirname(path), 0o700) - if (await fileExists(path)) { - const existing = (await readFile(path, 'utf8')) - .trim() - .split('\n') - .filter(Boolean) - .map((line) => JSON.parse(line) as IncidentCheckpoint) - if ( - existing.some((entry) => - entry.windowSequence === checkpoint.windowSequence && - entry.checkpointMinute === checkpoint.checkpointMinute - ) - ) return - } - await appendFile(path, `${JSON.stringify(checkpoint)}\n`, { mode: 0o600 }) - await chmod(path, 0o600) - await syncFile(path) -} - -function markdownFailure(checkpoint: IncidentCheckpoint): string { - if (checkpoint.failures.length === 0) return 'none' - return checkpoint.failures - .map((failure) => { - const signal = failure.signal ? `/${failure.signal}` : '' - return `${failure.source}/${failure.code}${signal}` - }) - .join(', ') -} - -async function writeMarkdownSummary( - path: string, - summaryPath: string, - incidentId: string, - environment: string -): Promise { - const checkpoints = (await readFile(summaryPath, 'utf8')) - .trim() - .split('\n') - .filter(Boolean) - .map((line) => JSON.parse(line) as IncidentCheckpoint) - const rows = checkpoints.map((checkpoint) => [ - `| ${checkpoint.windowSequence}`, - checkpoint.checkpointMinute, - checkpoint.status, - checkpoint.sampleCount, - `${markdownFailure(checkpoint)} |` - ].join(' | ')) - const markdown = [ - '# Relay incident monitor', - '', - `Incident: \`${incidentId}\``, - '', - `Environment: \`${environment}\``, - '', - `Expected selector: \`${JSON.stringify(checkpoints[0]?.expectedSelector ?? null)}\``, - '', - `Migration policy: \`${checkpoints[0]?.migrationPolicy ?? 'unknown'}\``, - '', - `Recovery source: \`${checkpoints[0]?.recoverySourceCellId ?? 'none'}\``, - '', - `Capacity cell: \`${checkpoints[0]?.capacityCellId ?? 'none'}\``, - '', - '| Window | Minute | Status | Samples | Failures |', - '| ---: | ---: | --- | ---: | --- |', - ...rows, - '' - ].join('\n') - const temporaryPath = `${path}.tmp` - await writeFile(temporaryPath, markdown, { mode: 0o600 }) - await chmod(temporaryPath, 0o600) - await syncFile(temporaryPath) - await rename(temporaryPath, path) - await syncFile(path) -} - -export function suppliedIdentityToken(value: string | undefined): string | null { - if (value === undefined) return null - if ( - value.length > 8_192 || - !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(value) - ) { - throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is invalid') - } - return value -} - -async function readInitialState( - options: CliOptions, - now: () => number = Date.now -): Promise { - const exists = await fileExists(options.stateFile) - if (exists && !options.restart) { - throw new Error('incident state already exists; pass --restart to resume it') - } - if (!exists && options.restart) throw new Error('no incident state exists to restart') - if (!exists) { - return initialIncidentMonitorState({ - incidentId: options.incidentId, - environment: options.environment, - expectedSelector: options.expectedSelector, - preDrainDryRun: options.preDrainDryRun, - migrationPolicy: options.migrationPolicy, - recoverySourceCellId: options.recoverySourceCellId, - capacityCellId: options.capacityCellId, - startedAt: new Date(now()).toISOString(), - durationMinutes: options.durationMinutes, - intervalMs: options.intervalMs - }) - } - const state = StateSchema.parse( - JSON.parse(await readFile(options.stateFile, 'utf8')) - ) as IncidentMonitorState - if ( - state.incidentId !== options.incidentId || - state.environment !== options.environment || - JSON.stringify(state.expectedSelector) !== JSON.stringify(options.expectedSelector) || - state.preDrainDryRun !== options.preDrainDryRun || - state.migrationPolicy !== options.migrationPolicy || - state.recoverySourceCellId !== options.recoverySourceCellId || - state.capacityCellId !== options.capacityCellId || - state.durationMinutes !== options.durationMinutes || - state.intervalMs !== options.intervalMs - ) { - throw new Error('restart arguments do not match durable incident state') - } - return state -} - -export async function runIncidentMonitorCli( - argv: string[], - dependencies: { - cwd?: string - now?: () => number - wait?: (ms: number) => Promise - gcloud?: ReturnType - collect?: () => Promise - writeOutput?: (value: string) => void - environment?: NodeJS.ProcessEnv - } = {} -): Promise { - const options = parseIncidentMonitorArguments(argv, dependencies.cwd) - const state = await readInitialState(options, dependencies.now) - const baseGcloud = dependencies.gcloud ?? createGcloudClient() - const token = suppliedIdentityToken( - (dependencies.environment ?? process.env).ORCA_RELAY_ADMIN_ID_TOKEN - ) - await persistState(options.stateFile, state) - const gcloud = token - ? { ...baseGcloud, identityToken: async () => token } - : baseGcloud - const collect = - dependencies.collect ?? - createIncidentSampleCollector(gcloud, { - environment: options.environment, - expectedSelector: options.expectedSelector, - ...(dependencies.now ? { now: dependencies.now } : {}) - }) - let samplesThisRun = 0 - const segmentedCollect = async (): Promise => { - try { - return await collect() - } finally { - samplesThisRun++ - } - } - const wait = - dependencies.wait ?? - ((ms: number) => new Promise((resolvePromise) => setTimeout(resolvePromise, ms))) - const segmentComplete = Symbol('segment-complete') - const output = dependencies.writeOutput ?? ((value) => process.stdout.write(`${value}\n`)) - let result: IncidentMonitorState - try { - result = await runIncidentMonitor(state, { - now: dependencies.now ?? Date.now, - wait: async (ms) => { - if ( - options.maxSamplesThisRun !== null && - samplesThisRun >= options.maxSamplesThisRun - ) { - throw segmentComplete - } - await wait(ms) - }, - collect: segmentedCollect, - warn: (message) => console.warn(message), - persist: async (nextState) => await persistState(options.stateFile, nextState), - checkpoint: async (checkpoint) => { - await appendCheckpoint(options.summaryFile, checkpoint) - await writeMarkdownSummary( - options.markdownFile, - options.summaryFile, - options.incidentId, - options.environment - ) - output(JSON.stringify(checkpoint)) - } - }) - } catch (error) { - if (error !== segmentComplete) throw error - return 0 - } - if (options.preDrainDryRun && !preDrainDryRunPassed(result)) return 2 - return result.frozenAt ? 2 : 0 -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - runIncidentMonitorCli(process.argv.slice(2)) - .then((code) => { - process.exitCode = code - }) - .catch((error: unknown) => { - console.error(error instanceof Error ? error.message : 'incident monitor failed') - process.exitCode = 1 - }) -} diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts deleted file mode 100644 index 704f55717fa..00000000000 --- a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts +++ /dev/null @@ -1,655 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js' -import type { GcloudClient } from './gcloud-client.js' -import { effectiveAdmissionState } from './incident-selector.js' -import { INCIDENT_MONITOR_THRESHOLDS } from './incident-monitor.js' -import { - directorSignals, - GOOGLE_METRICS, - readGoogleMetric, - readGoogleMetricWithEmptyRetry, - relayFiveMinuteDeltaSignal -} from './incident-monitor-sources.js' - -const now = Date.parse('2026-07-28T10:00:00.000Z') -const startAt = new Date(now - 5 * 60_000).toISOString() -const endAt = new Date(now).toISOString() -const productionCells = RELAY_OPS_ENVIRONMENTS.production.cells.map( - ({ cellId }) => cellId -) - -describe('incident monitor sources', () => { - it('uses legacy booleans only at selector generation zero', () => { - const membership = { - existingOnly: ['c1'], - migrationOnly: [], - general: ['c2'] - } - expect( - effectiveAdmissionState({ generation: 0, membership }, true, 'c1') - ).toBe('general') - expect( - effectiveAdmissionState({ generation: 1, membership }, true, 'c1') - ).toBe('existing-only') - }) - - it('sums DELTA metrics across the window and scopes every target exactly', async () => { - const filters: string[] = [] - const fetchImpl: typeof fetch = async (input) => { - const url = new URL(String(input)) - filters.push(url.searchParams.get('filter') ?? '') - return Response.json({ - timeSeries: [ - { - points: [ - { - interval: { endTime: new Date(now - 120_000).toISOString() }, - value: { int64Value: '2' } - }, - { - interval: { endTime: new Date(now - 60_000).toISOString() }, - value: { int64Value: '3' } - } - ] - } - ] - }) - } - const environment = RELAY_OPS_ENVIRONMENTS.production - const directorErrors = GOOGLE_METRICS.find( - (definition) => definition.signal === 'director.errors' - )! - const deadlocks = GOOGLE_METRICS.find( - (definition) => definition.signal === 'cloud_sql.deadlocks' - )! - await expect( - readGoogleMetric( - environment, - directorErrors, - 'secret-access-token', - startAt, - endAt, - fetchImpl - ) - ).resolves.toEqual({ - value: 5, - observedAt: new Date(now - 60_000).toISOString() - }) - await readGoogleMetric( - environment, - deadlocks, - 'secret-access-token', - startAt, - endAt, - fetchImpl - ) - expect(filters[0]).toContain( - 'resource.label."service_name"="orca-cloud-relay"' - ) - expect(filters[0]).toContain('metric.label."response_code"!="503"') - expect(filters[1]).toContain( - 'resource.label."database_id"="onorca-cloud:orca-cloud-auth-db"' - ) - }) - - it('zero-fills an expired sparse lock-wait point', async () => { - let pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs - const fetchImpl: typeof fetch = async () => Response.json({ - timeSeries: [{ - points: [{ - interval: { endTime: new Date(pointAt).toISOString() }, - value: { int64Value: '1' } - }] - }] - }) - const definition = GOOGLE_METRICS.find( - ({ signal }) => signal === 'cloud_sql.lock_waits' - )! - await expect(readGoogleMetric( - RELAY_OPS_ENVIRONMENTS.production, - definition, - 'secret-access-token', - startAt, - endAt, - fetchImpl - )).resolves.toEqual({ - value: 1, - observedAt: new Date(pointAt).toISOString() - }) - await expect(readGoogleMetric( - RELAY_OPS_ENVIRONMENTS.production, - definition, - 'secret-access-token', - startAt, - endAt, - fetchImpl, - () => now + 3_000 - )).resolves.toEqual({ - value: 1, - observedAt: new Date(pointAt).toISOString() - }) - pointAt-- - await expect(readGoogleMetric( - RELAY_OPS_ENVIRONMENTS.production, - definition, - 'secret-access-token', - startAt, - endAt, - fetchImpl - )).resolves.toEqual({ value: 0, observedAt: endAt }) - }) - - it('freshens a sparse zero without masking a recent nonzero lock wait', async () => { - let value = 0 - const pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs - const readAt = now + 11_879 - const fetchImpl: typeof fetch = async () => Response.json({ - timeSeries: [{ - points: [{ - interval: { endTime: new Date(pointAt).toISOString() }, - value: { int64Value: String(value) } - }] - }] - }) - const definition = GOOGLE_METRICS.find( - ({ signal }) => signal === 'cloud_sql.lock_waits' - )! - - const sparseZero = await readGoogleMetric( - RELAY_OPS_ENVIRONMENTS.production, - definition, - 'secret-access-token', - startAt, - endAt, - fetchImpl, - () => readAt - ) - expect(sparseZero).toEqual({ - value: 0, - observedAt: new Date(readAt).toISOString() - }) - expect(readAt - pointAt).toBe(191_879) - expect(readAt - Date.parse(sparseZero!.observedAt)).toBe(0) - - value = 20 - await expect(readGoogleMetric( - RELAY_OPS_ENVIRONMENTS.production, - definition, - 'secret-access-token', - startAt, - endAt, - fetchImpl, - () => readAt - )).resolves.toEqual({ - value: 20, - observedAt: new Date(pointAt).toISOString() - }) - }) - - it('preserves a recent nonzero lock wait across staggered series', async () => { - const nonzeroAt = now - 179_000 - const zeroAt = now - 178_000 - const definition = GOOGLE_METRICS.find( - ({ signal }) => signal === 'cloud_sql.lock_waits' - )! - const metric = await readGoogleMetric( - RELAY_OPS_ENVIRONMENTS.production, - definition, - 'secret-access-token', - startAt, - endAt, - async () => Response.json({ - timeSeries: [ - { - points: [{ - interval: { endTime: new Date(nonzeroAt).toISOString() }, - value: { int64Value: '7' } - }] - }, - { - points: [{ - interval: { endTime: new Date(zeroAt).toISOString() }, - value: { int64Value: '0' } - }] - } - ] - }) - ) - - expect(metric).toEqual({ - value: 7, - observedAt: new Date(nonzeroAt).toISOString() - }) - - await expect(readGoogleMetric( - RELAY_OPS_ENVIRONMENTS.production, - definition, - 'secret-access-token', - startAt, - endAt, - async () => Response.json({ - timeSeries: [{ - points: [ - { - interval: { endTime: new Date(nonzeroAt).toISOString() }, - value: { int64Value: '7' } - }, - { - interval: { endTime: new Date(zeroAt).toISOString() }, - value: { int64Value: '0' } - } - ] - }] - }) - )).resolves.toEqual({ value: 0, observedAt: endAt }) - }) - - it('retries an empty required metric without weakening its value', async () => { - let calls = 0 - const waits: number[] = [] - const definition = GOOGLE_METRICS.find( - ({ signal }) => signal === 'cloud_sql.cpu' - )! - await expect(readGoogleMetricWithEmptyRetry( - RELAY_OPS_ENVIRONMENTS.production, - definition, - 'secret-access-token', - startAt, - endAt, - async () => Response.json(calls++ === 0 - ? { timeSeries: [] } - : { - timeSeries: [{ - points: [{ - interval: { endTime: new Date(now - 60_000).toISOString() }, - value: { doubleValue: 0.81 } - }] - }] - }), - () => now, - async (ms) => { waits.push(ms) } - )).resolves.toEqual({ - value: 0.81, - observedAt: new Date(now - 60_000).toISOString() - }) - expect(calls).toBe(2) - expect(waits).toEqual([2_000]) - }) - - it('still reports a required metric missing after bounded retries', async () => { - let calls = 0 - const waits: number[] = [] - const definition = GOOGLE_METRICS.find( - ({ signal }) => signal === 'director.concurrency' - )! - await expect(readGoogleMetricWithEmptyRetry( - RELAY_OPS_ENVIRONMENTS.production, - definition, - 'secret-access-token', - startAt, - endAt, - async () => { - calls++ - return Response.json({ timeSeries: [] }) - }, - () => now, - async (ms) => { waits.push(ms) } - )).resolves.toBeNull() - expect(calls).toBe(3) - expect(waits).toEqual([2_000, 2_000]) - }) - - it('timestamps sparse retry aggregates at query completion', () => { - expect(relayFiveMinuteDeltaSignal({ - available: true, - points: [ - { at: new Date(now - 22 * 60_000).toISOString(), value: 7 }, - { at: new Date(now - 4 * 60_000).toISOString(), value: 2 } - ] - }, endAt)).toEqual({ value: 2, observedAt: endAt }) - expect(relayFiveMinuteDeltaSignal({ - available: true, - points: [{ at: new Date(now - 22 * 60_000).toISOString(), value: 7 }] - }, endAt)).toEqual({ value: 0, observedAt: endAt }) - expect(relayFiveMinuteDeltaSignal({ - available: false, - points: [] - }, endAt)).toBeNull() - }) - - // Why: the per-region cell latency bar is only correct if the tfvars region - // reaches the evaluator on every cell expectation. - it('carries the configured region onto every cell expectation', async () => { - const gcloud: GcloudClient = { - accessToken: async () => 'unused', - identityToken: async () => 'unused' - } - const selector = { - generation: 1, - membership: { - existingOnly: [], - migrationOnly: [], - general: productionCells - } - } - const fetchImpl: typeof fetch = async (_input, init) => { - const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string } - if (!body.cellId && !body.sourceCellId) return Response.json({ selector }) - if (body.cellId) { - return Response.json({ - status: { - enabled: true, - connectionCapacity: { hardCap: 600 }, - runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true } - } - }) - } - return Response.json({ - blocked: 0, - blockedExpiredUnregistered: 0, - registeredTargetInactive: 0 - }) - } - const result = await directorSignals('production', selector, gcloud, now, fetchImpl) - const regionById = new Map(result.cells.map((cell) => [cell.cellId, cell.region])) - expect(regionById.get('production-gce-c1')).toBe('us-central1') - expect(regionById.get('production-gce-c27')).toBe('asia-east2') - expect(result.cells).toHaveLength(productionCells.length) - for (const cell of RELAY_OPS_ENVIRONMENTS.production.cells) { - expect(regionById.get(cell.cellId)).toBe(cell.region) - } - }) - - it('aggregates admin state without returning tokens or response identities', async () => { - const identityToken = 'secret.header.signature' - const sensitiveIdentity = 'user@example.test' - const gcloud: GcloudClient = { - accessToken: async () => 'unused', - identityToken: async () => identityToken - } - let activeRequests = 0 - let maximumActiveRequests = 0 - let requestCount = 0 - const fetchImpl: typeof fetch = async (_input, init) => { - requestCount++ - activeRequests++ - maximumActiveRequests = Math.max(maximumActiveRequests, activeRequests) - expect(new Headers(init?.headers).get('authorization')).toBe( - `Bearer ${identityToken}` - ) - const body = JSON.parse(String(init?.body)) as { - cellId?: string - sourceCellId?: string - targetCellId?: string - } - await Promise.resolve() - activeRequests-- - if (!body.cellId && !body.sourceCellId) { - return Response.json({ - selector: { - generation: 1, - membership: { - existingOnly: productionCells.slice(2), - migrationOnly: ['production-gce-c2'], - general: ['production-gce-c1'] - } - } - }) - } - if (body.cellId) { - return Response.json({ - status: { - // Selector-era monitoring must ignore this legacy compatibility bit. - enabled: body.cellId !== 'production-gce-c1', - connectionCapacity: { - hardCap: body.cellId === 'production-gce-c1' ? 1_000 : 600 - }, - runtime: { - lastHeartbeatAt: now - 1_000, - heartbeatFresh: true - }, - userId: sensitiveIdentity - } - }) - } - return Response.json({ - blocked: 0, - blockedExpiredUnregistered: - body.sourceCellId === 'production-gce-c1' && - body.targetCellId === 'production-gce-c2' - ? 1 - : 0, - registeredTargetInactive: 0, - userId: sensitiveIdentity - }) - } - const result = await directorSignals( - 'production', - { - generation: 1, - membership: { - existingOnly: productionCells.slice(2), - migrationOnly: ['production-gce-c2'], - general: ['production-gce-c1'] - } - }, - gcloud, - now, - fetchImpl - ) - expect(requestCount).toBe(productionCells.length * 2) - expect(maximumActiveRequests).toBe(1) - expect( - result.source.signals['cell.production-gce-c1.migration_blocked'] - ).toMatchObject({ value: 1 }) - expect(result.cells.find((cell) => cell.cellId === 'production-gce-c1')).toMatchObject({ - expectedAdmissionState: 'general' - }) - expect( - result.source.signals['cell.production-gce-c1.admission_state'] - ).toMatchObject({ value: 2 }) - expect( - result.source.signals['cell.production-gce-c1.connection_hard_cap'] - ).toMatchObject({ value: 1_000 }) - expect( - result.source.signals['cell.production-gce-c2.admission_state'] - ).toMatchObject({ value: 1 }) - const serialized = JSON.stringify(result) - expect(serialized).not.toContain(identityToken) - expect(serialized).not.toContain(sensitiveIdentity) - }) - - // Why: the director maps a Cloud SQL pool connect timeout in cell-status onto - // a 404, so without this the whole sample dies on one 2 s database stall. - it('retries a transient director admin failure and then reports the cell', async () => { - const gcloud: GcloudClient = { - accessToken: async () => 'unused', - identityToken: async () => 'unused' - } - const selector = { - generation: 1, - membership: { existingOnly: [], migrationOnly: [], general: productionCells } - } - const waits: number[] = [] - let cellStatusCalls = 0 - const fetchImpl: typeof fetch = async (_input, init) => { - const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string } - if (!body.cellId && !body.sourceCellId) return Response.json({ selector }) - if (body.cellId) { - cellStatusCalls++ - if (cellStatusCalls === 1) { - return Response.json( - { error: 'timeout exceeded when trying to connect' }, - { status: 404 } - ) - } - return Response.json({ - status: { - enabled: true, - connectionCapacity: { hardCap: 600 }, - runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true } - } - }) - } - return Response.json({ - blocked: 0, - blockedExpiredUnregistered: 0, - registeredTargetInactive: 0 - }) - } - const result = await directorSignals( - 'production', - selector, - gcloud, - now, - fetchImpl, - async (ms) => { - waits.push(ms) - } - ) - expect(waits).toEqual([5_000]) - expect(cellStatusCalls).toBe(productionCells.length + 1) - expect(result.cells).toHaveLength(productionCells.length) - expect( - result.source.signals['cell.production-gce-c1.connection_hard_cap'] - ).toMatchObject({ value: 600 }) - }) - - // A rejected admin token is a decision, not weather: retrying it only burns - // the sample budget and hides the misconfiguration. - it('fails immediately on an unauthorized director admin response', async () => { - const gcloud: GcloudClient = { - accessToken: async () => 'unused', - identityToken: async () => 'unused' - } - const selector = { - generation: 1, - membership: { existingOnly: [], migrationOnly: [], general: productionCells } - } - const waits: number[] = [] - let requestCount = 0 - const fetchImpl: typeof fetch = async (_input, init) => { - requestCount++ - const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string } - if (!body.cellId && !body.sourceCellId) return Response.json({ selector }) - return Response.json({ error: 'invalid_token' }, { status: 401 }) - } - await expect( - directorSignals('production', selector, gcloud, now, fetchImpl, async (ms) => { - waits.push(ms) - }) - ).rejects.toThrow('Relay admin telemetry returned 401') - expect(requestCount).toBe(2) - expect(waits).toEqual([]) - }) - - // Why: on 2026-10-19 cell-status refused reserve production-gce-c7 with a 404 on every sample. - it('marks a reserve cell unprobed and skips its refused migration reads', async () => { - const gcloud: GcloudClient = { - accessToken: async () => 'unused', - identityToken: async () => 'unused' - } - const reserveCellId = 'production-gce-c7' - const selector = { - generation: 1, - membership: { - existingOnly: [], - migrationOnly: ['production-gce-c2'], - general: productionCells.filter((cellId) => cellId !== 'production-gce-c2') - } - } - const waits: number[] = [] - const fetchImpl: typeof fetch = async (input, init) => { - const path = new URL(String(input)).pathname - const body = JSON.parse(String(init?.body)) as Record - if (path === '/v1/admin/admission-selector/status') return Response.json({ selector }) - if (path === '/v1/admin/cell-status') { - if (body.cellId === reserveCellId) { - return Response.json({ error: 'cell_in_reserve_mode' }, { status: 404 }) - } - return Response.json({ - status: { - enabled: true, - connectionCapacity: { hardCap: 600 }, - runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true } - } - }) - } - expect(path).toBe('/v1/admin/evacuation-status') - expect(body.sourceCellId).not.toBe(reserveCellId) - return Response.json({ - blocked: 0, - blockedExpiredUnregistered: 0, - registeredTargetInactive: 0 - }) - } - const result = await directorSignals('production', selector, gcloud, now, fetchImpl, async (ms) => { - waits.push(ms) - }) - expect(waits).toEqual([]) - expect(result.cells.find((cell) => cell.cellId === reserveCellId)).toMatchObject({ - reserve: true - }) - expect(result.cells.find((cell) => cell.cellId === 'production-gce-c1')).not.toHaveProperty( - 'reserve' - ) - const signals = result.source.signals - expect(signals[`cell.${reserveCellId}.admission_state`]).toMatchObject({ value: 2 }) - // Why: 10-19 run 39225932592 froze reserve c7 at 731 connections against the 500 default. - const configuredCap = RELAY_OPS_ENVIRONMENTS.production.cells.find( - (cell) => cell.cellId === reserveCellId - )?.connectionHardCap - expect(configuredCap).toBe(1_000) - expect(signals[`cell.${reserveCellId}.connection_hard_cap`]).toMatchObject({ - value: configuredCap - }) - for (const suffix of ['heartbeat_fresh', 'heartbeat_age_ms', 'migration_blocked']) { - expect(signals[`cell.${reserveCellId}.${suffix}`]).toBeUndefined() - } - expect(signals['cell.production-gce-c1.heartbeat_age_ms']).toMatchObject({ value: 1_000 }) - }) - - it('still fails a cell-status 404 that is not the reserve refusal', async () => { - const gcloud: GcloudClient = { - accessToken: async () => 'unused', - identityToken: async () => 'unused' - } - const selector = { - generation: 1, - membership: { existingOnly: [], migrationOnly: [], general: productionCells } - } - const collect = (error: string) => - directorSignals('production', selector, gcloud, now, async (input) => { - const path = new URL(String(input)).pathname - if (path === '/v1/admin/admission-selector/status') return Response.json({ selector }) - return Response.json({ error }, { status: 404 }) - }, async () => {}) - for (const error of ['cell_not_found', 'cell_in_reserve_mode_soon', '']) { - await expect(collect(error)).rejects.toThrow('Relay admin telemetry returned 404') - } - }) - - // A 404 the director means (wrong role) must not be retried either. - it('does not retry a director-only 404', async () => { - const gcloud: GcloudClient = { - accessToken: async () => 'unused', - identityToken: async () => 'unused' - } - let requestCount = 0 - const fetchImpl: typeof fetch = async () => { - requestCount++ - return Response.json({ error: 'director_only' }, { status: 404 }) - } - await expect( - directorSignals( - 'production', - { generation: 1, membership: { existingOnly: [], migrationOnly: [], general: productionCells } }, - gcloud, - now, - fetchImpl, - async () => {} - ) - ).rejects.toThrow('Relay admin telemetry returned 404') - expect(requestCount).toBe(1) - }) -}) diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.ts deleted file mode 100644 index f69d57b11bd..00000000000 --- a/cloud/apps/relay-ops/src/incident-monitor-sources.ts +++ /dev/null @@ -1,783 +0,0 @@ -import { z } from 'zod' -import { buildDashboardSnapshot } from './dashboard-snapshot.js' -import type { - RelayOpsEnvironment, - RelayOpsEnvironmentId -} from './environment-config.js' -import { relayOpsEnvironment } from './environment-config.js' -import type { GcloudClient } from './gcloud-client.js' -import type { RelayMetricSnapshot } from './monitoring-snapshot.js' -import { - AdmissionSelectorSchema, - effectiveAdmissionState, - normalizeSelectorMembership, - selectorCellState, - type AdmissionSelector, -} from './incident-selector.js' -import { - INCIDENT_MONITOR_THRESHOLDS, - type IncidentSample, - type IncidentSignal, - type IncidentSource -} from './incident-monitor.js' - -const NumericSchema = z.union([z.number(), z.string()]) - .transform(Number) - .pipe(z.number().finite()) -export const MonitoringPointSchema = z.object({ - interval: z.object({ endTime: z.string() }), - value: z.object({ - doubleValue: NumericSchema.optional(), - int64Value: NumericSchema.optional(), - distributionValue: z.object({ - mean: NumericSchema.optional(), - range: z.object({ max: NumericSchema.optional() }).optional() - }).optional() - }) -}) -export const MonitoringResponseSchema = z.object({ - timeSeries: z.array(z.object({ points: z.array(MonitoringPointSchema) })).default([]), - nextPageToken: z.string().optional() -}) -const CellStatusSchema = z.object({ - status: z.object({ - enabled: z.boolean(), - connectionCapacity: z - .object({ hardCap: z.number().int().positive() }) - .nullable(), - runtime: z.object({ - lastHeartbeatAt: z.number(), - heartbeatFresh: z.boolean() - }).nullable() - }) -}) -const SelectorStatusSchema = z.object({ - selector: AdmissionSelectorSchema -}) -const MigrationStatusSchema = z.object({ - blocked: z.number().int().nonnegative(), - registeredTargetInactive: z.number().int().nonnegative(), - blockedExpiredUnregistered: z.number().int().nonnegative() -}) - -export type GoogleMetricDefinition = { - signal: string - type: string - resourceFilter: string - aggregation: 'latest-max' | 'latest-sum' | 'window-sum' - emptyIsZero?: boolean - zeroAfterMs?: number -} - -export const GOOGLE_METRICS: GoogleMetricDefinition[] = [ - { - signal: 'cloud_sql.cpu', - type: 'cloudsql.googleapis.com/database/cpu/utilization', - resourceFilter: 'resource.type="cloudsql_database"', - aggregation: 'latest-max' - }, - { - signal: 'cloud_sql.memory', - type: 'cloudsql.googleapis.com/database/memory/utilization', - resourceFilter: 'resource.type="cloudsql_database"', - aggregation: 'latest-max' - }, - { - signal: 'cloud_sql.backends', - type: 'cloudsql.googleapis.com/database/postgresql/num_backends', - resourceFilter: 'resource.type="cloudsql_database"', - aggregation: 'latest-sum' - }, - { - signal: 'cloud_sql.lock_waits', - type: 'cloudsql.googleapis.com/database/postgresql/backends_in_wait', - resourceFilter: - 'resource.type="cloudsql_database" AND metric.label."wait_event_type"="Lock"', - aggregation: 'latest-max', - emptyIsZero: true, - zeroAfterMs: INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs - }, - { - signal: 'cloud_sql.deadlocks', - type: 'cloudsql.googleapis.com/database/postgresql/deadlock_count', - resourceFilter: 'resource.type="cloudsql_database"', - aggregation: 'window-sum', - emptyIsZero: true - }, - { - signal: 'director.instances', - type: 'run.googleapis.com/container/instance_count', - resourceFilter: 'resource.type="cloud_run_revision"', - aggregation: 'latest-sum' - }, - { - signal: 'director.cpu', - type: 'run.googleapis.com/container/cpu/utilizations', - resourceFilter: 'resource.type="cloud_run_revision"', - aggregation: 'latest-max' - }, - { - signal: 'director.memory', - type: 'run.googleapis.com/container/memory/utilizations', - resourceFilter: 'resource.type="cloud_run_revision"', - aggregation: 'latest-max' - }, - { - signal: 'director.concurrency', - type: 'run.googleapis.com/container/max_request_concurrencies', - resourceFilter: - 'resource.type="cloud_run_revision" AND metric.label."state"="active"', - aggregation: 'latest-max' - }, - { - signal: 'director.errors', - type: 'run.googleapis.com/request_count', - resourceFilter: - 'resource.type="cloud_run_revision" AND metric.label."response_code_class"="5xx" AND metric.label."response_code"!="503"', - aggregation: 'window-sum', - emptyIsZero: true - }, - { - signal: 'auth.errors', - type: 'run.googleapis.com/request_count', - resourceFilter: - 'resource.type="cloud_run_revision" AND metric.label."response_code_class"="5xx"', - aggregation: 'window-sum', - emptyIsZero: true - }, - { - signal: 'auth.requests', - type: 'run.googleapis.com/request_count', - resourceFilter: 'resource.type="cloud_run_revision"', - aggregation: 'window-sum', - emptyIsZero: true - } -] - -export function pointValue(point: z.infer): number { - return ( - point.value.doubleValue ?? - point.value.int64Value ?? - point.value.distributionValue?.range?.max ?? - point.value.distributionValue?.mean ?? - 0 - ) -} - -function serviceFilter(signal: string, directorService: string, authService: string): string { - if (signal.startsWith('director.')) { - return `resource.label."service_name"="${directorService}"` - } - if (signal.startsWith('auth.')) { - return `resource.label."service_name"="${authService}"` - } - return '' -} - -function targetFilter( - definition: GoogleMetricDefinition, - environment: RelayOpsEnvironment -): string { - if (definition.signal.startsWith('cloud_sql.')) { - return `resource.label."database_id"="${environment.project}:${environment.sqlInstance}"` - } - return serviceFilter( - definition.signal, - environment.directorService, - environment.authService - ) -} - -export async function googleJson( - fetchImpl: typeof fetch, - token: string, - url: URL | string, - init: RequestInit = {} -): Promise { - const response = await fetchImpl(url, { - ...init, - headers: { - authorization: `Bearer ${token}`, - ...(init.body ? { 'content-type': 'application/json' } : {}) - }, - signal: AbortSignal.timeout(30_000) - }) - if (!response.ok) throw new Error(`Google telemetry returned ${response.status}`) - return await response.json() -} - -export async function readGoogleMetric( - environment: RelayOpsEnvironment, - definition: GoogleMetricDefinition, - token: string, - startAt: string, - endAt: string, - fetchImpl: typeof fetch, - now: () => number = () => Date.parse(endAt) -): Promise { - const url = new URL( - `https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries` - ) - const filters = [ - `metric.type="${definition.type}"`, - definition.resourceFilter, - targetFilter(definition, environment) - ].filter(Boolean) - url.searchParams.set('filter', filters.join(' AND ')) - url.searchParams.set('interval.startTime', startAt) - url.searchParams.set('interval.endTime', endAt) - url.searchParams.set('view', 'FULL') - url.searchParams.set('pageSize', '1000') - const parsed = MonitoringResponseSchema.parse( - await googleJson(fetchImpl, token, url) - ) - if (parsed.nextPageToken) throw new Error('Google metric pagination is incomplete') - const queryEndMs = Date.parse(endAt) - const readAtMs = Math.max(queryEndMs, now()) - const readAt = new Date(readAtMs).toISOString() - const points = parsed.timeSeries.flatMap((series) => series.points) - if (points.length === 0) { - return definition.emptyIsZero ? { value: 0, observedAt: readAt } : null - } - const zeroAfterMs = definition.zeroAfterMs - if (definition.emptyIsZero && zeroAfterMs !== undefined) { - const latestSeriesPoints = parsed.timeSeries.flatMap((series) => { - const seriesNewestAt = Math.max( - ...series.points.map((point) => Date.parse(point.interval.endTime)) - ) - return series.points.filter( - (point) => Date.parse(point.interval.endTime) === seriesNewestAt - ) - }) - const futurePoints = latestSeriesPoints.filter( - (point) => Date.parse(point.interval.endTime) > queryEndMs - ) - if (futurePoints.length > 0) { - return { - value: Math.max(...futurePoints.map(pointValue)), - observedAt: new Date(Math.max( - ...futurePoints.map((point) => Date.parse(point.interval.endTime)) - )).toISOString() - } - } - const recentNonzero = latestSeriesPoints.filter((point) => { - const pointAt = Date.parse(point.interval.endTime) - return pointValue(point) > 0 && queryEndMs - pointAt <= zeroAfterMs - }) - if (recentNonzero.length === 0) return { value: 0, observedAt: readAt } - return { - value: Math.max(...recentNonzero.map(pointValue)), - observedAt: new Date(Math.min( - ...recentNonzero.map((point) => Date.parse(point.interval.endTime)) - )).toISOString() - } - } - const newestAt = Math.max(...points.map((point) => Date.parse(point.interval.endTime))) - const selected = definition.aggregation === 'window-sum' - ? points - : points.filter((point) => Date.parse(point.interval.endTime) === newestAt) - const values = selected.map(pointValue) - const value = - definition.aggregation !== 'latest-max' - ? values.reduce((total, entry) => total + entry, 0) - : Math.max(...values) - return { - value, - observedAt: new Date(newestAt).toISOString() - } -} - -export async function readGoogleMetricWithEmptyRetry( - environment: RelayOpsEnvironment, - definition: GoogleMetricDefinition, - token: string, - startAt: string, - endAt: string, - fetchImpl: typeof fetch, - now: () => number = () => Date.parse(endAt), - wait: (ms: number) => Promise = async (ms) => - await new Promise((resolve) => setTimeout(resolve, ms)) -): Promise { - for (let attempt = 0; attempt < 3; attempt++) { - const signal = await readGoogleMetric( - environment, - definition, - token, - startAt, - endAt, - fetchImpl, - now - ) - if (signal !== null || definition.emptyIsZero) return signal - if (attempt < 2) await wait(2_000) - } - return null -} - -function addSignal( - signals: Record, - name: string, - value: number | null, - observedAt: string | null -): void { - if (value === null || observedAt === null) return - signals[name] = { value, observedAt } -} - -function endpointSignals( - snapshot: Awaited>, - nowAt: string -): IncidentSource { - const signals: Record = {} - const addEndpoint = ( - prefix: string, - endpoint: { health: boolean | null; ready: boolean | null; latencyMs: number | null }, - unavailableIsZero = false - ) => { - addSignal( - signals, - `${prefix}.health`, - endpoint.health === null ? (unavailableIsZero ? 0 : null) : Number(endpoint.health), - nowAt - ) - addSignal( - signals, - `${prefix}.ready`, - endpoint.ready === null ? (unavailableIsZero ? 0 : null) : Number(endpoint.ready), - nowAt - ) - addSignal(signals, `${prefix}.latency_ms`, endpoint.latencyMs, nowAt) - } - addEndpoint('director', snapshot.resources.directorEndpoint) - addEndpoint('auth', snapshot.resources.authEndpoint) - for (const cell of snapshot.resources.cells) { - addEndpoint(`cell.${cell.cellId}`, cell.endpoint, true) - } - return { observedAt: nowAt, signals } -} - -export function relayFiveMinuteDeltaSignal( - metric: Pick, - endAt: string -): IncidentSignal | null { - if (!metric.available) return null - const endMs = Date.parse(endAt) - if (!Number.isFinite(endMs)) throw new Error('Relay telemetry end time is invalid') - return { - value: metric.points - .filter((point) => { - const pointMs = Date.parse(point.at) - return pointMs >= endMs - 300_000 && pointMs <= endMs - }) - .reduce((total, point) => total + point.value, 0), - observedAt: endAt - } -} - -function relaySignals( - snapshot: Awaited> -): IncidentSource { - const metrics = snapshot.monitoring.metrics - const signals: Record = {} - addSignal( - signals, - 'relay.pool_waiting', - metrics.db_waiters_max.latest, - metrics.db_waiters_max.latestAt - ) - addSignal( - signals, - 'relay.pool_wait_ms', - metrics.db_wait_ms_max.latest, - metrics.db_wait_ms_max.latestAt - ) - const retries = relayFiveMinuteDeltaSignal( - metrics.postgres_retries, - snapshot.monitoring.endAt - ) - if (retries) signals['relay.postgres_retries'] = retries - const retryExhausted = relayFiveMinuteDeltaSignal( - metrics.postgres_retry_exhausted, - snapshot.monitoring.endAt - ) - if (retryExhausted) signals['relay.postgres_retry_exhausted'] = retryExhausted - for (const cell of snapshot.resources.cells) { - addSignal( - signals, - `cell.${cell.cellId}.connections`, - metrics.total_connections.latestByCell[cell.cellId] ?? null, - metrics.total_connections.latestAt - ) - addSignal( - signals, - `cell.${cell.cellId}.queued_bytes`, - metrics.queued_bytes.latestByCell[cell.cellId] ?? null, - metrics.queued_bytes.latestAt - ) - } - const observedTimes = Object.values(signals).map((entry) => Date.parse(entry.observedAt)) - if (observedTimes.length === 0) throw new Error('Relay telemetry is unavailable') - const observedAt = new Date(Math.max(...observedTimes)).toISOString() - return { observedAt, signals } -} - -const ADMIN_RETRY_ATTEMPTS = 3 -const ADMIN_RETRY_INTERVAL_MS = 5_000 -const ADMIN_RETRYABLE_STATUSES = new Set([500, 502, 503, 504]) -const AdminErrorBodySchema = z.object({ error: z.string() }) -// The director's /v1/admin/cell-status maps any thrown operation error onto 404 -// with an {error: message} body, so a Cloud SQL pool connect timeout arrives -// here as a 404. These are the pool-acquire and transient SQLSTATE messages the -// relay app's own isRelayDatabaseTransientError treats as re-runnable. -const ADMIN_TRANSIENT_ERROR_MESSAGES = [ - 'timeout exceeded when trying to connect', - 'Connection terminated due to connection timeout', - 'Connection terminated unexpectedly', - 'database_temporarily_unavailable', - 'deadlock detected', - 'canceling statement due to', - 'too many connections', - 'the database system is shutting down', - 'the database system is starting up' -] - -const defaultWait = async (ms: number): Promise => - await new Promise((resolveWait) => setTimeout(resolveWait, ms)) - -// A dropped socket or the 30 s AbortSignal firing; both leave the request with -// no verdict, and every admin read here is a plain query. -function transientFetchFailure(error: unknown): boolean { - if (error instanceof TypeError) return true - const name = (error as { name?: unknown } | null)?.name - return name === 'AbortError' || name === 'TimeoutError' -} - -function adminErrorMessage(body: string): string | null { - const parsed = AdminErrorBodySchema.safeParse( - ((): unknown => { - try { - return JSON.parse(body) - } catch { - return null - } - })() - ) - return parsed.success ? parsed.data.error : null -} - -function retryableAdminStatus(status: number, body: string): boolean { - if (ADMIN_RETRYABLE_STATUSES.has(status)) return true - // 401/403/400/413 and a 404 for 'director_only' are decisions, not weather. - if (status !== 404) return false - const message = adminErrorMessage(body) - return ( - message !== null && - ADMIN_TRANSIENT_ERROR_MESSAGES.some((transient) => message.includes(transient)) - ) -} - -// Step 5: cell-status refuses a reserve cell with exactly this 404 body; any other 404 stays a failure. -class AdminReserveCellRefusal extends Error { - constructor(status: number) { - super(`Relay admin telemetry returned ${status}`) - } -} - -async function adminPost( - fetchImpl: typeof fetch, - origin: string, - token: string, - path: string, - body: unknown, - wait: (ms: number) => Promise = defaultWait -): Promise { - for (let attempt = 1; ; attempt++) { - const lastAttempt = attempt >= ADMIN_RETRY_ATTEMPTS - let response: Response - try { - response = await fetchImpl(`${origin}${path}`, { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(30_000) - }) - } catch (error) { - if (lastAttempt || !transientFetchFailure(error)) throw error - console.warn( - `relay admin telemetry retrying ${path} after network failure` + - ` (attempt ${attempt}/${ADMIN_RETRY_ATTEMPTS})` - ) - await wait(ADMIN_RETRY_INTERVAL_MS) - continue - } - if (response.ok) return await response.json() - // Status only: the body carries a director error message, which must not - // reach the log the way an identity would. - const errorBody = await response.text().catch(() => '') - if (response.status === 404 && adminErrorMessage(errorBody) === 'cell_in_reserve_mode') { - throw new AdminReserveCellRefusal(response.status) - } - if (lastAttempt || !retryableAdminStatus(response.status, errorBody)) { - throw new Error(`Relay admin telemetry returned ${response.status}`) - } - console.warn( - `relay admin telemetry retrying ${path} after ${response.status}` + - ` (attempt ${attempt}/${ADMIN_RETRY_ATTEMPTS})` - ) - await wait(ADMIN_RETRY_INTERVAL_MS) - } -} - -export async function directorSignals( - environmentId: RelayOpsEnvironmentId, - expectedSelector: AdmissionSelector, - gcloud: GcloudClient, - nowMs: number, - fetchImpl: typeof fetch, - wait: (ms: number) => Promise = defaultWait -): Promise<{ - source: IncidentSource - selector: AdmissionSelector - cells: IncidentSample['cells'] -}> { - const environment = relayOpsEnvironment(environmentId) - if (!gcloud.identityToken) throw new Error('gcloud identity-token support is unavailable') - const token = await gcloud.identityToken(`${environment.directorOrigin}/v1/admin/drain`) - const configuredCellIds = new Set(environment.cells.map((cell) => cell.cellId)) - const rawSelector = SelectorStatusSchema.parse( - await adminPost( - fetchImpl, - environment.directorOrigin, - token, - '/v1/admin/admission-selector/status', - { v: 1 }, - wait - ) - ).selector - const selector = { - generation: rawSelector.generation, - membership: normalizeSelectorMembership(rawSelector.membership, configuredCellIds) - } - const statuses: Array<{ - cell: RelayOpsEnvironment['cells'][number] - status: ({ reserve: false } & z.infer['status']) | { reserve: true } - }> = [] - for (const cell of environment.cells) { - let response: unknown - try { - response = await adminPost( - fetchImpl, - environment.directorOrigin, - token, - '/v1/admin/cell-status', - { v: 1, cellId: cell.cellId }, - wait - ) - } catch (error) { - if (!(error instanceof AdminReserveCellRefusal)) throw error - statuses.push({ cell, status: { reserve: true } }) - continue - } - statuses.push({ - cell, - status: { reserve: false, ...CellStatusSchema.parse(response).status } - }) - } - const reserveCellIds = new Set( - statuses.filter(({ status }) => status.reserve).map(({ cell }) => cell.cellId) - ) - const migrationEntries: Array<{ - sourceCellId: string - migration: z.infer - }> = [] - const migrationTargets = new Set(expectedSelector.membership.migrationOnly) - for (const source of environment.cells) { - for (const target of environment.cells) { - if (source.cellId === target.cellId || !migrationTargets.has(target.cellId)) continue - // The director refuses migration reads, and every migration, on a reserve cell. - if (reserveCellIds.has(source.cellId)) continue - migrationEntries.push({ - sourceCellId: source.cellId, - migration: MigrationStatusSchema.parse( - await adminPost( - fetchImpl, - environment.directorOrigin, - token, - '/v1/admin/evacuation-status', - { - v: 1, - sourceCellId: source.cellId, - targetCellId: target.cellId, - completeReady: false - }, - wait - ) - ) - }) - } - } - const migrationBySource = new Map() - for (const { sourceCellId, migration } of migrationEntries) { - const aggregate = migrationBySource.get(sourceCellId) ?? { blocked: 0, targetInactive: 0 } - aggregate.blocked += migration.blocked + migration.blockedExpiredUnregistered - aggregate.targetInactive += migration.registeredTargetInactive - migrationBySource.set(sourceCellId, aggregate) - } - const nowAt = new Date(nowMs).toISOString() - const signals: Record = {} - for (const { cell, status } of statuses) { - const prefix = `cell.${cell.cellId}` - if (status.reserve) { - // cell-status carries the live cap; unset in Terraform keeps the evaluator's default, as there. - if (cell.connectionHardCap !== null) { - addSignal(signals, `${prefix}.connection_hard_cap`, cell.connectionHardCap, nowAt) - } - // Legacy admission needs relay_cells.enabled, which only cell-status reads: left missing. - if (selector.generation !== 0) { - addSignal( - signals, - `${prefix}.admission_state`, - ['existing-only', 'migration-only', 'general'].indexOf( - selectorCellState(selector, cell.cellId) - ), - nowAt - ) - } - continue - } - const admissionState = effectiveAdmissionState( - selector, - status.enabled, - cell.cellId - ) - addSignal( - signals, - `${prefix}.admission_state`, - ['existing-only', 'migration-only', 'general'].indexOf(admissionState), - nowAt - ) - addSignal( - signals, - `${prefix}.connection_hard_cap`, - status.connectionCapacity?.hardCap ?? - INCIDENT_MONITOR_THRESHOLDS.cellConnections, - nowAt - ) - if (status.runtime) { - addSignal( - signals, - `${prefix}.heartbeat_fresh`, - Number(status.runtime.heartbeatFresh), - nowAt - ) - addSignal( - signals, - `${prefix}.heartbeat_age_ms`, - Math.max(0, nowMs - status.runtime.lastHeartbeatAt), - nowAt - ) - } - const migration = migrationBySource.get(cell.cellId) ?? { blocked: 0, targetInactive: 0 } - addSignal(signals, `${prefix}.migration_blocked`, migration.blocked, nowAt) - addSignal( - signals, - `${prefix}.migration_target_inactive`, - migration.targetInactive, - nowAt - ) - } - return { - source: { observedAt: nowAt, signals }, - selector, - cells: statuses.map(({ cell, status }) => ({ - cellId: cell.cellId, - region: cell.region, - runtimeKnown: true, - powered: true, - expectedAdmissionState: selectorCellState(expectedSelector, cell.cellId), - ...(status.reserve ? { reserve: true } : {}) - })) - } -} - -export type IncidentSampleCollectorOptions = { - environment: RelayOpsEnvironmentId - expectedSelector: AdmissionSelector - fetchImpl?: typeof fetch - now?: () => number - wait?: (ms: number) => Promise -} - -export function createIncidentSampleCollector( - gcloud: GcloudClient, - options: IncidentSampleCollectorOptions -): () => Promise { - const fetchImpl = options.fetchImpl ?? fetch - const now = options.now ?? Date.now - return async () => { - const nowMs = now() - const nowAt = new Date(nowMs).toISOString() - const startAt = new Date(nowMs - 5 * 60_000).toISOString() - const environment = relayOpsEnvironment(options.environment) - const accessToken = gcloud.accessToken() - const cloudMetricEntries = accessToken.then(async (token) => await Promise.all( - GOOGLE_METRICS.map(async (definition) => [ - definition.signal, - await readGoogleMetricWithEmptyRetry( - environment, - definition, - token, - startAt, - nowAt, - fetchImpl, - now - ) - ] as const) - )) - const [snapshot, director, metricEntries] = await Promise.all([ - buildDashboardSnapshot(options.environment, gcloud, { - windowMinutes: 5, - now: new Date(nowMs), - fetchImpl - }), - directorSignals( - options.environment, - options.expectedSelector, - gcloud, - nowMs, - fetchImpl, - options.wait ?? defaultWait - ), - cloudMetricEntries - ]) - const cloudSignals = Object.fromEntries( - metricEntries.filter((entry): entry is [string, IncidentSignal] => entry[1] !== null) - ) - const relay = relaySignals(snapshot) - const poweredByCell = new Map( - snapshot.resources.cells.map((cell) => [ - cell.cellId, - { - runtimeKnown: cell.targetSize !== null, - powered: (cell.targetSize ?? 0) > 0 - } - ]) - ) - return { - collectedAt: nowAt, - selector: director.selector, - expectedSelector: options.expectedSelector, - sources: { - 'active-probe': endpointSignals(snapshot, nowAt), - 'cloud-monitoring': { observedAt: nowAt, signals: cloudSignals }, - 'relay-logs': relay, - 'director-admin': director.source - }, - cells: director.cells.map((cell) => ({ - ...cell, - runtimeKnown: poweredByCell.get(cell.cellId)?.runtimeKnown ?? false, - powered: poweredByCell.get(cell.cellId)?.powered ?? false - })) - } - } -} diff --git a/cloud/apps/relay-ops/src/incident-monitor.test.ts b/cloud/apps/relay-ops/src/incident-monitor.test.ts deleted file mode 100644 index a0eadd1b163..00000000000 --- a/cloud/apps/relay-ops/src/incident-monitor.test.ts +++ /dev/null @@ -1,1728 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - evaluateIncidentSample, - INCIDENT_CHECKPOINT_MINUTES, - INCIDENT_FRESHNESS_TOLERANCE_SAMPLES, - INCIDENT_MONITOR_THRESHOLDS, - INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS, - initialIncidentMonitorState, - preDrainDryRunPassed, - runIncidentMonitor, - type IncidentSample -} from './incident-monitor.js' - -const startedAt = Date.parse('2026-07-28T00:00:00.000Z') -const selector = { - generation: 1, - membership: { - existingOnly: [], - migrationOnly: [], - general: ['production-gce-c1'] - } -} -const signal = (value: number, at = startedAt) => ({ - value, - observedAt: new Date(at).toISOString() -}) - -function healthySample(at = startedAt): IncidentSample { - const observedAt = new Date(at).toISOString() - return { - collectedAt: observedAt, - selector, - expectedSelector: selector, - cells: [{ - cellId: 'production-gce-c1', - region: 'us-central1', - runtimeKnown: true, - powered: true, - expectedAdmissionState: 'general' - }], - sources: { - 'active-probe': { - observedAt, - signals: { - 'director.health': signal(1, at), - 'director.ready': signal(1, at), - 'director.latency_ms': signal(100, at), - 'auth.health': signal(1, at), - 'auth.ready': signal(1, at), - 'auth.latency_ms': signal(100, at), - 'cell.production-gce-c1.health': signal(1, at), - 'cell.production-gce-c1.ready': signal(1, at), - 'cell.production-gce-c1.latency_ms': signal(100, at) - } - }, - 'cloud-monitoring': { - observedAt, - signals: { - 'cloud_sql.cpu': signal(0.2, at), - 'cloud_sql.memory': signal(0.3, at), - 'cloud_sql.backends': signal(12, at), - 'cloud_sql.lock_waits': signal(0, at), - 'cloud_sql.deadlocks': signal(0, at), - 'director.instances': signal(5, at), - 'director.cpu': signal(0.2, at), - 'director.memory': signal(0.3, at), - 'director.concurrency': signal(5, at), - 'director.errors': signal(0, at), - 'auth.errors': signal(0, at), - 'auth.requests': signal(58_000, at) - } - }, - 'relay-logs': { - observedAt, - signals: { - 'relay.pool_waiting': signal(0, at), - 'relay.pool_wait_ms': signal(1, at), - 'relay.postgres_retries': signal(0, at), - 'relay.postgres_retry_exhausted': signal(0, at), - 'cell.production-gce-c1.connections': signal(100, at), - 'cell.production-gce-c1.queued_bytes': signal(0, at) - } - }, - 'director-admin': { - observedAt, - signals: { - 'cell.production-gce-c1.admission_state': signal(2, at), - 'cell.production-gce-c1.heartbeat_fresh': signal(1, at), - 'cell.production-gce-c1.heartbeat_age_ms': signal(1_000, at), - 'cell.production-gce-c1.migration_blocked': signal(0, at), - 'cell.production-gce-c1.migration_target_inactive': signal(0, at) - } - } - } - } -} - -describe('incident monitor evaluator', () => { - it('accepts a complete fresh sample at every exact boundary', () => { - const sample = healthySample() - sample.sources['active-probe']!.signals['director.latency_ms'] = - signal(INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs) - sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = - signal(INCIDENT_MONITOR_THRESHOLDS.cloudSqlCpuUtilization) - sample.sources['relay-logs']!.signals['relay.pool_wait_ms'] = - signal(INCIDENT_MONITOR_THRESHOLDS.relayPoolWaitMs) - sample.sources['relay-logs']!.signals['relay.postgres_retries'] = - signal(INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetries) - sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = - signal(INCIDENT_MONITOR_THRESHOLDS.cloudSqlBackends) - expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ - status: 'green', - failures: [] - }) - }) - - // Why: the global relay_cells lock made retries a steady-state rate (24 h p99 - // 1320/5min on 2026-09-04); the bar fences only unbounded growth beyond that. - it('tolerates the measured healthy retry rate and freezes above the bar', () => { - const healthy = healthySample() - healthy.sources['relay-logs']!.signals['relay.postgres_retries'] = signal(1504) - expect(evaluateIncidentSample(healthy, startedAt).status).toBe('green') - - const incident = healthySample() - incident.sources['relay-logs']!.signals['relay.postgres_retries'] = signal(2001) - expect(evaluateIncidentSample(incident, startedAt)).toMatchObject({ - status: 'freeze', - failures: [ - expect.objectContaining({ signal: 'relay.postgres_retries', threshold: 2000 }) - ] - }) - }) - - // Why: since #18521 the request path fails fast on the cell-inventory lock, so - // exhaustion is a steady contention rate (post-#18521 p90 147/5min, max 220), - // not an anomaly. The bar bounds it below the 2026-08-23 incident peak of 467. - it('tolerates the measured healthy exhaustion rate and freezes above the bar', () => { - const healthy = healthySample() - healthy.sources['relay-logs']!.signals['relay.postgres_retry_exhausted'] = signal(220) - expect(evaluateIncidentSample(healthy, startedAt).status).toBe('green') - - const atLimit = healthySample() - atLimit.sources['relay-logs']!.signals['relay.postgres_retry_exhausted'] = signal(300) - expect(evaluateIncidentSample(atLimit, startedAt).status).toBe('green') - - const incident = healthySample() - incident.sources['relay-logs']!.signals['relay.postgres_retry_exhausted'] = signal(301) - expect(evaluateIncidentSample(incident, startedAt)).toMatchObject({ - status: 'freeze', - failures: [ - expect.objectContaining({ signal: 'relay.postgres_retry_exhausted', threshold: 300 }) - ] - }) - }) - - // Why: an asia-east2 cell's /ready reaches auth and Cloud SQL in us-central1, so - // from the US runner it measures p50 0.88 s / max 2.7 s and the flat 2 000 bar - // froze three healthy gates on 2026-09-05 (c27 at 2568/2668/2685 ms). - it('holds cell endpoint latency to a per-region bar', () => { - const asiaTail = healthySample() - asiaTail.cells[0]!.region = 'asia-east2' - asiaTail.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] = - signal(2_685) - expect(evaluateIncidentSample(asiaTail, startedAt)).toMatchObject({ - status: 'green', - failures: [] - }) - - const asiaIncident = healthySample() - asiaIncident.cells[0]!.region = 'asia-east2' - asiaIncident.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] = - signal(4_001) - expect(evaluateIncidentSample(asiaIncident, startedAt)).toMatchObject({ - status: 'freeze', - failures: [ - expect.objectContaining({ - code: 'threshold_max', - source: 'active-probe', - signal: 'cell.production-gce-c1.latency_ms', - observed: 4_001, - threshold: 4_000 - }) - ] - }) - - const usIncident = healthySample() - usIncident.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] = - signal(2_001) - expect(evaluateIncidentSample(usIncident, startedAt)).toMatchObject({ - status: 'freeze', - failures: [ - expect.objectContaining({ - code: 'threshold_max', - signal: 'cell.production-gce-c1.latency_ms', - observed: 2_001, - threshold: 2_000 - }) - ] - }) - }) - - it('allows missing auth readiness and legacy existing-only connections', () => { - const sample = healthySample() - const legacySelector = { - generation: 1, - membership: { - existingOnly: ['production-gce-c1'], - migrationOnly: [], - general: [] - } - } - sample.selector = legacySelector - sample.expectedSelector = legacySelector - sample.cells[0]!.expectedAdmissionState = 'existing-only' - delete sample.sources['active-probe']!.signals['auth.ready'] - sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = - signal(900) - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.admission_state' - ] = signal(0) - expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ - status: 'green', - failures: [] - }) - }) - - it('fails loudly on every missing or stale source', () => { - const missing = healthySample() - delete missing.sources['relay-logs'] - expect(evaluateIncidentSample(missing, startedAt).failures).toContainEqual({ - code: 'source_missing', - source: 'relay-logs' - }) - const stale = healthySample( - startedAt - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1 - ) - const failures = evaluateIncidentSample(stale, startedAt).failures - expect(failures.some((failure) => failure.source === 'cloud-monitoring')).toBe(true) - expect(failures.some((failure) => failure.source === 'active-probe')).toBe(true) - }) - - // Why: production run 33944873727 at 2026-09-05T04:46:09Z read - // cloud_sql.lock_waits 189 286 ms old and restarted a 15-minute window on - // Google's publish lag. Cloud SQL documents 60 s sampling plus up to 165 s of - // invisibility, so that age is Google's clock, not our fleet. - it('reads a 189-second cloud signal as fresh and holds the other sources at 180 s', () => { - const lagged = healthySample() - lagged.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = - signal(0, startedAt - 189_286) - expect(evaluateIncidentSample(lagged, startedAt)).toMatchObject({ - status: 'green', - failures: [] - }) - const laggedDirector = healthySample() - laggedDirector.sources['director-admin']!.observedAt = - new Date(startedAt - 189_286).toISOString() - expect(evaluateIncidentSample(laggedDirector, startedAt).failures).toContainEqual( - expect.objectContaining({ code: 'source_stale', source: 'director-admin' }) - ) - }) - - it('still fails a cloud signal past the documented publish lag', () => { - const dark = healthySample() - dark.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal( - 0, - startedAt - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1 - ) - expect(evaluateIncidentSample(dark, startedAt).failures).toContainEqual( - expect.objectContaining({ - code: 'signal_stale', - source: 'cloud-monitoring', - signal: 'cloud_sql.lock_waits' - }) - ) - }) - - // Why: measured non-503 5xx per rolling five minutes over the 24 h to 2026-09-17 - // was p90 3 / p95 5 / p99 9 / max 52, so the old bar of 3 sat on the p90 and froze - // 29% of 15-minute gates on chronic director 500 bursts. - it('tolerates the measured chronic director error rate without relaxing other gates', () => { - for (const errors of [1, 3, 5, 9, 15]) { - const sample = healthySample() - sample.sources['cloud-monitoring']!.signals['director.errors'] = signal(errors) - expect(evaluateIncidentSample(sample, startedAt).status).toBe('green') - } - const excess = healthySample() - excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(16) - expect(evaluateIncidentSample(excess, startedAt).failures).toContainEqual( - expect.objectContaining({ signal: 'director.errors', observed: 16, threshold: 15 }) - ) - const pressure = healthySample() - pressure.sources['cloud-monitoring']!.signals['director.errors'] = signal(1) - pressure.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81) - expect(evaluateIncidentSample(pressure, startedAt).status).toBe('freeze') - }) - - it('tolerates isolated auth errors but freezes on a burst or a high error ratio', () => { - const isolated = healthySample() - isolated.sources['cloud-monitoring']!.signals['auth.errors'] = signal(1) - expect(evaluateIncidentSample(isolated, startedAt).status).toBe('green') - isolated.sources['cloud-monitoring']!.signals['auth.errors'] = signal(2) - expect(evaluateIncidentSample(isolated, startedAt).status).toBe('green') - - const burst = healthySample() - burst.sources['cloud-monitoring']!.signals['auth.errors'] = signal(3) - expect(evaluateIncidentSample(burst, startedAt).failures).toEqual([ - expect.objectContaining({ signal: 'auth.errors', observed: 3, threshold: 2 }) - ]) - - const ratio = healthySample() - ratio.sources['cloud-monitoring']!.signals['auth.errors'] = signal(2) - ratio.sources['cloud-monitoring']!.signals['auth.requests'] = signal(1_000) - expect(evaluateIncidentSample(ratio, startedAt).failures).toEqual([ - expect.objectContaining({ signal: 'auth.error_ratio', observed: 0.002, threshold: 0.001 }) - ]) - ratio.sources['cloud-monitoring']!.signals['auth.requests'] = signal(0) - expect(evaluateIncidentSample(ratio, startedAt).status).toBe('freeze') - - const unmeasured = healthySample() - unmeasured.sources['cloud-monitoring']!.signals['auth.errors'] = signal(1) - delete unmeasured.sources['cloud-monitoring']!.signals['auth.requests'] - expect(evaluateIncidentSample(unmeasured, startedAt).failures).toEqual([ - expect.objectContaining({ code: 'signal_missing', signal: 'auth.requests' }) - ]) - }) - - it('freezes on SQL, director, relay pool, heartbeat, and migration breaches', () => { - const sample = healthySample() - sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81) - sample.sources['cloud-monitoring']!.signals['director.instances'] = signal(7) - sample.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(801) - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.heartbeat_age_ms' - ] = signal(45_001) - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.migration_blocked' - ] = signal(1) - sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = - signal(501) - const evaluation = evaluateIncidentSample(sample, startedAt) - expect(evaluation.status).toBe('freeze') - expect(evaluation.failures.map((failure) => failure.signal)).toEqual( - expect.arrayContaining([ - 'cloud_sql.cpu', - 'director.instances', - 'relay.pool_waiting', - 'cell.production-gce-c1.connections', - 'cell.production-gce-c1.heartbeat_age_ms', - 'cell.production-gce-c1.migration_blocked' - ]) - ) - }) - - it('uses each cell reported physical connection cap', () => { - const sample = healthySample() - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.connection_hard_cap' - ] = signal(1_000) - sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = - signal(999) - - expect(evaluateIncidentSample(sample, startedAt).status).toBe('green') - sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = - signal(1_000) - expect(evaluateIncidentSample(sample, startedAt).status).toBe('freeze') - }) - - it('allows five active directors plus the warm rollback', () => { - const sample = healthySample() - sample.sources['cloud-monitoring']!.signals['director.instances'] = signal(6) - expect(evaluateIncidentSample(sample, startedAt).status).toBe('green') - }) - - it('allows bounded relay pool waiting below the latency ceiling', () => { - const sample = healthySample() - sample.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(800) - sample.sources['relay-logs']!.signals['relay.pool_wait_ms'] = signal(2_500) - expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ - status: 'green', - failures: [] - }) - sample.sources['relay-logs']!.signals['relay.pool_wait_ms'] = signal(2_501) - expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ - code: 'threshold_max', - source: 'relay-logs', - signal: 'relay.pool_wait_ms', - observed: 2_501, - threshold: 2_500 - }) - }) - - // Why: measured latest-sum over the 24 h to 2026-09-17 was p95 212 / p99 262 / - // max 282, so the old bar of 250 sat under the observed peak and froze 21.8% of - // 15-minute gates. 320 still fires at 65% of the 490 usable connections. - it('bounds Cloud SQL backends above measured healthy peaks', () => { - const sample = healthySample() - sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(282) - expect(evaluateIncidentSample(sample, startedAt).status).toBe('green') - sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(321) - expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ - code: 'threshold_max', - source: 'cloud-monitoring', - signal: 'cloud_sql.backends', - observed: 321, - threshold: 320 - }) - }) - - it('bounds SQL lock waiters and keeps deadlocks zero-tolerance', () => { - const sample = healthySample() - sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(20) - expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ - status: 'green', - failures: [] - }) - sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(21) - expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ - code: 'threshold_max', - source: 'cloud-monitoring', - signal: 'cloud_sql.lock_waits', - observed: 21, - threshold: 20 - }) - sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(0) - sample.sources['cloud-monitoring']!.signals['cloud_sql.deadlocks'] = signal(1) - expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ - code: 'threshold_max', - source: 'cloud-monitoring', - signal: 'cloud_sql.deadlocks', - observed: 1, - threshold: 0 - }) - }) - - it('allows only registered target inactivity during forward recovery', () => { - const sample = healthySample() - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.migration_target_inactive' - ] = signal(40) - expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ - code: 'threshold_max', - source: 'director-admin', - signal: 'cell.production-gce-c1.migration_target_inactive', - observed: 40, - threshold: 0 - }) - expect( - evaluateIncidentSample( - sample, - startedAt, - 'recover-forward', - 'production-gce-c1' - ) - ).toMatchObject({ - status: 'green', - failures: [] - }) - expect( - evaluateIncidentSample( - sample, - startedAt, - 'recover-forward', - 'production-gce-c2' - ).failures - ).toContainEqual(expect.objectContaining({ - signal: 'cell.production-gce-c1.migration_target_inactive' - })) - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.migration_blocked' - ] = signal(1) - expect( - evaluateIncidentSample( - sample, - startedAt, - 'recover-forward', - 'production-gce-c1' - ).failures - ).toContainEqual({ - code: 'threshold_max', - source: 'director-admin', - signal: 'cell.production-gce-c1.migration_blocked', - observed: 1, - threshold: 0 - }) - }) - - it('scopes registered target inactivity to the capacity cell', () => { - const sample = healthySample() - const scopedSelector = { - generation: 1, - membership: { - existingOnly: ['production-gce-c1'], - migrationOnly: [], - general: ['production-gce-c2', 'production-gce-c3'] - } - } - sample.selector = scopedSelector - sample.expectedSelector = scopedSelector - sample.cells[0]!.expectedAdmissionState = 'existing-only' - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.admission_state' - ] = signal(0) - sample.cells.push({ - cellId: 'production-gce-c2', - region: 'us-central1', - runtimeKnown: true, - powered: true, - expectedAdmissionState: 'general' - }) - sample.cells.push({ - cellId: 'production-gce-c3', - region: 'us-central1', - runtimeKnown: true, - powered: true, - expectedAdmissionState: 'general' - }) - for (const sourceName of ['active-probe', 'relay-logs', 'director-admin'] as const) { - const signals = sample.sources[sourceName]!.signals - for (const [name, value] of Object.entries(signals)) { - if (name.includes('production-gce-c1')) { - for (const cellId of ['production-gce-c2', 'production-gce-c3']) { - signals[name.replace('production-gce-c1', cellId)] = value - } - } - } - } - for (const cellId of ['production-gce-c2', 'production-gce-c3']) { - sample.sources['director-admin']!.signals[ - `cell.${cellId}.admission_state` - ] = signal(2) - } - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.migration_target_inactive' - ] = signal(40) - expect( - evaluateIncidentSample( - sample, - startedAt, - 'capacity-transition', - null, - 'production-gce-c2' - ) - ).toMatchObject({ status: 'green', failures: [] }) - expect( - evaluateIncidentSample( - sample, - startedAt, - 'capacity-transition', - null, - 'production-gce-c3' - ) - ).toMatchObject({ status: 'green', failures: [] }) - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c2.migration_target_inactive' - ] = signal(1) - expect( - evaluateIncidentSample( - sample, - startedAt, - 'capacity-transition', - null, - 'production-gce-c2' - ).failures - ).toContainEqual(expect.objectContaining({ - signal: 'cell.production-gce-c2.migration_target_inactive' - })) - }) - - it('freezes when expected admission has no powered runtime', () => { - const sample = healthySample() - sample.cells[0]!.powered = false - const evaluation = evaluateIncidentSample(sample, startedAt) - expect(evaluation.failures).toContainEqual({ - code: 'expected_admission_without_runtime', - source: 'director-admin', - signal: 'cell.production-gce-c1.powered', - observed: 0, - threshold: 1 - }) - }) - - it('ignores stale runtime signals for an expected offline existing-only cell', () => { - const sample = healthySample() - sample.cells[0] = { - ...sample.cells[0]!, - powered: false, - expectedAdmissionState: 'existing-only' - } - sample.expectedSelector = { - generation: sample.expectedSelector.generation, - membership: { - existingOnly: ['production-gce-c1'], - migrationOnly: [], - general: [] - } - } - sample.selector = sample.expectedSelector - sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] = signal(0) - sample.sources['active-probe']!.signals['cell.production-gce-c1.ready'] = signal(0) - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.admission_state' - ] = signal(0) - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.heartbeat_fresh' - ] = signal(0) - sample.sources['director-admin']!.signals[ - 'cell.production-gce-c1.heartbeat_age_ms' - ] = signal(9_000_000) - - expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ - status: 'green', - failures: [] - }) - }) - - it('freezes when cell power inventory is unavailable', () => { - const sample = healthySample() - sample.cells[0]!.runtimeKnown = false - expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({ - code: 'runtime_power_unknown', - source: 'cloud-monitoring', - signal: 'cell.production-gce-c1.powered' - }) - }) - - it('freezes on selector generation or tri-state membership drift', () => { - const generation = healthySample() - generation.selector = { ...generation.selector, generation: 2 } - expect(evaluateIncidentSample(generation, startedAt).failures).toContainEqual( - expect.objectContaining({ code: 'selector_mismatch' }) - ) - - const membership = healthySample() - membership.selector = { - generation: 1, - membership: { - existingOnly: ['production-gce-c1'], - migrationOnly: [], - general: [] - } - } - expect(evaluateIncidentSample(membership, startedAt).failures).toContainEqual( - expect.objectContaining({ code: 'selector_mismatch' }) - ) - }) - - // Why: the director refuses cell-status and migration reads on a reserve cell (2026-10-19, c7). - it('judges a reserve cell on probes, logs and selector, without its refused reads', () => { - const reserveSample = (): IncidentSample => { - const sample = healthySample() - sample.cells[0] = { ...sample.cells[0]!, reserve: true } - const admin = sample.sources['director-admin']!.signals - delete admin['cell.production-gce-c1.heartbeat_fresh'] - delete admin['cell.production-gce-c1.heartbeat_age_ms'] - delete admin['cell.production-gce-c1.migration_blocked'] - delete admin['cell.production-gce-c1.migration_target_inactive'] - return sample - } - expect(evaluateIncidentSample(reserveSample(), startedAt)).toMatchObject({ - status: 'green', - failures: [] - }) - - const crowded = reserveSample() - crowded.sources['relay-logs']!.signals['cell.production-gce-c1.queued_bytes'] = signal( - INCIDENT_MONITOR_THRESHOLDS.cellQueuedBytes + 1 - ) - expect(evaluateIncidentSample(crowded, startedAt).failures).toContainEqual( - expect.objectContaining({ signal: 'cell.production-gce-c1.queued_bytes' }) - ) - - // The connections ceiling is never skipped: the collector's configured cap, else the default. - const busy = reserveSample() - busy.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = signal(731) - expect(evaluateIncidentSample(busy, startedAt).failures).toContainEqual( - expect.objectContaining({ signal: 'cell.production-gce-c1.connections', threshold: 500 }) - ) - busy.sources['director-admin']!.signals['cell.production-gce-c1.connection_hard_cap'] = - signal(1_000) - expect(evaluateIncidentSample(busy, startedAt).status).toBe('green') - busy.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = signal(1_000) - expect(evaluateIncidentSample(busy, startedAt).failures).toContainEqual( - expect.objectContaining({ signal: 'cell.production-gce-c1.connections', threshold: 999 }) - ) - - const unready = reserveSample() - unready.sources['active-probe']!.signals['cell.production-gce-c1.ready'] = signal(0) - expect(evaluateIncidentSample(unready, startedAt).failures).toContainEqual( - expect.objectContaining({ signal: 'cell.production-gce-c1.ready' }) - ) - - const unknownAdmission = reserveSample() - delete unknownAdmission.sources['director-admin']!.signals[ - 'cell.production-gce-c1.admission_state' - ] - expect(evaluateIncidentSample(unknownAdmission, startedAt).failures).toContainEqual({ - code: 'signal_missing', - source: 'director-admin', - signal: 'cell.production-gce-c1.admission_state' - }) - - const notReserve = reserveSample() - notReserve.cells[0] = { ...notReserve.cells[0]!, reserve: false } - expect(evaluateIncidentSample(notReserve, startedAt).failures).toContainEqual( - expect.objectContaining({ - code: 'signal_missing', - signal: 'cell.production-gce-c1.heartbeat_fresh' - }) - ) - }) -}) - -describe('incident monitor lifecycle', () => { - it('persists the exact 90-minute checkpoints while polling every minute', async () => { - let now = startedAt - const checkpoints: number[] = [] - const state = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: false, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 90, - intervalMs: 60_000 - }) - const result = await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => healthySample(now), - persist: async () => {}, - checkpoint: async (summary) => { - checkpoints.push(summary.checkpointMinute) - } - }) - expect(checkpoints).toEqual([...INCIDENT_CHECKPOINT_MINUTES]) - expect(result.sampleCount).toBe(91) - expect(result.completedAt).not.toBeNull() - expect(result.frozenAt).toBeNull() - }) - - it('latches monitor freeze across a restart without rewriting its time', async () => { - let now = startedAt - let unhealthy = true - let persisted = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: false, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - const stop = new Error('stop after first persistence') - await expect( - runIncidentMonitor(persisted, { - now: () => now, - wait: async () => { - throw stop - }, - collect: async () => { - const sample = healthySample(now) - if (unhealthy) { - sample.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(31, now) - } - return sample - }, - persist: async (state) => { - persisted = structuredClone(state) - }, - checkpoint: async () => {} - }) - ).rejects.toThrow('stop after first persistence') - const frozenAt = persisted.frozenAt - unhealthy = false - now += 60_000 - const result = await runIncidentMonitor(persisted, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => healthySample(now), - persist: async () => {}, - checkpoint: async () => {} - }) - expect(result.frozenAt).toBe(frozenAt) - expect(preDrainDryRunPassed(result)).toBe(false) - }) - - it.each([15, 90])( - 'restarts a %i-minute continuous window after stale telemetry', - async (durationMinutes) => { - let now = startedAt - let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 - const checkpoints: Array<[number, number]> = [] - const state = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: durationMinutes === 15, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes, - intervalMs: 60_000 - }) - const result = await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - if (staleSamples > 0 && now >= startedAt + 5 * 60_000) { - staleSamples-- - return healthySample( - now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1 - ) - } - return healthySample(now) - }, - persist: async () => {}, - checkpoint: async (summary) => { - checkpoints.push([summary.windowSequence, summary.checkpointMinute]) - } - }) - const restartMinute = 5 + INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 - expect(result.windowSequence).toBe(1) - expect(result.windowStartedAt).toBe( - new Date(startedAt + restartMinute * 60_000).toISOString() - ) - expect(result.completedAt).toBe( - new Date(startedAt + (durationMinutes + restartMinute) * 60_000).toISOString() - ) - expect(result.sampleCount).toBe(durationMinutes + 1) - expect(result.continuityEvents.map((event) => event.tolerated)).toEqual([ - ...Array(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES).fill(true), - false - ]) - expect(result.continuityEvents.at(-1)!.failures).toEqual( - expect.arrayContaining([ - expect.objectContaining({ code: 'source_stale' }) - ]) - ) - expect(checkpoints).toContainEqual([1, durationMinutes]) - expect(result.frozenAt).toBeNull() - } - ) - - // Why: run 33944873727 on 2026-09-05 restarted at 04:46:09Z on a single - // 189-second cloud reading and then blew the 25-minute lineage cap, so a - // green fleet produced no verdict at all. One unread sample now continues the - // window; the sample is still checked against every threshold it can read. - it('carries a 15-minute window through a single stale cloud sample', async () => { - let now = startedAt - const state = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - const result = await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - const sample = healthySample(now) - if (now === startedAt + 10 * 60_000) { - sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = - signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1) - } - return sample - }, - persist: async () => {}, - checkpoint: async () => {} - }) - expect(result.windowSequence).toBe(0) - expect(result.windowStartedAt).toBe(new Date(startedAt).toISOString()) - expect(result.completedAt).toBe(new Date(startedAt + 15 * 60_000).toISOString()) - expect(result.sampleCount).toBe(16) - expect(result.frozenAt).toBeNull() - expect(result.continuityEvents).toEqual([{ - recordedAt: new Date(startedAt + 10 * 60_000).toISOString(), - windowSequence: 0, - tolerated: true, - failures: [expect.objectContaining({ - code: 'signal_stale', - source: 'cloud-monitoring', - signal: 'cloud_sql.lock_waits' - })] - }]) - expect(preDrainDryRunPassed(result)).toBe(true) - }) - - // Why: dry-run 35258662628 read a healthy fleet clean for 13 minutes, then one - // unreadable Cloud Monitoring sample restarted the window and the restart blew - // the lineage cap, so a green fleet produced no verdict. - it('carries a 15-minute window through a single collector failure', async () => { - let now = startedAt - const warnings: string[] = [] - const state = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - const result = await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - if (now === startedAt + 10 * 60_000) { - throw new Error('cloud monitoring read failed') - } - return healthySample(now) - }, - persist: async () => {}, - checkpoint: async () => {}, - warn: (message) => { - warnings.push(message) - } - }) - expect(result.windowSequence).toBe(0) - expect(result.windowStartedAt).toBe(new Date(startedAt).toISOString()) - expect(result.completedAt).toBe(new Date(startedAt + 15 * 60_000).toISOString()) - expect(result.sampleCount).toBe(16) - expect(result.frozenAt).toBeNull() - expect(result.failures).toEqual([]) - expect(result.continuityEvents).toEqual([{ - recordedAt: new Date(startedAt + 10 * 60_000).toISOString(), - windowSequence: 0, - tolerated: true, - failures: [{ code: 'collector_failed', source: 'cloud-monitoring' }] - }]) - expect(warnings).toEqual([ - 'incident monitor collector failed: cloud monitoring read failed' - ]) - expect(preDrainDryRunPassed(result)).toBe(true) - }) - - it('restarts the window after three consecutive collector failures', async () => { - let now = startedAt - let failures = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 - const state = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - const result = await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - if (failures > 0 && now >= startedAt + 10 * 60_000) { - failures-- - throw new Error('cloud monitoring read failed') - } - return healthySample(now) - }, - persist: async () => {}, - checkpoint: async () => {}, - warn: () => {} - }) - const restartMinute = 10 + INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 - expect(result.windowSequence).toBe(1) - expect(result.windowStartedAt).toBe( - new Date(startedAt + restartMinute * 60_000).toISOString() - ) - expect(result.completedAt).toBe( - new Date(startedAt + (restartMinute + 15) * 60_000).toISOString() - ) - expect(result.sampleCount).toBe(16) - expect(result.continuityEvents.map((event) => event.tolerated)).toEqual([ - ...Array(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES).fill(true), - false - ]) - expect(result.continuityEvents.at(-1)!.failures).toEqual([ - { code: 'collector_failed', source: 'cloud-monitoring' } - ]) - expect(result.frozenAt).toBeNull() - expect(preDrainDryRunPassed(result)).toBe(true) - }) - - it('still reaches a dry-run verdict after a restart on the last sample', async () => { - let now = startedAt - let failures = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 - const state = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - const result = await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - if (failures > 0 && now >= startedAt + 13 * 60_000) { - failures-- - throw new Error('cloud monitoring read failed') - } - return healthySample(now) - }, - persist: async () => {}, - checkpoint: async () => {}, - warn: () => {} - }) - expect(result.windowSequence).toBe(1) - expect(result.windowStartedAt).toBe(new Date(startedAt + 16 * 60_000).toISOString()) - expect(result.completedAt).toBe(new Date(startedAt + 31 * 60_000).toISOString()) - expect(result.sampleCount).toBe(16) - expect(result.frozenAt).toBeNull() - expect(result.failures).toEqual([]) - expect(preDrainDryRunPassed(result)).toBe(true) - }) - - it('gives a signal a fresh budget only after it reads fresh again', async () => { - let now = startedAt - const staleMinutes = new Set([3, 5, 6, 9, 10]) - const state = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - const result = await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - const sample = healthySample(now) - if (staleMinutes.has((now - startedAt) / 60_000)) { - sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = - signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1) - } - return sample - }, - persist: async () => {}, - checkpoint: async () => {} - }) - expect(result.windowSequence).toBe(0) - expect(result.continuityEvents).toHaveLength(staleMinutes.size) - expect(result.continuityEvents.every((event) => event.tolerated)).toBe(true) - expect(preDrainDryRunPassed(result)).toBe(true) - }) - - it('does not hand a resumed monitor a fresh tolerance budget', async () => { - let now = startedAt + 3 * 60_000 - const resumed = { - ...initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }), - windowStartedAt: new Date(startedAt).toISOString(), - lastSampleAt: new Date(startedAt + 2 * 60_000).toISOString(), - sampleCount: 3, - totalSampleCount: 3, - continuityEvents: Array.from( - { length: INCIDENT_FRESHNESS_TOLERANCE_SAMPLES }, - (_, index) => ({ - recordedAt: new Date(startedAt + (index + 1) * 60_000).toISOString(), - windowSequence: 0, - tolerated: true, - failures: [{ - code: 'signal_stale', - source: 'cloud-monitoring' as const, - signal: 'cloud_sql.lock_waits' - }] - }) - ) - } - const stop = new Error('stop after the resumed sample') - await expect(runIncidentMonitor(resumed, { - now: () => now, - wait: async () => { - throw stop - }, - collect: async () => { - const sample = healthySample(now) - sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = - signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1) - return sample - }, - persist: async (state) => { - expect(state.windowSequence).toBe(1) - expect(state.windowStartedAt).toBeNull() - expect(state.continuityEvents.at(-1)!.tolerated).toBe(false) - }, - checkpoint: async () => {} - })).rejects.toThrow(stop) - }) - - it('freezes on a threshold breach that arrives with a tolerated stale signal', async () => { - let now = startedAt - const state = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - const result = await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - const sample = healthySample(now) - if (now === startedAt + 2 * 60_000) { - sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = - signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1) - sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81, now) - } - return sample - }, - persist: async () => {}, - checkpoint: async () => {} - }) - expect(result.frozenAt).toBe(new Date(startedAt + 2 * 60_000).toISOString()) - expect(result.failures).toContainEqual(expect.objectContaining({ - code: 'threshold_max', - signal: 'cloud_sql.cpu' - })) - expect(preDrainDryRunPassed(result)).toBe(false) - }) - - it('resets at the next fresh sample after a runner gap', async () => { - let now = startedAt + 10 * 60_000 - const state = { - ...initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }), - lastSampleAt: new Date(startedAt).toISOString(), - sampleCount: 1, - totalSampleCount: 1 - } - const result = await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => healthySample(now), - persist: async () => {}, - checkpoint: async () => {} - }) - expect(result.windowSequence).toBe(1) - expect(result.windowStartedAt).toBe(new Date(startedAt + 10 * 60_000).toISOString()) - expect(result.continuityEvents[0]!.failures[0]!.code).toBe('monitor_gap') - expect(result.sampleCount).toBe(16) - }) - - it('fails a dry run after 35 total minutes of continuity resets', async () => { - let now = startedAt - const state = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - // Two restarts: the first on the window's last sample, the second far enough - // into the replacement window that no third window can finish in the lineage. - const staleMinutes = new Set([13, 14, 15, 24, 25, 26]) - const result = await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - if (staleMinutes.has((now - startedAt) / 60_000)) { - return healthySample( - now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1 - ) - } - return healthySample(now) - }, - persist: async () => {}, - checkpoint: async () => {} - }) - - expect(result.completedAt).toBe( - new Date(startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS).toISOString() - ) - expect(result.frozenAt).not.toBeNull() - expect(result.windowSequence).toBe(2) - expect(result.sampleCount).toBe(9) - expect(result.failures).toContainEqual({ - code: 'continuity_deadline_exceeded', - source: 'active-probe', - observed: INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS, - threshold: INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS - }) - expect(preDrainDryRunPassed(result)).toBe(false) - }) - - it('fails an overdue resumed dry run before collecting again', async () => { - const now = startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS + 1 - let collections = 0 - const state = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - const result = await runIncidentMonitor(state, { - now: () => now, - wait: async () => {}, - collect: async () => { - collections++ - return healthySample(now) - }, - persist: async () => {}, - checkpoint: async () => {} - }) - - expect(collections).toBe(0) - expect(result.failures).toContainEqual(expect.objectContaining({ - code: 'continuity_deadline_exceeded' - })) - }) - - it('requires a completed green 15-minute dry run', () => { - const state = { - ...initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }), - sampleCount: 16, - completedAt: new Date(startedAt + 15 * 60_000).toISOString() - } - expect(preDrainDryRunPassed(state)).toBe(true) - expect(preDrainDryRunPassed({ ...state, frozenAt: state.startedAt })).toBe(false) - expect(preDrainDryRunPassed({ - ...state, - completedAt: new Date(startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS + 1).toISOString() - })).toBe(false) - }) - - it('keeps poll starts on the configured cadence after collection time', async () => { - let now = startedAt - const starts: number[] = [] - const waits: number[] = [] - const state = initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - waits.push(ms) - now += ms - }, - collect: async () => { - starts.push(now) - now += 15_000 - return healthySample(now) - }, - persist: async () => {}, - checkpoint: async () => {} - }) - expect(starts.slice(0, 3)).toEqual([ - startedAt, - startedAt + 60_000, - startedAt + 120_000 - ]) - expect(waits[0]).toBe(45_000) - }) -}) - -// Why: the asia-east2 cells' readiness probe runs SELECT 1 against Cloud SQL in -// us-central1 behind a 2 s timeout, so a saturated pool makes the load balancer -// answer "no healthy upstream" for about 30 s. Every one of 39 pre-roll gates froze -// on that, and 7 of the last 14 froze on this signal alone. -describe('incident monitor cell probe tolerance', () => { - const dryRunState = () => - initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - - // Returns the finished state of a 15-minute dry-run whose cell probe reads - // health=0 and ready=0 on the sample indexes in `badSamples`. - const runWithCellProbeGaps = async (badSamples: Set) => { - let now = startedAt - let index = -1 - return await runIncidentMonitor(dryRunState(), { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - index++ - const sample = healthySample(now) - if (badSamples.has(index)) { - sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] = - signal(0, now) - sample.sources['active-probe']!.signals['cell.production-gce-c1.ready'] = - signal(0, now) - } - return sample - }, - persist: async () => {}, - checkpoint: async () => {} - }) - } - - it('passes a dry-run through a probe outage no longer than the tolerance', async () => { - const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - const result = await runWithCellProbeGaps( - new Set(Array.from({ length: tolerance }, (_, offset) => 3 + offset)) - ) - expect(result.frozenAt).toBeNull() - expect(result.failures).toEqual([]) - expect(preDrainDryRunPassed(result)).toBe(true) - // The blip is absorbed, not hidden: the sealed state still carries it. - expect(result.toleratedProbeEvents).toHaveLength(tolerance) - expect(result.toleratedProbeEvents[0]!.failures).toContainEqual( - expect.objectContaining({ - source: 'active-probe', - signal: 'cell.production-gce-c1.health', - observed: 0, - threshold: 1 - }) - ) - // A recovered probe hands back the full budget rather than a partial one. - expect(result.probeStreaks).toEqual({}) - }) - - it('freezes once a cell probe fails past the tolerance', async () => { - const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - const result = await runWithCellProbeGaps( - new Set(Array.from({ length: tolerance + 1 }, (_, offset) => 3 + offset)) - ) - expect(result.frozenAt).not.toBeNull() - expect(preDrainDryRunPassed(result)).toBe(false) - expect(result.failures).toContainEqual( - expect.objectContaining({ - source: 'active-probe', - signal: 'cell.production-gce-c1.health', - observed: 0, - threshold: 1 - }) - ) - // Only the samples past the tolerance freeze; the first two are still absorbed. - expect(result.toleratedProbeEvents).toHaveLength(tolerance) - }) - - it('does not accumulate a streak across a recovered sample', async () => { - const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - // Repeated single-sample outages, each separated by a healthy sample, never - // reach the tolerance however many times they recur. - const spaced = new Set([2, 4, 6, 8, 10]) - expect(spaced.size).toBeGreaterThan(tolerance) - const result = await runWithCellProbeGaps(spaced) - expect(result.frozenAt).toBeNull() - expect(preDrainDryRunPassed(result)).toBe(true) - }) - - // Why: health, ready and latency_ms all describe the same round trip, so the streak - // is keyed by cell. Keyed per signal, this cell holds every individual streak at one - // and never reaches the tolerance, yet it is unhealthy without a break from sample 2. - it('freezes on a cell that alternates between slow and unanswered', async () => { - let now = startedAt - let index = -1 - const result = await runIncidentMonitor(dryRunState(), { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - index++ - const sample = healthySample(now) - if (index < 2) return sample - // Two samples slow, then two samples down, repeating: never the same signal - // twice in a row beyond the tolerance, but never healthy either. - if (Math.floor((index - 2) / 2) % 2 === 0) { - sample.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] = - signal(9_000, now) - } else { - sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] = - signal(0, now) - sample.sources['active-probe']!.signals['cell.production-gce-c1.ready'] = - signal(0, now) - } - return sample - }, - persist: async () => {}, - checkpoint: async () => {} - }) - expect(result.frozenAt).not.toBeNull() - expect(preDrainDryRunPassed(result)).toBe(false) - }) - - // Why: the streak lives in the state file, so a resumed run must not hand a cell - // that was already failing a fresh budget. - it('freezes immediately when a resumed state carries a full streak', async () => { - let now = startedAt - const resumed = { - ...dryRunState(), - lastSampleAt: new Date(startedAt).toISOString(), - probeStreaks: { - 'active-probe/cell.production-gce-c1': - INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - } - } - const result = await runIncidentMonitor(resumed, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - const sample = healthySample(now) - sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] = - signal(0, now) - return sample - }, - persist: async () => {}, - checkpoint: async () => {} - }) - expect(result.frozenAt).toBe(new Date(startedAt).toISOString()) - expect(result.toleratedProbeEvents).toEqual([]) - expect(result.failures).toContainEqual( - expect.objectContaining({ - source: 'active-probe', - signal: 'cell.production-gce-c1.health' - }) - ) - }) - - it('gives the director and auth probes no tolerance', async () => { - let now = startedAt - let index = -1 - const result = await runIncidentMonitor(dryRunState(), { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - index++ - const sample = healthySample(now) - if (index === 3) { - sample.sources['active-probe']!.signals['director.health'] = signal(0, now) - } - return sample - }, - persist: async () => {}, - checkpoint: async () => {} - }) - expect(result.frozenAt).not.toBeNull() - expect(result.toleratedProbeEvents).toEqual([]) - expect(result.failures).toContainEqual( - expect.objectContaining({ source: 'active-probe', signal: 'director.health' }) - ) - }) -}) - -// Why: Cloud Run replaces director instances in place, so the count leaves the -// [5, 6] band for about one sample roughly twice a day, and a deploy overlap -// raises it the same way. Neither is an unhealthy fleet, and freezing on it -// blocks the roll that fixes the measured condition. -describe('incident monitor director instance tolerance', () => { - const dryRunState = () => - initialIncidentMonitorState({ - incidentId: 'incident-1', - environment: 'production', - expectedSelector: selector, - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - startedAt: new Date(startedAt).toISOString(), - durationMinutes: 15, - intervalMs: 60_000 - }) - - // Finished state of a 15-minute dry run whose director instance count reads - // `counts[index]` on the sample indexes that map has, and 5 everywhere else. - const runWithInstanceCounts = async ( - counts: Map, - state = dryRunState() - ) => { - let now = startedAt - let index = -1 - return await runIncidentMonitor(state, { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - index++ - const sample = healthySample(now) - const count = counts.get(index) - if (count !== undefined) { - sample.sources['cloud-monitoring']!.signals['director.instances'] = - signal(count, now) - } - return sample - }, - persist: async () => {}, - checkpoint: async () => {} - }) - } - - it('passes a dry run through an instance dip no longer than the tolerance', async () => { - const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - const result = await runWithInstanceCounts( - new Map(Array.from({ length: tolerance }, (_, offset) => [3 + offset, 2])) - ) - expect(result.frozenAt).toBeNull() - expect(result.failures).toEqual([]) - expect(preDrainDryRunPassed(result)).toBe(true) - // Absorbed, not hidden: the sealed state still carries the dip. - expect(result.toleratedProbeEvents).toHaveLength(tolerance) - expect(result.toleratedProbeEvents[0]!.failures).toContainEqual( - expect.objectContaining({ - code: 'threshold_min', - source: 'cloud-monitoring', - signal: 'director.instances', - observed: 2, - threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMin - }) - ) - expect(result.probeStreaks).toEqual({}) - }) - - it('passes a dry run through a deploy-overlap overshoot within the tolerance', async () => { - const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - const result = await runWithInstanceCounts( - new Map(Array.from({ length: tolerance }, (_, offset) => [3 + offset, 9])) - ) - expect(result.frozenAt).toBeNull() - expect(preDrainDryRunPassed(result)).toBe(true) - expect(result.toleratedProbeEvents[0]!.failures).toContainEqual( - expect.objectContaining({ - code: 'threshold_max', - signal: 'director.instances', - observed: 9, - threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMax - }) - ) - }) - - it('freezes once the instance count stays out of band past the tolerance', async () => { - const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - const result = await runWithInstanceCounts( - new Map(Array.from({ length: tolerance + 1 }, (_, offset) => [3 + offset, 2])) - ) - expect(result.frozenAt).not.toBeNull() - expect(preDrainDryRunPassed(result)).toBe(false) - expect(result.failures).toContainEqual( - expect.objectContaining({ - code: 'threshold_min', - source: 'cloud-monitoring', - signal: 'director.instances', - observed: 2 - }) - ) - expect(result.toleratedProbeEvents).toHaveLength(tolerance) - }) - - it('does not accumulate a streak across a recovered sample', async () => { - const spaced = new Map([2, 4, 6, 8, 10].map((index) => [index, 2] as const)) - expect(spaced.size).toBeGreaterThan( - INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - ) - const result = await runWithInstanceCounts(new Map(spaced)) - expect(result.frozenAt).toBeNull() - expect(preDrainDryRunPassed(result)).toBe(true) - }) - - // Why min and max share one streak: a count that alternates above and below - // the band would otherwise hold each streak at one and never freeze, yet the - // director is never at its configured size. - it('freezes on a count that alternates above and below the band', async () => { - const counts = new Map() - for (let index = 2; index < 12; index += 1) { - counts.set(index, index % 2 === 0 ? 2 : 9) - } - const result = await runWithInstanceCounts(counts) - expect(result.frozenAt).not.toBeNull() - expect(preDrainDryRunPassed(result)).toBe(false) - }) - - // Why: the streak lives in the state file, so a resumed run must not hand a - // director that was already out of band a fresh budget. - it('freezes immediately when a resumed state carries a full streak', async () => { - const result = await runWithInstanceCounts(new Map([[0, 2]]), { - ...dryRunState(), - lastSampleAt: new Date(startedAt).toISOString(), - probeStreaks: { - 'cloud-monitoring/director.instances': - INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - } - }) - expect(result.frozenAt).toBe(new Date(startedAt).toISOString()) - expect(result.toleratedProbeEvents).toEqual([]) - expect(result.failures).toContainEqual( - expect.objectContaining({ signal: 'director.instances' }) - ) - }) - - it('keeps every other cloud-monitoring signal at zero tolerance', async () => { - let now = startedAt - let index = -1 - const result = await runIncidentMonitor(dryRunState(), { - now: () => now, - wait: async (ms) => { - now += ms - }, - collect: async () => { - index++ - const sample = healthySample(now) - if (index === 3) { - sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.99, now) - } - return sample - }, - persist: async () => {}, - checkpoint: async () => {} - }) - expect(result.frozenAt).not.toBeNull() - expect(result.toleratedProbeEvents).toEqual([]) - expect(result.failures).toContainEqual( - expect.objectContaining({ source: 'cloud-monitoring', signal: 'cloud_sql.cpu' }) - ) - }) -}) - diff --git a/cloud/apps/relay-ops/src/incident-monitor.ts b/cloud/apps/relay-ops/src/incident-monitor.ts deleted file mode 100644 index bcb3c30bf09..00000000000 --- a/cloud/apps/relay-ops/src/incident-monitor.ts +++ /dev/null @@ -1,1111 +0,0 @@ -import type { RelayOpsRegion } from './environment-config.js' -import { - exactAdmissionSelector, - type AdmissionSelector, - type AdmissionState -} from './incident-selector.js' - -export const INCIDENT_MONITOR_THRESHOLDS = { - activeProbeMaxAgeMs: 60_000, - // Why: Cloud Monitoring publishes on Google's clock, not ours. Per the metric - // list read 2026-09-05, Cloud Run instance_count / cpu / memory / - // max_request_concurrencies / request_count are "Sampled every 60 seconds. - // After sampling, data is not visible for up to 120 seconds" (60+120=180 s), - // and Cloud SQL cpu / memory / num_backends / backends_in_wait / - // deadlock_count say "up to 165 seconds" (60+165=225 s). Window-sum signals - // age differently: observedAt is the newest point in the 5-minute query - // window, so a label series that stops emitting reads as 300 s old while its - // summed value is still complete. 330 s clears the worst of the three (the - // 300 s query window) plus ~30 s of collect-to-evaluate latency. The old - // 180 s bar restarted healthy 15-minute windows at 181 s, 189 s and 255 s on - // 2026-09-04/05, once burning the whole 25-minute lineage with no verdict. - cloudDataMaxAgeMs: 330_000, - // Why: the director admin API answers live on our own request, so hold its - // freshness bar where it sat while it shared cloudDataMaxAgeMs. - directorAdminMaxAgeMs: 180_000, - // Why: how long a nonzero backends-in-wait point is carried before it reads as - // zero. Held at the pre-2026-09-05 cloud bar: carrying it for the full - // cloudDataMaxAgeMs would hand the evaluator a point older than its own - // freshness bar as soon as collection latency is added. - cloudLockWaitCarryMs: 180_000, - relayLogMaxAgeMs: 180_000, - heartbeatMaxAgeMs: 45_000, - endpointLatencyMs: 2_000, - // Why: a cell's /ready fetches the auth JWKS and runs SELECT 1 against Cloud SQL, - // both in us-central1, so from the US runner asia-east2 cells measure p50 0.88 s / - // max 2.7 s against 0.08-0.5 s for us-central1. The flat 2 000 bar froze three - // healthy 15-minute gates on 2026-09-05 (c27 at 2568/2668/2685 ms); hard faults - // are still caught by the .health/.ready equal-1 checks and the 8 s fetch timeout. - cellEndpointLatencyMs: { - 'us-central1': 2_000, - 'asia-east2': 4_000 - } as const satisfies Record, - cloudSqlCpuUtilization: 0.8, - cloudSqlMemoryUtilization: 0.9, - // Why: 320, recalibrated 2026-09-17 from 250. The auth instance sums seven - // databases, and its steady load has grown past the 2026-08-26 measurement the - // old bar came from. Measured latest-sum over the 24 h to 2026-09-17, aligned - // per minute exactly as this signal reads it: p50 118 / p90 165 / p95 212 / - // p99 262 / max 282. 250 was under the observed max, so 1.95% of minutes and - // 21.8% of 15-minute pre-drain gates froze on ordinary load. 320 clears every - // measured healthy minute with 13% of headroom above the peak and still fires - // at 65% of the 490 connections the budget work (#21165) treats as usable out - // of max_connections 500, so exhaustion-class growth is caught with 170 - // connections still in hand. The retry signals below discriminate incident-class - // contention. Re-tighten when the auth connection model lands (#21165). - cloudSqlBackends: 320, - // Bound the observed recovery load; deadlocks remain zero-tolerance. - cloudSqlLockWaits: 20, - cloudSqlDeadlocks: 0, - // Why: pool amplitude cannot discriminate the 2026-08-23 incident. Healthy - // fleet-wide bursts reach 43 waiters / 2.03s waits several times an hour, - // and a cell roll's reconnect surge peaks at 676 waiters, while the real - // incident peaked at 356 waiters and never crossed 2.5s (waits cap ~2s - // structurally). The old bars of 30/1000 froze pre-drain gates on baseline - // noise (~17% per 15-minute window). Incident-class contention is caught by - // the retry signals below at ~10x separation; these bars now fence only - // genuinely unbounded queueing, which grows past both. - relayPoolWaiting: 800, - relayPoolWaitMs: 2_500, - // Why: successful lock retries are the contention machinery working, not harm. - // Recalibrated 2026-09-04 from 300, which was set 2026-08-26 when healthy bursts - // reached 234/5min. The global relay_cells FOR UPDATE lock has since become the - // fleet's steady state: measured fleet-wide (director + cells, summed per five - // minutes) 2026-09-03T05Z..2026-09-04T05Z p50 430 / p90 924 / p99 1320 / max - // 1504, with 55% of windows over 300 and only 22% of 15-minute gates clean, so - // the bar blocked the very cell roll that carries the 500 ms lock wait (#18521) - // and the beginProof crash guard to the cells. The 2026-08-23 lock incident on - // this same metric peaked at 1510 in one window and 646 in the next, so it is - // not separable from today's contention by retries alone; it is caught by - // relayPostgresRetryExhausted (467 at the peak vs a 300 bar), director - // concurrency, and the pool bars. 2000 passes every healthy 15-minute window - // measured in the last 24 h and still fences unbounded growth. Re-tighten once - // the fleet is on the 500 ms lock wait and the baseline is re-measured. - relayPostgresRetries: 2000, - // Why: 300 per five minutes, recalibrated 2026-09-04 from a bar of zero that no - // production window has cleared since #18521 shipped to the director. That - // change cut the request-path cell-inventory wait from the 1 s pool lock_timeout - // to 500 ms, so a contended waiter now fails fast (one /v1/assign 503 with - // Retry-After, which the client retries) instead of succeeding slowly, and the - // exhaustion count became a steady-state contention rate rather than an - // anomaly. Measured fleet-wide (director + cells) per five minutes over - // 2026-09-03T03Z..2026-09-04T02Z: every one of 236 windows was non-zero; - // quiet hours p50 2 / max 36; pre-#18521 daytime p50 10 / p90 25 / max 87; - // post-#18521 p50 42 / p90 147 / max 220. The 2026-08-23 lock incident peaked - // at 467. 300 clears every measured healthy window and still sits below the - // incident shape; retries above fence only unbounded growth. - // User-facing /v1/assign 503 share did not move with #18521 (13.9% old image - // vs 12.3% new, same evening), so exhaustion is not a proxy for user harm. - relayPostgresRetryExhausted: 300, - // Why: public admission is a per-instance semaphore, so fleet assignment capacity is - // concurrency x instances. A floor of 1 let the 2026-08-04 collapse from five instances - // to two pass unnoticed, which is the exact failure this monitor exists to catch. Keep in - // step with relay_min_instances in infra/terraform/environments/production.tfvars. - directorInstancesMin: 5, - // Five serving instances plus one warm scale-to-zero rollback during recovery. - directorInstancesMax: 6, - directorCpuUtilization: 0.8, - directorMemoryUtilization: 0.8, - directorConcurrency: 64, - // Why: 15, recalibrated 2026-09-17 from 3. This counts non-503 5xx answers from - // the director over the rolling five-minute query window; 503s are excluded - // because they are the documented back-pressure answer a client retries. - // Measured over the 24 h to 2026-09-17 (272 non-503 5xx against 71 941 503s): - // p50 0 / p90 3 / p95 5 / p99 9 / max 52. A bar of 3 sits at the p90, so 9.2% - // of windows and 29.0% of 15-minute pre-drain gates froze on the chronic 500 - // bursts that /v1/assign, /v1/regions and /v1/resolve emit alongside the - // recurring Cloud SQL stall. 15 clears the chronic p99 with margin and drops - // the baseline gate-freeze rate to 1.5%, while leaving the exceptional 20-52 - // bursts detectable. A director that is actually broken answers 5xx on a large - // share of its traffic: it serves ~50 requests a minute in 503s alone, so a - // real fault lands in the hundreds per window, an order of magnitude clear of - // this bar. - directorErrors: 15, - // Why: 2, raised 2026-10-07 from 0. Auth answers ~58k requests per five minutes - // (24 h to 2026-10-07: min 44k) and logged two 5xx in that day, so a single 500 on - // the desktop sign-in callback froze the c13 and c28 waves with auth healthy. A real - // auth fault fails hundreds per window; the ratio bar keeps a near-idle auth strict. - authErrors: 2, - authErrorRatio: 0.001, - // Why: 800 exceeded the 600 hard cap, so this could never trigger on a capped cell. 500 is - // the ordinary admission limit a cell actually stops at (600 cap - 100 control-rebind reserve). - cellConnections: 500, - cellQueuedBytes: 48 * 1024 * 1024, - migrationBlocked: 0, - // Why: one sample is one HTTP round trip from one GitHub runner to one cell, so - // a single bad reading is evidence about that round trip, not about the fleet. - // The asia-east2 cells' readiness probe runs SELECT 1 against Cloud SQL in - // us-central1 over a 176 ms round trip behind a 2 s statement timeout, so a - // saturated pool makes the load balancer answer "no healthy upstream" for about - // 30 s. That answer is an HTTP 503, not a transport failure, so provenance - // cannot separate it from a cell that genuinely serves health=0 -- persistence - // can. At the 60 s sample interval a 30 s outage shows up in one sample and at - // worst two, so a cell's probe must fail more than this many consecutive samples - // before it freezes the run. The streak is per cell, not per signal, so a cell - // that alternates between slow and unanswered still accumulates one. This applies - // to per-cell probes and the director instance count; the director and auth health - // probes stay at zero tolerance. - cellProbeToleranceSamples: 2 -} as const - -export const INCIDENT_CHECKPOINT_MINUTES = [0, 5, 15, 30, 45, 60, 75, 90] as const -// Why: 35 minutes, raised 2026-09-17 from 25. A 15-minute window plus one -// restart must fit: a continuity reset on the window's last sample restarts at -// minute 16 and finishes at 31. Under 25 a reset past minute 9 cost the whole -// verdict, which is what run 35258662628 hit on a healthy fleet. -export const INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS = 35 * 60_000 - -export type IncidentSourceName = - | 'active-probe' - | 'cloud-monitoring' - | 'relay-logs' - | 'director-admin' - -export type IncidentMigrationPolicy = - | 'strict' - | 'recover-forward' - | 'capacity-transition' - -export type IncidentSignal = { - value: number - observedAt: string -} - -export type IncidentSource = { - observedAt: string - signals: Record -} - -export type IncidentCellExpectation = { - cellId: string - region: RelayOpsRegion - runtimeKnown: boolean - powered: boolean - expectedAdmissionState: AdmissionState - // Step 5 reserve mode: the director's cell-status and migration reads refuse the cell. - reserve?: boolean -} - -export type IncidentSample = { - collectedAt: string - selector: AdmissionSelector - expectedSelector: AdmissionSelector - sources: Partial> - cells: IncidentCellExpectation[] -} - -export type IncidentFailure = { - code: string - source: IncidentSourceName - signal?: string - observed?: number - threshold?: number -} - -// Keep the source/code prefix other tooling matches on, then name the signal and -// its numbers so a frozen wave is attributable without re-reading the sample. -export function describeIncidentFailure(failure: IncidentFailure): string { - const detail = [ - failure.signal, - failure.observed === undefined ? null : `observed=${failure.observed}`, - failure.threshold === undefined ? null : `threshold=${failure.threshold}` - ].filter((part): part is string => part !== null && part !== undefined) - return [`${failure.source}/${failure.code}`, ...detail].join(' ') -} - -export type IncidentEvaluation = { - status: 'green' | 'freeze' - evaluatedAt: string - failures: IncidentFailure[] -} - -export type IncidentCheckpoint = { - schemaVersion: 4 - incidentId: string - environment: 'production' | 'staging' - expectedSelector: AdmissionSelector - preDrainDryRun: boolean - migrationPolicy: IncidentMigrationPolicy - recoverySourceCellId: string | null - capacityCellId: string | null - windowSequence: number - windowStartedAt: string - checkpointMinute: number - scheduledAt: string - recordedAt: string - status: 'green' | 'freeze' - frozenAt: string | null - sampleCount: number - failures: IncidentFailure[] - thresholds: typeof INCIDENT_MONITOR_THRESHOLDS -} - -export type IncidentMonitorState = { - schemaVersion: 4 - incidentId: string - environment: 'production' | 'staging' - expectedSelector: AdmissionSelector - preDrainDryRun: boolean - migrationPolicy: IncidentMigrationPolicy - recoverySourceCellId: string | null - capacityCellId: string | null - startedAt: string - windowStartedAt: string | null - windowSequence: number - durationMinutes: number - intervalMs: number - nextCheckpointIndex: number - sampleCount: number - totalSampleCount: number - lastSampleAt: string | null - continuityEvents: { - recordedAt: string - windowSequence: number - tolerated: boolean - failures: IncidentFailure[] - }[] - frozenAt: string | null - failures: IncidentFailure[] - // Consecutive samples each tolerated reading has currently been failing for, - // keyed by cell so its health, ready and latency readings share one streak, and - // by signal for the director instance count. - probeStreaks: Record - // Cell-probe breaches absorbed by the tolerance, kept so a green artifact still - // shows what the gate chose not to freeze on. - toleratedProbeEvents: { - recordedAt: string - windowSequence: number - failures: IncidentFailure[] - }[] - completedAt: string | null -} - -type NumericRule = { - source: IncidentSourceName - signal: string - comparison: 'max' | 'min' | 'equal' - threshold: number -} - -const NUMERIC_RULES: NumericRule[] = [ - { source: 'active-probe', signal: 'director.health', comparison: 'equal', threshold: 1 }, - { source: 'active-probe', signal: 'director.ready', comparison: 'equal', threshold: 1 }, - { - source: 'active-probe', - signal: 'director.latency_ms', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs - }, - { source: 'active-probe', signal: 'auth.health', comparison: 'equal', threshold: 1 }, - { - source: 'active-probe', - signal: 'auth.latency_ms', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs - }, - { - source: 'cloud-monitoring', - signal: 'cloud_sql.cpu', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlCpuUtilization - }, - { - source: 'cloud-monitoring', - signal: 'cloud_sql.memory', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlMemoryUtilization - }, - { - source: 'cloud-monitoring', - signal: 'cloud_sql.backends', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlBackends - }, - { - source: 'cloud-monitoring', - signal: 'director.instances', - comparison: 'min', - threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMin - }, - { - source: 'cloud-monitoring', - signal: 'director.instances', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMax - }, - { - source: 'cloud-monitoring', - signal: 'director.cpu', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.directorCpuUtilization - }, - { - source: 'cloud-monitoring', - signal: 'director.memory', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.directorMemoryUtilization - }, - { - source: 'cloud-monitoring', - signal: 'director.concurrency', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.directorConcurrency - }, - { - source: 'cloud-monitoring', - signal: 'director.errors', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.directorErrors - }, - { - source: 'cloud-monitoring', - signal: 'auth.errors', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.authErrors - }, - { - source: 'cloud-monitoring', - signal: 'cloud_sql.lock_waits', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlLockWaits - }, - { - source: 'cloud-monitoring', - signal: 'cloud_sql.deadlocks', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlDeadlocks - }, - { - source: 'relay-logs', - signal: 'relay.pool_waiting', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.relayPoolWaiting - }, - { - source: 'relay-logs', - signal: 'relay.pool_wait_ms', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.relayPoolWaitMs - }, - { - source: 'relay-logs', - signal: 'relay.postgres_retries', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetries - }, - { - source: 'relay-logs', - signal: 'relay.postgres_retry_exhausted', - comparison: 'max', - threshold: INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetryExhausted - } -] - -const SOURCE_MAX_AGE: Record = { - 'active-probe': INCIDENT_MONITOR_THRESHOLDS.activeProbeMaxAgeMs, - 'cloud-monitoring': INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs, - 'relay-logs': INCIDENT_MONITOR_THRESHOLDS.relayLogMaxAgeMs, - 'director-admin': INCIDENT_MONITOR_THRESHOLDS.directorAdminMaxAgeMs -} - -function ageMs(timestamp: string, nowMs: number): number { - const parsed = Date.parse(timestamp) - return Number.isFinite(parsed) ? nowMs - parsed : Number.POSITIVE_INFINITY -} - -function addMissingSignal( - failures: IncidentFailure[], - source: IncidentSourceName, - signal: string -): void { - failures.push({ code: 'signal_missing', source, signal }) -} - -function checkRule( - failures: IncidentFailure[], - source: IncidentSourceName, - signals: Record, - rule: NumericRule -): void { - const signal = signals[rule.signal] - if (!signal) return addMissingSignal(failures, source, rule.signal) - const failed = - (rule.comparison === 'max' && signal.value > rule.threshold) || - (rule.comparison === 'min' && signal.value < rule.threshold) || - (rule.comparison === 'equal' && signal.value !== rule.threshold) - if (failed) { - failures.push({ - code: `threshold_${rule.comparison}`, - source, - signal: rule.signal, - observed: signal.value, - threshold: rule.threshold - }) - } -} - -// Why: a few errors are noise only against real traffic; with almost no requests they are the outage. -function checkAuthErrorRatio( - failures: IncidentFailure[], - signals: Record -): void { - const errors = signals['auth.errors']?.value ?? 0 - if (errors === 0) return - const requests = signals['auth.requests'] - if (!requests) return addMissingSignal(failures, 'cloud-monitoring', 'auth.requests') - const ratio = requests.value > 0 ? errors / requests.value : Number.POSITIVE_INFINITY - if (ratio > INCIDENT_MONITOR_THRESHOLDS.authErrorRatio) { - failures.push({ - code: 'threshold_max', - source: 'cloud-monitoring', - signal: 'auth.error_ratio', - observed: ratio, - threshold: INCIDENT_MONITOR_THRESHOLDS.authErrorRatio - }) - } -} - -function checkCell( - failures: IncidentFailure[], - sample: IncidentSample, - cell: IncidentCellExpectation, - migrationPolicy: IncidentMigrationPolicy, - recoverySourceCellId: string | null, - capacityCellId: string | null -): void { - const probe = sample.sources['active-probe']?.signals - const relay = sample.sources['relay-logs']?.signals - const admin = sample.sources['director-admin']?.signals - if (!cell.runtimeKnown) { - failures.push({ - code: 'runtime_power_unknown', - source: 'cloud-monitoring', - signal: `cell.${cell.cellId}.powered` - }) - } - if ( - cell.runtimeKnown && - cell.expectedAdmissionState !== 'existing-only' && - !cell.powered - ) { - failures.push({ - code: 'expected_admission_without_runtime', - source: 'director-admin', - signal: `cell.${cell.cellId}.powered`, - observed: 0, - threshold: 1 - }) - } - const checks = [ - ['active-probe', probe, `cell.${cell.cellId}.health`, cell.powered ? 1 : 0, 'equal'], - ['active-probe', probe, `cell.${cell.cellId}.ready`, cell.powered ? 1 : 0, 'equal'], - [ - 'active-probe', - probe, - `cell.${cell.cellId}.latency_ms`, - INCIDENT_MONITOR_THRESHOLDS.cellEndpointLatencyMs[cell.region], - 'max' - ], - [ - 'director-admin', - admin, - `cell.${cell.cellId}.admission_state`, - ['existing-only', 'migration-only', 'general'].indexOf( - cell.expectedAdmissionState - ), - 'equal' - ], - [ - 'director-admin', - admin, - `cell.${cell.cellId}.heartbeat_fresh`, - 1, - 'equal' - ], - [ - 'director-admin', - admin, - `cell.${cell.cellId}.heartbeat_age_ms`, - INCIDENT_MONITOR_THRESHOLDS.heartbeatMaxAgeMs, - 'max' - ], - [ - 'director-admin', - admin, - `cell.${cell.cellId}.migration_blocked`, - INCIDENT_MONITOR_THRESHOLDS.migrationBlocked, - 'max' - ], - [ - 'director-admin', - admin, - `cell.${cell.cellId}.migration_target_inactive`, - INCIDENT_MONITOR_THRESHOLDS.migrationBlocked, - 'max' - ], - [ - 'relay-logs', - relay, - `cell.${cell.cellId}.connections`, - (admin?.[`cell.${cell.cellId}.connection_hard_cap`]?.value ?? - INCIDENT_MONITOR_THRESHOLDS.cellConnections + 1) - 1, - 'max' - ], - [ - 'relay-logs', - relay, - `cell.${cell.cellId}.queued_bytes`, - INCIDENT_MONITOR_THRESHOLDS.cellQueuedBytes, - 'max' - ] - ] as const - for (const [source, signals, signalName, threshold, comparison] of checks) { - // Probes, logs and selector still judge a reserve cell; the reserve watch reads its runtime. - if ( - cell.reserve && - ['heartbeat_fresh', 'heartbeat_age_ms', 'migration_blocked', 'migration_target_inactive'].some( - (suffix) => signalName.endsWith(suffix) - ) - ) { - continue - } - if ( - migrationPolicy === 'recover-forward' && - cell.cellId === recoverySourceCellId && - signalName.endsWith('.migration_target_inactive') - ) { - if (!signals?.[signalName]) addMissingSignal(failures, source, signalName) - continue - } - if ( - migrationPolicy === 'capacity-transition' && - capacityCellId !== null && - cell.cellId !== capacityCellId && - cell.expectedAdmissionState === 'existing-only' && - signalName.endsWith('.migration_target_inactive') - ) { - if (!signals?.[signalName]) addMissingSignal(failures, source, signalName) - continue - } - if ( - cell.expectedAdmissionState === 'existing-only' && - signalName.endsWith('.connections') - ) { - continue - } - if ( - !cell.powered && - [ - 'latency_ms', - 'heartbeat_fresh', - 'heartbeat_age_ms', - 'connections', - 'queued_bytes' - ].some((suffix) => signalName.endsWith(suffix)) - ) { - continue - } - if (!signals?.[signalName]) { - addMissingSignal(failures, source, signalName) - continue - } - const value = signals[signalName].value - const failed = comparison === 'equal' ? value !== threshold : value > threshold - if (failed) { - failures.push({ - code: `threshold_${comparison}`, - source, - signal: signalName, - observed: value, - threshold - }) - } - } -} - -export function evaluateIncidentSample( - sample: IncidentSample, - nowMs = Date.now(), - migrationPolicy: IncidentMigrationPolicy = 'strict', - recoverySourceCellId: string | null = null, - capacityCellId: string | null = null -): IncidentEvaluation { - const failures: IncidentFailure[] = [] - if (!exactAdmissionSelector(sample.selector, sample.expectedSelector)) { - failures.push({ - code: 'selector_mismatch', - source: 'director-admin', - signal: 'selector.generation', - observed: sample.selector.generation, - threshold: sample.expectedSelector.generation - }) - } - for (const [sourceName, maxAge] of Object.entries(SOURCE_MAX_AGE) as [ - IncidentSourceName, - number - ][]) { - const source = sample.sources[sourceName] - if (!source) { - failures.push({ code: 'source_missing', source: sourceName }) - continue - } - if (ageMs(source.observedAt, nowMs) < 0 || ageMs(source.observedAt, nowMs) > maxAge) { - failures.push({ - code: 'source_stale', - source: sourceName, - observed: ageMs(source.observedAt, nowMs), - threshold: maxAge - }) - } - for (const [signalName, signal] of Object.entries(source.signals)) { - if (ageMs(signal.observedAt, nowMs) < 0 || ageMs(signal.observedAt, nowMs) > maxAge) { - failures.push({ - code: 'signal_stale', - source: sourceName, - signal: signalName, - observed: ageMs(signal.observedAt, nowMs), - threshold: maxAge - }) - } - } - } - for (const rule of NUMERIC_RULES) { - const source = sample.sources[rule.source] - if (source) checkRule(failures, rule.source, source.signals, rule) - } - const monitoring = sample.sources['cloud-monitoring'] - if (monitoring) checkAuthErrorRatio(failures, monitoring.signals) - for (const cell of sample.cells) { - checkCell( - failures, - sample, - cell, - migrationPolicy, - recoverySourceCellId, - capacityCellId - ) - } - return { - status: failures.length === 0 ? 'green' : 'freeze', - evaluatedAt: new Date(nowMs).toISOString(), - failures - } -} - -export function initialIncidentMonitorState(input: { - incidentId: string - environment: 'production' | 'staging' - expectedSelector: AdmissionSelector - preDrainDryRun: boolean - migrationPolicy: IncidentMigrationPolicy - recoverySourceCellId: string | null - capacityCellId: string | null - startedAt: string - durationMinutes: number - intervalMs: number -}): IncidentMonitorState { - if (input.intervalMs < 1_000 || input.intervalMs > 60_000) { - throw new Error('incident monitor interval must be between 1 and 60 seconds') - } - if (input.durationMinutes < 15 || input.durationMinutes > 90) { - throw new Error('incident monitor duration must be between 15 and 90 minutes') - } - return { - schemaVersion: 4, - ...input, - windowStartedAt: input.startedAt, - windowSequence: 0, - nextCheckpointIndex: 0, - sampleCount: 0, - totalSampleCount: 0, - lastSampleAt: null, - continuityEvents: [], - frozenAt: null, - failures: [], - probeStreaks: {}, - toleratedProbeEvents: [], - completedAt: null - } -} - -export type IncidentMonitorDependencies = { - now(): number - wait(ms: number): Promise - collect(): Promise - persist(state: IncidentMonitorState): Promise - checkpoint(summary: IncidentCheckpoint): Promise - warn?(message: string): void -} - -function checkpointMinutes(durationMinutes: number): number[] { - return INCIDENT_CHECKPOINT_MINUTES.filter((minute) => minute <= durationMinutes) -} - -// Freshness-only failures: we could not read a signal this sample. Distinct from -// collector_failed / monitor_gap, where the whole sample is absent. -export const FRESHNESS_FAILURE_CODES = new Set([ - 'signal_missing', - 'signal_stale', - 'source_missing', - 'source_stale' -]) - -const CONTINUITY_FAILURE_CODES = new Set([ - 'collector_failed', - 'monitor_gap', - ...FRESHNESS_FAILURE_CODES -]) - -// A whole sample we could not read gets the same consecutive-sample budget as an -// unread signal, for the same reason: one failed collector round trip is evidence -// about that round trip, not about the fleet. `monitor_gap` is excluded because it -// means the run itself stopped sampling, so the window genuinely has a hole. -const TOLERABLE_CONTINUITY_FAILURE_CODES = new Set([ - 'collector_failed', - ...FRESHNESS_FAILURE_CODES -]) - -// Why: Cloud Monitoring overshoots its own publish bar, and one unread sample is -// not evidence of an unhealthy fleet. Under the 25-minute lineage cap in force -// then, a restart past minute 10 cost the entire verdict, so a healthy fleet -// produced none on 2026-09-05. A signal may miss this many consecutive samples before the window -// restarts; the sample is still evaluated against every threshold it can read, -// and a threshold breach still freezes the run outright. -export const INCIDENT_FRESHNESS_TOLERANCE_SAMPLES = 2 - -function freshnessKey(failure: IncidentFailure): string { - return `${failure.source}/${failure.signal ?? '*'}` -} - -// The streak key for a per-cell active-probe reading, or null if the failure is not -// one. A cell probe is a single HTTP round trip and so is subject to -// cellProbeToleranceSamples; director and auth probes return null on purpose, -// because they are the single points of failure this gate exists to catch. -// -// Why the key is the cell and not the signal: health, ready and latency_ms all -// describe the same round trip. Keyed per signal, a cell that alternates between -// answering slowly and not answering at all holds every individual streak at one and -// never reaches the tolerance, so a continuously unhealthy cell passes the gate. -export function cellProbeStreakKey(failure: IncidentFailure): string | null { - const signal = failure.signal - if (failure.source !== 'active-probe' || signal === undefined) return null - if (!signal.startsWith('cell.')) return null - const lastDot = signal.lastIndexOf('.') - if (lastDot < 'cell.'.length) return null - return `${failure.source}/${signal.slice(0, lastDot)}` -} - -// Cloud Run replaces director instances in place rather than holding the count, -// so the reading leaves [min, max] for about one sample roughly twice a day, and a -// deploy that briefly serves two revisions raises it the same way. Neither is an -// unhealthy fleet, and on 2026-09-17 this was one of the signals freezing the -// pre-drain gate on a condition the roll exists to fix. Min and max share one -// streak on purpose: a count that alternates above and below the band would -// otherwise hold each individual streak at one and never reach the tolerance. -export function directorInstancesStreakKey(failure: IncidentFailure): string | null { - if (failure.source !== 'cloud-monitoring' || failure.signal !== 'director.instances') { - return null - } - return `${failure.source}/${failure.signal}` -} - -// The streak key for any reading subject to cellProbeToleranceSamples, or null -// for a reading that freezes the run on its first bad sample. -export function toleratedStreakKey(failure: IncidentFailure): string | null { - return cellProbeStreakKey(failure) ?? directorInstancesStreakKey(failure) -} - -// Rebuild the per-signal tolerated streak from the trailing continuity events so a -// resumed monitor cannot hand a signal a fresh budget. -function resumeFreshnessStreaks( - state: IncidentMonitorState -): Map { - const events = state.continuityEvents - const streaks = new Map() - const last = events[events.length - 1] - if (!last?.tolerated) return streaks - for (const key of new Set(last.failures.map(freshnessKey))) { - let streak = 0 - let laterAt: number | null = null - for (let index = events.length - 1; index >= 0; index--) { - const event = events[index]! - const recordedAt = Date.parse(event.recordedAt) - if (!event.tolerated) break - if (laterAt !== null && laterAt - recordedAt > state.intervalMs * 1.5) break - if (!event.failures.some((failure) => freshnessKey(failure) === key)) break - streak++ - laterAt = recordedAt - } - streaks.set(key, streak) - } - return streaks -} - -function resetContinuousWindow( - state: IncidentMonitorState, - recordedAt: string, - failures: IncidentFailure[] -): void { - if (state.windowStartedAt !== null) { - state.windowSequence++ - state.windowStartedAt = null - state.nextCheckpointIndex = 0 - state.sampleCount = 0 - state.completedAt = null - } - state.continuityEvents.push({ - recordedAt, - windowSequence: state.windowSequence, - tolerated: false, - failures - }) -} - -function completeContinuityDeadline( - state: IncidentMonitorState, - nowMs: number, - lineageStartMs: number -): void { - const recordedAt = new Date(nowMs).toISOString() - state.frozenAt ??= recordedAt - state.failures.push({ - code: 'continuity_deadline_exceeded', - source: 'active-probe', - observed: nowMs - lineageStartMs, - threshold: INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS - }) - state.completedAt = recordedAt -} - -export async function runIncidentMonitor( - initialState: IncidentMonitorState, - dependencies: IncidentMonitorDependencies -): Promise { - const state = structuredClone(initialState) - const lineageStartMs = Date.parse(state.startedAt) - if (!Number.isFinite(lineageStartMs)) { - throw new Error('incident monitor start time is invalid') - } - const checkpoints = checkpointMinutes(state.durationMinutes) - const lineageDeadlineMs = state.preDrainDryRun - ? lineageStartMs + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS - : Number.POSITIVE_INFINITY - const resumedAt = dependencies.now() - const priorSampleMs = state.lastSampleAt - ? Date.parse(state.lastSampleAt) - : lineageStartMs - const gapThreshold = state.intervalMs - if (resumedAt - priorSampleMs > gapThreshold) { - resetContinuousWindow(state, new Date(resumedAt).toISOString(), [{ - code: 'monitor_gap', - source: 'active-probe', - observed: resumedAt - priorSampleMs, - threshold: gapThreshold - }]) - } - if (state.completedAt !== null) { - await dependencies.persist(state) - return state - } - const freshnessStreaks = resumeFreshnessStreaks(state) - while (state.completedAt === null) { - if (dependencies.now() > lineageDeadlineMs) { - completeContinuityDeadline(state, dependencies.now(), lineageStartMs) - await dependencies.persist(state) - break - } - const sampleStartedAt = dependencies.now() - let evaluation: IncidentEvaluation - try { - evaluation = evaluateIncidentSample( - await dependencies.collect(), - dependencies.now(), - state.migrationPolicy, - state.recoverySourceCellId, - state.capacityCellId - ) - } catch (error) { - dependencies.warn?.( - `incident monitor collector failed: ${ - error instanceof Error ? error.message : 'unknown error' - }` - ) - evaluation = { - status: 'freeze', - evaluatedAt: new Date(dependencies.now()).toISOString(), - failures: [{ - code: 'collector_failed', - source: 'cloud-monitoring' - }] - } - } - state.totalSampleCount++ - state.lastSampleAt = evaluation.evaluatedAt - const continuityFailures = evaluation.failures.filter((failure) => - CONTINUITY_FAILURE_CODES.has(failure.code) - ) - const thresholdFailures = evaluation.failures.filter((failure) => - !CONTINUITY_FAILURE_CODES.has(failure.code) - ) - const toleratedKeys = new Set( - state.windowStartedAt !== null && - continuityFailures.length > 0 && - continuityFailures.every((failure) => - TOLERABLE_CONTINUITY_FAILURE_CODES.has(failure.code)) - ? continuityFailures.map(freshnessKey) - : [] - ) - for (const key of [...freshnessStreaks.keys()]) { - if (!toleratedKeys.has(key)) freshnessStreaks.delete(key) - } - let tolerated = toleratedKeys.size > 0 - for (const key of toleratedKeys) { - const streak = (freshnessStreaks.get(key) ?? 0) + 1 - freshnessStreaks.set(key, streak) - if (streak > INCIDENT_FRESHNESS_TOLERANCE_SAMPLES) tolerated = false - } - if (continuityFailures.length > 0 && !tolerated) { - freshnessStreaks.clear() - resetContinuousWindow(state, evaluation.evaluatedAt, continuityFailures) - } else { - if (tolerated) { - state.continuityEvents.push({ - recordedAt: evaluation.evaluatedAt, - windowSequence: state.windowSequence, - tolerated: true, - failures: continuityFailures - }) - } - if (state.windowStartedAt === null) { - state.windowStartedAt = evaluation.evaluatedAt - } - state.sampleCount++ - } - const probeFailures = new Map() - for (const failure of thresholdFailures) { - const key = toleratedStreakKey(failure) - if (key === null) continue - probeFailures.set(key, [...(probeFailures.get(key) ?? []), failure]) - } - for (const key of Object.keys(state.probeStreaks)) { - if (!probeFailures.has(key)) delete state.probeStreaks[key] - } - const sustainedProbeFailures: IncidentFailure[] = [] - const toleratedProbeFailures: IncidentFailure[] = [] - for (const [key, entries] of probeFailures) { - const streak = (state.probeStreaks[key] ?? 0) + 1 - state.probeStreaks[key] = streak - const target = - streak > INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - ? sustainedProbeFailures - : toleratedProbeFailures - target.push(...entries) - } - if (toleratedProbeFailures.length > 0) { - state.toleratedProbeEvents.push({ - recordedAt: evaluation.evaluatedAt, - windowSequence: state.windowSequence, - failures: toleratedProbeFailures - }) - } - const freezingFailures = [ - ...thresholdFailures.filter((failure) => toleratedStreakKey(failure) === null), - ...sustainedProbeFailures - ] - if (freezingFailures.length > 0) { - state.frozenAt ??= evaluation.evaluatedAt - state.failures = [...state.failures, ...freezingFailures] - } - if (state.windowStartedAt === null) { - if (dependencies.now() >= lineageDeadlineMs) { - completeContinuityDeadline(state, dependencies.now(), lineageStartMs) - await dependencies.persist(state) - break - } - await dependencies.persist(state) - await dependencies.wait( - Math.max(0, Math.min(state.intervalMs, lineageDeadlineMs - dependencies.now())) - ) - continue - } - const startMs = Date.parse(state.windowStartedAt) - const endMs = startMs + state.durationMinutes * 60_000 - const elapsedMinutes = (dependencies.now() - startMs) / 60_000 - while ( - state.nextCheckpointIndex < checkpoints.length && - elapsedMinutes >= checkpoints[state.nextCheckpointIndex]! - ) { - const minute = checkpoints[state.nextCheckpointIndex]! - await dependencies.checkpoint({ - schemaVersion: 4, - incidentId: state.incidentId, - environment: state.environment, - expectedSelector: state.expectedSelector, - preDrainDryRun: state.preDrainDryRun, - migrationPolicy: state.migrationPolicy, - recoverySourceCellId: state.recoverySourceCellId, - capacityCellId: state.capacityCellId, - windowSequence: state.windowSequence, - windowStartedAt: state.windowStartedAt, - checkpointMinute: minute, - scheduledAt: new Date(startMs + minute * 60_000).toISOString(), - recordedAt: new Date(dependencies.now()).toISOString(), - status: state.frozenAt ? 'freeze' : 'green', - frozenAt: state.frozenAt, - sampleCount: state.sampleCount, - failures: state.failures, - thresholds: INCIDENT_MONITOR_THRESHOLDS - }) - state.nextCheckpointIndex++ - } - if (state.preDrainDryRun && state.frozenAt !== null) { - state.completedAt = new Date(dependencies.now()).toISOString() - await dependencies.persist(state) - break - } - if (dependencies.now() >= endMs) { - state.completedAt = new Date(dependencies.now()).toISOString() - await dependencies.persist(state) - break - } - if (dependencies.now() >= lineageDeadlineMs) { - completeContinuityDeadline(state, dependencies.now(), lineageStartMs) - await dependencies.persist(state) - break - } - await dependencies.persist(state) - await dependencies.wait( - Math.max( - 0, - Math.min(sampleStartedAt + state.intervalMs, endMs, lineageDeadlineMs) - dependencies.now() - ) - ) - } - return state -} - -export function preDrainDryRunPassed(state: Pick< - IncidentMonitorState, - | 'completedAt' - | 'durationMinutes' - | 'frozenAt' - | 'intervalMs' - | 'preDrainDryRun' - | 'sampleCount' - | 'startedAt' ->): boolean { - const minimumSamples = - Math.ceil((state.durationMinutes * 60_000) / state.intervalMs) + 1 - const lineageElapsedMs = state.completedAt === null - ? Number.POSITIVE_INFINITY - : Date.parse(state.completedAt) - Date.parse(state.startedAt) - return ( - state.preDrainDryRun && - state.durationMinutes === 15 && - state.completedAt !== null && - lineageElapsedMs >= 0 && - lineageElapsedMs <= INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS && - state.frozenAt === null && - state.sampleCount >= minimumSamples - ) -} diff --git a/cloud/apps/relay-ops/src/incident-selector.ts b/cloud/apps/relay-ops/src/incident-selector.ts deleted file mode 100644 index 7e7ddd91e59..00000000000 --- a/cloud/apps/relay-ops/src/incident-selector.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { z } from 'zod' - -export const AdmissionStateSchema = z.enum([ - 'existing-only', - 'migration-only', - 'general' -]) - -export type AdmissionState = z.infer - -export const SelectorMembershipSchema = z.object({ - existingOnly: z.array(z.string()), - migrationOnly: z.array(z.string()), - general: z.array(z.string()) -}) - -export type SelectorMembership = z.infer - -export const AdmissionSelectorSchema = z.object({ - generation: z.number().int().nonnegative(), - membership: SelectorMembershipSchema -}) - -export type AdmissionSelector = z.infer - -export function normalizeSelectorMembership( - membership: SelectorMembership, - configuredCellIds: ReadonlySet -): SelectorMembership { - const normalized = { - existingOnly: [...membership.existingOnly].sort(), - migrationOnly: [...membership.migrationOnly].sort(), - general: [...membership.general].sort() - } - const all = [ - ...normalized.existingOnly, - ...normalized.migrationOnly, - ...normalized.general - ] - if ( - all.length !== configuredCellIds.size || - new Set(all).size !== all.length || - all.some((cellId) => !configuredCellIds.has(cellId)) - ) { - throw new Error('selector membership must contain every configured cell exactly once') - } - return normalized -} - -export function selectorCellState( - selector: AdmissionSelector, - cellId: string -): AdmissionState { - if (selector.membership.existingOnly.includes(cellId)) return 'existing-only' - if (selector.membership.migrationOnly.includes(cellId)) return 'migration-only' - if (selector.membership.general.includes(cellId)) return 'general' - throw new Error(`selector does not contain ${cellId}`) -} - -export function effectiveAdmissionState( - selector: AdmissionSelector, - legacyEnabled: boolean, - cellId: string -): AdmissionState { - if (selector.generation === 0) return legacyEnabled ? 'general' : 'existing-only' - return selectorCellState(selector, cellId) -} - -export function exactAdmissionSelector( - actual: AdmissionSelector, - expected: AdmissionSelector -): boolean { - return ( - actual.generation === expected.generation && - JSON.stringify(actual.membership) === JSON.stringify(expected.membership) - ) -} diff --git a/cloud/apps/relay-ops/src/index.ts b/cloud/apps/relay-ops/src/index.ts deleted file mode 100644 index 0544cfe7595..00000000000 --- a/cloud/apps/relay-ops/src/index.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { randomBytes, timingSafeEqual } from 'node:crypto' -import { readFile } from 'node:fs/promises' -import { resolve } from 'node:path' -import { serve } from '@hono/node-server' -import { Hono } from 'hono' -import { z } from 'zod' -import { DashboardSnapshotCache } from './dashboard-snapshot.js' -import { createGcloudClient } from './gcloud-client.js' -import { - dispatchStagingPowerWorkflow, - parseStagingPowerRequest -} from './staging-workflow.js' - -const QuerySchema = z.object({ - environment: z.enum(['production', 'staging']).default('production'), - window: z.coerce.number().int().min(30).max(1440).default(360) -}) -const port = z.coerce.number().int().min(1024).max(65_535).parse(process.env.PORT ?? 2455) -const controlsEnabled = process.env.RELAY_OPS_ENABLE_STAGING_CONTROLS === '1' -const csrfToken = randomBytes(32).toString('base64url') -const publicDirectory = resolve(import.meta.dirname, '../public') -const cache = new DashboardSnapshotCache(createGcloudClient()) -const app = new Hono() - -function safeEqual(left: string, right: string): boolean { - const leftBuffer = Buffer.from(left) - const rightBuffer = Buffer.from(right) - return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer) -} - -app.use('*', async (context, next) => { - await next() - context.header('Cache-Control', 'no-store') - context.header('Content-Security-Policy', [ - "default-src 'self'", - "script-src 'self'", - "style-src 'self'", - "font-src 'self'", - "connect-src 'self'", - "img-src 'self' data:", - "object-src 'none'", - "base-uri 'none'", - "frame-ancestors 'none'", - "form-action 'self'" - ].join('; ')) - context.header('Referrer-Policy', 'no-referrer') - context.header('X-Content-Type-Options', 'nosniff') - context.header('X-Frame-Options', 'DENY') -}) - -app.get('/health', (context) => context.json({ status: 'ok' })) -app.get('/api/config', (context) => context.json({ - stagingControlsEnabled: controlsEnabled, - csrfToken: controlsEnabled ? csrfToken : null -})) -app.get('/api/snapshot', async (context) => { - const query = QuerySchema.safeParse(context.req.query()) - if (!query.success) return context.json({ error: 'Invalid dashboard query' }, 400) - try { - return context.json(await cache.read(query.data.environment, query.data.window)) - } catch { - return context.json({ - error: 'Relay operations data is unavailable. Check local gcloud and gh authentication.' - }, 503) - } -}) -app.post('/api/staging/power', async (context) => { - if (!controlsEnabled) return context.json({ error: 'Staging controls are disabled' }, 403) - const origin = context.req.header('origin') - const expectedOrigin = `http://127.0.0.1:${port}` - if (origin !== expectedOrigin) return context.json({ error: 'Origin rejected' }, 403) - if (!safeEqual(context.req.header('x-csrf-token') ?? '', csrfToken)) { - return context.json({ error: 'Request token rejected' }, 403) - } - try { - const request = parseStagingPowerRequest(await context.req.json()) - await dispatchStagingPowerWorkflow(request) - return context.json({ accepted: true }) - } catch { - return context.json({ error: 'Invalid or failed staging workflow dispatch' }, 400) - } -}) - -const staticTypes: Record = { - '/app.js': 'text/javascript; charset=utf-8', - '/styles.css': 'text/css; charset=utf-8' -} -for (const [route, contentType] of Object.entries(staticTypes)) { - app.get(route, async (context) => { - const file = route.slice(1) - try { - const content = await readFile(resolve(publicDirectory, file)) - return context.body(content, 200, { 'Content-Type': contentType }) - } catch { - return context.notFound() - } - }) -} -app.get('/', async (context) => { - const html = await readFile(resolve(publicDirectory, 'index.html'), 'utf8') - return context.html(html) -}) - -serve({ fetch: app.fetch, hostname: '127.0.0.1', port }, () => { - // Loopback is intentional; operators may add authenticated Tailscale Serve separately. - console.log(`Orca Relay Operations: http://127.0.0.1:${port}`) - console.log(`Staging controls: ${controlsEnabled ? 'enabled through GitHub workflow' : 'read-only'}`) -}) diff --git a/cloud/apps/relay-ops/src/monitoring-snapshot.test.ts b/cloud/apps/relay-ops/src/monitoring-snapshot.test.ts deleted file mode 100644 index 3bcd40c3c4b..00000000000 --- a/cloud/apps/relay-ops/src/monitoring-snapshot.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { RELAY_METRICS, readMonitoringSnapshot } from './monitoring-snapshot.js' -import type { GcloudClient } from './gcloud-client.js' -import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js' - -const gcloud: GcloudClient = { - accessToken: async () => 'a'.repeat(40) -} - -function distribution(at: string, mean: number | undefined, count = 1) { - return { - interval: { endTime: at }, - value: { distributionValue: { count, ...(mean === undefined ? {} : { mean }) } } - } -} - -describe('readMonitoringSnapshot', () => { - it('aggregates gauge series per minute and distribution deltas by count', async () => { - const fetchImpl: typeof fetch = async (input) => { - const url = new URL(String(input)) - if (url.pathname.endsWith('/alertPolicies')) { - return Response.json({ alertPolicies: [] }) - } - const filter = url.searchParams.get('filter') ?? '' - if (filter.includes('orca_relay_controls')) { - return Response.json({ timeSeries: [ - { - metric: { labels: { cell_id: 'production-gce-c1' } }, - resource: { type: 'gce_instance', labels: { instance_id: 'one' } }, - points: [ - distribution('2026-07-15T12:00:10Z', 1), - distribution('2026-07-15T12:00:50Z', 2) - ] - }, - { - metric: { labels: { cell_id: 'production-gce-c1' } }, - resource: { type: 'gce_instance', labels: { instance_id: 'two' } }, - points: [distribution('2026-07-15T12:00:20Z', 3)] - }, - { - metric: { labels: { cell_id: 'production-gce-c1' } }, - resource: { type: 'gce_instance', labels: { instance_id: 'stale' } }, - points: [distribution('2026-07-15T11:59:20Z', 100)] - } - ] }) - } - if (filter.includes('orca_relay_forwarded_bytes')) { - return Response.json({ timeSeries: [{ - metric: { labels: { cell_id: 'production-gce-c1' } }, - resource: { type: 'gce_instance', labels: { instance_id: 'one' } }, - points: [distribution('2026-07-15T12:00:20Z', 10, 4)] - }] }) - } - return Response.json({ timeSeries: [] }) - } - - const result = await readMonitoringSnapshot(RELAY_OPS_ENVIRONMENTS.production, gcloud, { - now: new Date('2026-07-15T12:01:00Z'), - windowMinutes: 30, - fetchImpl - }) - - expect(result.warnings).toEqual([]) - expect(result.metrics.controls.points).toEqual([ - { at: '2026-07-15T11:59:00.000Z', value: 100 }, - { at: '2026-07-15T12:00:00.000Z', value: 5 } - ]) - expect(result.metrics.controls.latestByCell).toEqual({ 'production-gce-c1': 5 }) - expect(result.metrics.forwarded_bytes.latest).toBe(40) - expect(result.metrics.postgres_retries.available).toBe(true) - expect(Object.keys(result.metrics)).toHaveLength(RELAY_METRICS.length) - }) - - it('degrades safely when credentials are unavailable', async () => { - const unavailable: GcloudClient = { - accessToken: async () => { throw new Error('sensitive context') } - } - const result = await readMonitoringSnapshot(RELAY_OPS_ENVIRONMENTS.production, unavailable) - expect(result.warnings).toEqual([ - 'Cloud Monitoring credentials are unavailable. Run gcloud auth login.' - ]) - expect(result.metrics.postgres_retries.available).toBe(false) - expect(JSON.stringify(result)).not.toContain('sensitive context') - }) -}) diff --git a/cloud/apps/relay-ops/src/monitoring-snapshot.ts b/cloud/apps/relay-ops/src/monitoring-snapshot.ts deleted file mode 100644 index 320dd04fbe6..00000000000 --- a/cloud/apps/relay-ops/src/monitoring-snapshot.ts +++ /dev/null @@ -1,379 +0,0 @@ -import { z } from 'zod' -import type { RelayOpsEnvironment } from './environment-config.js' -import type { GcloudClient } from './gcloud-client.js' - -type MetricMode = 'gauge-sum' | 'delta-sum' | 'maximum' - -export type RelayMetricName = - | 'total_connections' - | 'controls' - | 'splices' - | 'pending_splices' - | 'queued_bytes' - | 'http_latency_ms' - | 'sql_latency_ms' - | 'heap_used_bytes' - | 'event_loop_ms_p99' - | 'forwarded_bytes' - | 'auth_successes' - | 'auth_failures' - | 'reconnects' - | 'sql_queries' - | 'sql_failures' - | 'assignment_5xx' - | 'postgres_retries' - | 'postgres_retry_exhausted' - | 'db_pool_total' - | 'db_pool_idle' - | 'db_pool_waiting' - | 'db_waiters_max' - | 'db_oldest_wait_ms' - | 'db_wait_ms_max' - -type MetricDefinition = { - name: RelayMetricName - label: string - unit: 'count' | 'bytes' | 'milliseconds' - mode: MetricMode -} - -export const RELAY_METRICS: MetricDefinition[] = [ - { name: 'total_connections', label: 'Connections', unit: 'count', mode: 'gauge-sum' }, - { name: 'controls', label: 'Desktop controls', unit: 'count', mode: 'gauge-sum' }, - { name: 'splices', label: 'Phone splices', unit: 'count', mode: 'gauge-sum' }, - { name: 'pending_splices', label: 'Pending splices', unit: 'count', mode: 'gauge-sum' }, - { name: 'queued_bytes', label: 'Queued bytes', unit: 'bytes', mode: 'maximum' }, - { name: 'http_latency_ms', label: 'HTTP latency', unit: 'milliseconds', mode: 'maximum' }, - { name: 'sql_latency_ms', label: 'SQL latency', unit: 'milliseconds', mode: 'maximum' }, - { name: 'heap_used_bytes', label: 'Heap used', unit: 'bytes', mode: 'maximum' }, - { - name: 'event_loop_ms_p99', - label: 'Event-loop p99', - unit: 'milliseconds', - mode: 'maximum' - }, - { name: 'forwarded_bytes', label: 'Forwarded bytes', unit: 'bytes', mode: 'delta-sum' }, - { name: 'auth_successes', label: 'Auth successes', unit: 'count', mode: 'delta-sum' }, - { name: 'auth_failures', label: 'Auth failures', unit: 'count', mode: 'delta-sum' }, - { name: 'reconnects', label: 'Reconnects', unit: 'count', mode: 'delta-sum' }, - { name: 'sql_queries', label: 'SQL queries', unit: 'count', mode: 'delta-sum' }, - { name: 'sql_failures', label: 'SQL failures', unit: 'count', mode: 'delta-sum' }, - { name: 'assignment_5xx', label: 'Assignment 5xx', unit: 'count', mode: 'delta-sum' }, - { - name: 'postgres_retries', - label: 'PostgreSQL retries', - unit: 'count', - mode: 'delta-sum' - }, - { - name: 'postgres_retry_exhausted', - label: 'PostgreSQL retry exhausted', - unit: 'count', - mode: 'delta-sum' - }, - { name: 'db_pool_total', label: 'Database pool total', unit: 'count', mode: 'gauge-sum' }, - { name: 'db_pool_idle', label: 'Database pool idle', unit: 'count', mode: 'gauge-sum' }, - { - name: 'db_pool_waiting', - label: 'Database pool waiting', - unit: 'count', - mode: 'gauge-sum' - }, - { - name: 'db_waiters_max', - label: 'Database waiters max', - unit: 'count', - mode: 'maximum' - }, - { - name: 'db_oldest_wait_ms', - label: 'Database oldest wait', - unit: 'milliseconds', - mode: 'maximum' - }, - { - name: 'db_wait_ms_max', - label: 'Database wait max', - unit: 'milliseconds', - mode: 'maximum' - } -] - -const NumericSchema = z.union([z.number(), z.string()]).transform((value) => Number(value)) -const DistributionSchema = z.object({ - count: NumericSchema.default(0), - mean: NumericSchema.optional() -}) -const PointSchema = z.object({ - interval: z.object({ endTime: z.string() }), - value: z.object({ - doubleValue: NumericSchema.optional(), - int64Value: NumericSchema.optional(), - distributionValue: DistributionSchema.optional() - }) -}) -const TimeSeriesSchema = z.object({ - metric: z.object({ labels: z.record(z.string(), z.string()).default({}) }), - resource: z.object({ type: z.string(), labels: z.record(z.string(), z.string()).default({}) }), - points: z.array(PointSchema).default([]) -}) -const TimeSeriesResponseSchema = z.object({ - timeSeries: z.array(TimeSeriesSchema).default([]) -}) - -export type MetricPoint = { at: string; value: number } - -export type RelayMetricSnapshot = MetricDefinition & { - available: boolean - points: MetricPoint[] - latest: number | null - latestAt: string | null - latestByCell: Record -} - -export type AlertPolicySnapshot = { - id: string - displayName: string - enabled: boolean - documentation: string | null -} - -export type MonitoringSnapshot = { - startAt: string - endAt: string - resolutionSeconds: number - metrics: Record - alertPolicies: AlertPolicySnapshot[] - warnings: string[] -} - -type ParsedPoint = { - atMs: number - bucketMs: number - value: number - sampleTotal: number - seriesKey: string - cellId: string -} - -function pointValue(point: z.infer): { value: number; sampleTotal: number } { - if (point.value.distributionValue) { - const count = point.value.distributionValue.count - const value = point.value.distributionValue.mean ?? 0 - return { value, sampleTotal: value * count } - } - const value = point.value.doubleValue ?? point.value.int64Value ?? 0 - return { value, sampleTotal: value } -} - -function parsePoints(series: z.infer[]): ParsedPoint[] { - return series.flatMap((entry) => { - const cellId = entry.metric.labels.cell_id ?? 'unknown' - const resourceId = - entry.resource.labels.instance_id ?? entry.resource.labels.revision_name ?? entry.resource.type - const seriesKey = `${cellId}:${resourceId}` - return entry.points.flatMap((point) => { - const atMs = Date.parse(point.interval.endTime) - if (!Number.isFinite(atMs)) return [] - const values = pointValue(point) - return [{ - atMs, - bucketMs: Math.floor(atMs / 60_000) * 60_000, - value: values.value, - sampleTotal: values.sampleTotal, - seriesKey, - cellId - }] - }) - }) -} - -function aggregatePoints(points: ParsedPoint[], mode: MetricMode): MetricPoint[] { - if (mode === 'gauge-sum') { - const buckets = new Map>() - for (const point of points) { - const bySeries = buckets.get(point.bucketMs) ?? new Map() - const previous = bySeries.get(point.seriesKey) - if (!previous || previous.atMs < point.atMs) bySeries.set(point.seriesKey, point) - buckets.set(point.bucketMs, bySeries) - } - return [...buckets.entries()] - .sort(([left], [right]) => left - right) - .map(([at, bySeries]) => ({ - at: new Date(at).toISOString(), - value: [...bySeries.values()].reduce((total, point) => total + point.value, 0) - })) - } - const buckets = new Map() - for (const point of points) { - const value = mode === 'delta-sum' ? point.sampleTotal : point.value - const previous = buckets.get(point.bucketMs) - buckets.set( - point.bucketMs, - mode === 'maximum' ? Math.max(previous ?? 0, value) : (previous ?? 0) + value - ) - } - return [...buckets.entries()] - .sort(([left], [right]) => left - right) - .map(([at, value]) => ({ at: new Date(at).toISOString(), value })) -} - -function latestByCell(points: ParsedPoint[], mode: MetricMode): Record { - const newestBucketByCell = new Map() - for (const point of points) { - newestBucketByCell.set( - point.cellId, - Math.max(newestBucketByCell.get(point.cellId) ?? 0, point.bucketMs) - ) - } - const newestBySeries = new Map() - for (const point of points) { - if (point.bucketMs !== newestBucketByCell.get(point.cellId)) continue - const previous = newestBySeries.get(point.seriesKey) - if (!previous || previous.atMs < point.atMs) newestBySeries.set(point.seriesKey, point) - } - const totals = new Map() - for (const point of newestBySeries.values()) { - const value = mode === 'delta-sum' ? point.sampleTotal : point.value - const previous = totals.get(point.cellId) - totals.set( - point.cellId, - mode === 'maximum' ? Math.max(previous ?? 0, value) : (previous ?? 0) + value - ) - } - return Object.fromEntries(totals) -} - -async function monitoringRequest( - fetchImpl: typeof fetch, - token: string, - url: URL -): Promise { - const response = await fetchImpl(url, { - headers: { authorization: `Bearer ${token}` }, - signal: AbortSignal.timeout(30_000) - }) - if (!response.ok) throw new Error(`Cloud Monitoring returned ${response.status}`) - return await response.json() -} - -async function readMetric( - environment: RelayOpsEnvironment, - definition: MetricDefinition, - token: string, - startAt: string, - endAt: string, - fetchImpl: typeof fetch -): Promise { - const url = new URL( - `https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries` - ) - url.searchParams.set( - 'filter', - `metric.type="logging.googleapis.com/user/orca_relay_${definition.name}"` - ) - url.searchParams.set('interval.startTime', startAt) - url.searchParams.set('interval.endTime', endAt) - url.searchParams.set('view', 'FULL') - url.searchParams.set('pageSize', '1000') - const body = TimeSeriesResponseSchema.parse( - await monitoringRequest(fetchImpl, token, url) - ) - const parsed = parsePoints(body.timeSeries) - const points = aggregatePoints(parsed, definition.mode) - const latest = points.at(-1) ?? null - return { - ...definition, - available: true, - points, - latest: latest?.value ?? null, - latestAt: latest?.at ?? null, - latestByCell: latestByCell(parsed, definition.mode) - } -} - -async function readAlertPolicies( - environment: RelayOpsEnvironment, - token: string, - fetchImpl: typeof fetch -): Promise { - const url = new URL( - `https://monitoring.googleapis.com/v3/projects/${environment.project}/alertPolicies` - ) - url.searchParams.set('pageSize', '100') - const body = (await monitoringRequest(fetchImpl, token, url)) as { - alertPolicies?: Array<{ - name?: string - displayName?: string - enabled?: boolean - documentation?: { content?: string } - }> - } - return (body.alertPolicies ?? []) - .filter((policy) => policy.displayName?.startsWith('Orca Relay:')) - .map((policy) => ({ - id: policy.name ?? '', - displayName: policy.displayName ?? 'Orca Relay alert', - enabled: policy.enabled === true, - documentation: policy.documentation?.content ?? null - })) - .sort((left, right) => left.displayName.localeCompare(right.displayName)) -} - -function emptyMetric(definition: MetricDefinition): RelayMetricSnapshot { - return { - ...definition, - available: false, - points: [], - latest: null, - latestAt: null, - latestByCell: {} - } -} - -export async function readMonitoringSnapshot( - environment: RelayOpsEnvironment, - gcloud: GcloudClient, - options: { now?: Date; windowMinutes?: number; fetchImpl?: typeof fetch } = {} -): Promise { - const now = options.now ?? new Date() - const windowMinutes = Math.min(24 * 60, Math.max(30, options.windowMinutes ?? 360)) - const endAt = now.toISOString() - const startAt = new Date(now.getTime() - windowMinutes * 60_000).toISOString() - const fetchImpl = options.fetchImpl ?? fetch - const warnings: string[] = [] - let token: string - try { - token = await gcloud.accessToken() - } catch { - return { - startAt, - endAt, - resolutionSeconds: 60, - metrics: Object.fromEntries( - RELAY_METRICS.map((definition) => [definition.name, emptyMetric(definition)]) - ) as Record, - alertPolicies: [], - warnings: ['Cloud Monitoring credentials are unavailable. Run gcloud auth login.'] - } - } - const settled = await Promise.allSettled( - RELAY_METRICS.map((definition) => - readMetric(environment, definition, token, startAt, endAt, fetchImpl) - ) - ) - const metrics = {} as Record - settled.forEach((result, index) => { - const definition = RELAY_METRICS[index]! - if (result.status === 'fulfilled') metrics[definition.name] = result.value - else { - metrics[definition.name] = emptyMetric(definition) - warnings.push(`${definition.label} metric is unavailable.`) - } - }) - const alertPolicies = await readAlertPolicies(environment, token, fetchImpl).catch(() => { - warnings.push('Cloud Monitoring alert policies are unavailable.') - return [] - }) - return { startAt, endAt, resolutionSeconds: 60, metrics, alertPolicies, warnings } -} diff --git a/cloud/apps/relay-ops/src/pre-drain-hard-rule-sources.test.ts b/cloud/apps/relay-ops/src/pre-drain-hard-rule-sources.test.ts deleted file mode 100644 index 6bcff6b10f4..00000000000 --- a/cloud/apps/relay-ops/src/pre-drain-hard-rule-sources.test.ts +++ /dev/null @@ -1,210 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { relayOpsEnvironment } from './environment-config.js' -import { - CELL_PROCESS_EXIT_METRIC, - createPreDrainHardRuleReader, - INSTANCE_CELL_LOOKBACK_MS, - type PreDrainExitScope -} from './pre-drain-hard-rule-sources.js' - -const environment = relayOpsEnvironment('production') -const now = Date.parse('2026-10-01T12:00:00.000Z') -const scope: PreDrainExitScope = { - targetCellId: 'production-gce-c25', - placementCellIds: new Set(['production-gce-c25', 'production-gce-c26']), - configuredCellIds: new Set(environment.cells.map((cell) => cell.cellId)) -} - -function points(values: number[], kind: 'int64Value' | 'doubleValue' = 'int64Value') { - return values.map((value, index) => ({ - interval: { endTime: new Date(now - index * 60_000).toISOString() }, - value: { [kind]: kind === 'int64Value' ? String(value) : value } - })) -} - -type Fake = { - // instance id -> per-minute exit counts - exits?: Record - // instance id -> the cell its runtime-metrics lines name; absent means it logged none - cells?: Record - director503?: number[][] - concurrency?: number[][] - descriptorStatus?: number - loggingStatus?: number -} - -function fakeGoogle(fake: Fake) { - const requests: { url: URL; body: string | null }[] = [] - const fetchImpl: typeof fetch = async (input, init) => { - const url = new URL(String(input)) - const body = typeof init?.body === 'string' ? init.body : null - requests.push({ url, body }) - if (url.pathname.includes('/metricDescriptors/')) { - return new Response('{}', { status: fake.descriptorStatus ?? 200 }) - } - if (url.hostname === 'logging.googleapis.com') { - if (fake.loggingStatus) return new Response('{}', { status: fake.loggingStatus }) - const filter = String(JSON.parse(body ?? '{}').filter ?? '') - const instanceId = /instance_id="([0-9]+)"/.exec(filter)?.[1] ?? '' - const cellId = fake.cells?.[instanceId] - return Response.json({ entries: cellId ? [{ jsonPayload: { cellId } }] : [] }) - } - const filter = url.searchParams.get('filter') ?? '' - if (filter.includes(CELL_PROCESS_EXIT_METRIC)) { - return Response.json({ - timeSeries: Object.entries(fake.exits ?? {}).map(([instanceId, values]) => ({ - resource: { labels: { instance_id: instanceId } }, - points: points(values) - })) - }) - } - const series = filter.includes('request_count') - ? (fake.director503 ?? []).map((values) => points(values)) - : (fake.concurrency ?? [[1]]).map((values) => points(values, 'doubleValue')) - return Response.json({ timeSeries: series.map((entry) => ({ points: entry })) }) - } - return { fetchImpl, requests } -} - -function reader(fetchImpl: typeof fetch) { - return createPreDrainHardRuleReader(environment, async () => 'token', scope, { - fetchImpl, - now: () => now - }) -} - -describe('pre-drain hard rule reader', () => { - it('takes the busiest 503 minute and the highest concurrency p99', async () => { - const { fetchImpl } = fakeGoogle({ - director503: [[40, 620, 55]], - concurrency: [[12.5, 66.8]] - }) - await expect(reader(fetchImpl)()).resolves.toEqual({ - cellProcessExits: 0, - unattributedExitInstances: [], - director503PeakPerMinute: 620, - directorConcurrencyP99: 66.8 - }) - }) - - it('reads an empty concurrency series as unknown, not calm', async () => { - const { fetchImpl } = fakeGoogle({ concurrency: [] }) - await expect(reader(fetchImpl)()).resolves.toMatchObject({ directorConcurrencyP99: null }) - }) - - it('ignores the target cell rolling the fix for its own crashes', async () => { - const { fetchImpl } = fakeGoogle({ - exits: { '2525': [3, 4] }, - cells: { '2525': 'production-gce-c25' } - }) - await expect(reader(fetchImpl)()).resolves.toMatchObject({ - cellProcessExits: 0, - unattributedExitInstances: [] - }) - }) - - it('counts an exit on another cell that takes placements', async () => { - const { fetchImpl } = fakeGoogle({ - exits: { '2626': [1], '2525': [5] }, - cells: { '2626': 'production-gce-c26', '2525': 'production-gce-c25' } - }) - await expect(reader(fetchImpl)()).resolves.toMatchObject({ - cellProcessExits: 1, - unattributedExitInstances: [] - }) - }) - - it('ignores an existing-only legacy cell', async () => { - const { fetchImpl } = fakeGoogle({ - exits: { '505': [15] }, - cells: { '505': 'production-gce-c5' } - }) - await expect(reader(fetchImpl)()).resolves.toMatchObject({ - cellProcessExits: 0, - unattributedExitInstances: [] - }) - }) - - it('reports an exit it cannot name a cell for as unattributed', async () => { - const { fetchImpl } = fakeGoogle({ exits: { '999': [1] } }) - await expect(reader(fetchImpl)()).resolves.toMatchObject({ - cellProcessExits: 0, - unattributedExitInstances: ['999'] - }) - }) - - it('fails the read, rather than guessing, when the cell lookup itself fails', async () => { - const { fetchImpl } = fakeGoogle({ exits: { '2626': [1] }, loggingStatus: 403 }) - await expect(reader(fetchImpl)()).rejects.toThrow('Google telemetry returned 403') - }) - - it('names an instance from its own runtime-metrics lines, once', async () => { - const { fetchImpl, requests } = fakeGoogle({ - exits: { '2626': [1] }, - cells: { '2626': 'production-gce-c26' } - }) - const read = reader(fetchImpl) - await read() - await read() - const lookups = requests.filter(({ url }) => url.hostname === 'logging.googleapis.com') - expect(lookups).toHaveLength(1) - const body = JSON.parse(lookups[0]?.body ?? '{}') - expect(body.filter).toContain('resource.labels.instance_id="2626"') - expect(body.filter).toContain('jsonPayload.event="orca_relay_runtime_metrics"') - expect(body.filter).toContain( - `timestamp>="${new Date(now - INSTANCE_CELL_LOOKBACK_MS).toISOString()}"` - ) - expect(body.orderBy).toBe('timestamp desc') - }) - - it('asks again next sample for an instance it could not name', async () => { - const fake: Fake = { exits: { '2626': [1] } } - const { fetchImpl, requests } = fakeGoogle(fake) - const read = reader(fetchImpl) - await expect(read()).resolves.toMatchObject({ unattributedExitInstances: ['2626'] }) - fake.cells = { '2626': 'production-gce-c26' } - await expect(read()).resolves.toMatchObject({ - cellProcessExits: 1, - unattributedExitInstances: [] - }) - expect(requests.filter(({ url }) => url.hostname === 'logging.googleapis.com')) - .toHaveLength(2) - }) - - it('aligns each read per minute over its own lookback on the director service', async () => { - const { fetchImpl, requests } = fakeGoogle({}) - await reader(fetchImpl)() - const reads = requests.map(({ url }) => url).filter((url) => url.pathname.endsWith('/timeSeries')) - expect(reads).toHaveLength(3) - const byMetric = (needle: string): URL => { - const url = reads.find((entry) => entry.searchParams.get('filter')?.includes(needle)) - if (!url) throw new Error(`no read for ${needle}`) - return url - } - const exits = byMetric(CELL_PROCESS_EXIT_METRIC) - expect(exits.searchParams.get('interval.startTime')).toBe('2026-10-01T11:50:00.000Z') - expect(exits.searchParams.get('aggregation.groupByFields')) - .toBe('resource.label."instance_id"') - const director503 = byMetric('run.googleapis.com/request_count') - expect(director503.searchParams.get('filter')).toContain('metric.label."response_code"="503"') - expect(director503.searchParams.get('filter')).toContain( - `resource.label."service_name"="${environment.directorService}"` - ) - expect(director503.searchParams.get('aggregation.perSeriesAligner')).toBe('ALIGN_DELTA') - expect(director503.searchParams.get('aggregation.crossSeriesReducer')).toBe('REDUCE_SUM') - const concurrency = byMetric('max_request_concurrencies') - expect(concurrency.searchParams.get('interval.startTime')).toBe('2026-10-01T11:56:00.000Z') - expect(concurrency.searchParams.get('aggregation.perSeriesAligner')) - .toBe('ALIGN_PERCENTILE_99') - for (const url of reads) { - expect(url.searchParams.get('aggregation.alignmentPeriod')).toBe('60s') - expect(url.searchParams.get('interval.endTime')).toBe('2026-10-01T12:00:00.000Z') - } - }) - - // Why: a deleted or renamed log metric reads as an empty series, which is zero exits forever. - it('refuses to count exits from a metric that does not exist', async () => { - const { fetchImpl } = fakeGoogle({ descriptorStatus: 404 }) - await expect(reader(fetchImpl)()).rejects.toThrow('Google telemetry returned 404') - }) -}) diff --git a/cloud/apps/relay-ops/src/pre-drain-hard-rule-sources.ts b/cloud/apps/relay-ops/src/pre-drain-hard-rule-sources.ts deleted file mode 100644 index 037a50e8872..00000000000 --- a/cloud/apps/relay-ops/src/pre-drain-hard-rule-sources.ts +++ /dev/null @@ -1,217 +0,0 @@ -import { z } from 'zod' -import type { RelayOpsEnvironment } from './environment-config.js' -import { - googleJson, - MonitoringPointSchema, - MonitoringResponseSchema, - pointValue -} from './incident-monitor-sources.js' -import { - attributeCellExits, - PRE_DRAIN_HARD_RULES, - type PreDrainHardRuleReadings -} from './pre-drain-sample.js' - -// The Docker `container die` count per relay cell, from infra/terraform/relay-observability.tf. -export const CELL_PROCESS_EXIT_METRIC = 'logging.googleapis.com/user/orca_relay_cell_process_exit' - -// How far back a cell's own runtime-metrics line is searched to name an instance. The instance id -// survives a container restart, so a crash-looping container is still named by its earlier lines. -export const INSTANCE_CELL_LOOKBACK_MS = 2 * 60 * 60_000 - -const ExitsByInstanceSchema = z.object({ - timeSeries: z.array(z.object({ - resource: z.object({ labels: z.object({ instance_id: z.string().regex(/^[0-9]+$/) }) }), - points: z.array(MonitoringPointSchema) - })).default([]), - nextPageToken: z.string().optional() -}) - -const RuntimeMetricsEntriesSchema = z.object({ - entries: z.array(z.object({ - jsonPayload: z.object({ cellId: z.string() }) - })).default([]) -}) - -export type PreDrainExitScope = { - targetCellId: string - placementCellIds: ReadonlySet - configuredCellIds: ReadonlySet -} - -type AlignedRead = { - filter: string - lookbackMs: number - aligner: 'ALIGN_DELTA' | 'ALIGN_PERCENTILE_99' - reducer: 'REDUCE_SUM' | 'REDUCE_MAX' -} - -// Server-side per-minute alignment, so a burst is counted rather than sampled and no read can be -// truncated the way a log read is. -async function readPerMinute( - environment: RelayOpsEnvironment, - token: string, - fetchImpl: typeof fetch, - nowMs: number, - read: AlignedRead -): Promise { - const url = new URL( - `https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries` - ) - url.searchParams.set('filter', read.filter) - url.searchParams.set('interval.startTime', new Date(nowMs - read.lookbackMs).toISOString()) - url.searchParams.set('interval.endTime', new Date(nowMs).toISOString()) - url.searchParams.set('aggregation.alignmentPeriod', '60s') - url.searchParams.set('aggregation.perSeriesAligner', read.aligner) - url.searchParams.set('aggregation.crossSeriesReducer', read.reducer) - url.searchParams.set('pageSize', '1000') - const parsed = MonitoringResponseSchema.parse(await googleJson(fetchImpl, token, url)) - if (parsed.nextPageToken) throw new Error('Google metric pagination is incomplete') - return parsed.timeSeries.flatMap((series) => series.points.map(pointValue)) -} - -function directorFilter(environment: RelayOpsEnvironment, metricType: string): string { - return [ - `metric.type="${metricType}"`, - 'resource.type="cloud_run_revision"', - `resource.label."service_name"="${environment.directorService}"` - ].join(' AND ') -} - -async function readExitsByInstance( - environment: RelayOpsEnvironment, - token: string, - fetchImpl: typeof fetch, - nowMs: number -): Promise> { - const url = new URL( - `https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries` - ) - url.searchParams.set( - 'filter', - `metric.type="${CELL_PROCESS_EXIT_METRIC}" AND resource.type="gce_instance"` - ) - url.searchParams.set( - 'interval.startTime', - new Date(nowMs - PRE_DRAIN_HARD_RULES.cellProcessExitLookbackMs).toISOString() - ) - url.searchParams.set('interval.endTime', new Date(nowMs).toISOString()) - url.searchParams.set('aggregation.alignmentPeriod', '60s') - url.searchParams.set('aggregation.perSeriesAligner', 'ALIGN_SUM') - url.searchParams.set('aggregation.crossSeriesReducer', 'REDUCE_SUM') - url.searchParams.set('aggregation.groupByFields', 'resource.label."instance_id"') - url.searchParams.set('pageSize', '1000') - const parsed = ExitsByInstanceSchema.parse(await googleJson(fetchImpl, token, url)) - if (parsed.nextPageToken) throw new Error('Google metric pagination is incomplete') - const exits = new Map() - for (const series of parsed.timeSeries) { - const instanceId = series.resource.labels.instance_id - const total = series.points.map(pointValue).reduce((sum, value) => sum + value, 0) - exits.set(instanceId, (exits.get(instanceId) ?? 0) + total) - } - return exits -} - -// The cell an instance serves, from that instance's own newest runtime-metrics line; the exit -// metric carries only the instance id. Null when the instance logged none in the lookback. -async function readInstanceCell( - environment: RelayOpsEnvironment, - token: string, - fetchImpl: typeof fetch, - nowMs: number, - instanceId: string -): Promise { - const since = new Date(nowMs - INSTANCE_CELL_LOOKBACK_MS).toISOString() - const body = RuntimeMetricsEntriesSchema.parse(await googleJson( - fetchImpl, - token, - 'https://logging.googleapis.com/v2/entries:list', - { - method: 'POST', - body: JSON.stringify({ - resourceNames: [`projects/${environment.project}`], - filter: [ - 'resource.type="gce_instance"', - `resource.labels.instance_id="${instanceId}"`, - 'jsonPayload.event="orca_relay_runtime_metrics"', - `timestamp>="${since}"` - ].join(' AND '), - orderBy: 'timestamp desc', - pageSize: 1 - }) - } - )) - return body.entries[0]?.jsonPayload.cellId ?? null -} - -export function createPreDrainHardRuleReader( - environment: RelayOpsEnvironment, - accessToken: () => Promise, - scope: PreDrainExitScope, - options: { fetchImpl?: typeof fetch; now?: () => number } = {} -): () => Promise { - const fetchImpl = options.fetchImpl ?? fetch - const now = options.now ?? Date.now - // An exit count of zero is only evidence if the metric exists; a renamed or deleted log metric - // reads as an empty, calm series forever. Checked once, before the first reading is trusted. - let descriptorChecked: Promise | null = null - const requireExitMetric = (token: string): Promise => { - descriptorChecked ??= googleJson( - fetchImpl, - token, - `https://monitoring.googleapis.com/v3/projects/${environment.project}` + - `/metricDescriptors/${CELL_PROCESS_EXIT_METRIC}` - ).then(() => undefined, (error: unknown) => { - descriptorChecked = null - throw error - }) - return descriptorChecked - } - // An instance belongs to one cell for its whole life, so a resolved name is kept; an unresolved - // one is asked again next sample. - const cellByInstance = new Map() - return async () => { - const token = await accessToken() - await requireExitMetric(token) - const nowMs = now() - const [exitsByInstance, director503, concurrency] = await Promise.all([ - readExitsByInstance(environment, token, fetchImpl, nowMs), - readPerMinute(environment, token, fetchImpl, nowMs, { - filter: `${directorFilter(environment, 'run.googleapis.com/request_count')}` + - ' AND metric.label."response_code"="503"', - lookbackMs: PRE_DRAIN_HARD_RULES.director503LookbackMs, - aligner: 'ALIGN_DELTA', - reducer: 'REDUCE_SUM' - }), - readPerMinute(environment, token, fetchImpl, nowMs, { - filter: `${directorFilter( - environment, - 'run.googleapis.com/container/max_request_concurrencies' - )} AND metric.label."state"="active"`, - lookbackMs: PRE_DRAIN_HARD_RULES.directorConcurrencyLookbackMs, - aligner: 'ALIGN_PERCENTILE_99', - reducer: 'REDUCE_MAX' - }) - ]) - const resolved = new Map() - for (const [instanceId, exits] of exitsByInstance) { - if (exits <= 0) continue - const known = cellByInstance.get(instanceId) - const cellId = known ?? await readInstanceCell(environment, token, fetchImpl, nowMs, instanceId) - if (cellId !== null) cellByInstance.set(instanceId, cellId) - resolved.set(instanceId, cellId) - } - const attribution = attributeCellExits({ - exitsByInstance, - cellByInstance: resolved, - ...scope - }) - return { - cellProcessExits: attribution.counted, - unattributedExitInstances: attribution.unattributed, - // 503s are a counter, so a minute with no point had none. - director503PeakPerMinute: Math.max(0, ...director503), - directorConcurrencyP99: concurrency.length === 0 ? null : Math.max(...concurrency) - } - } -} diff --git a/cloud/apps/relay-ops/src/pre-drain-sample-cli.test.ts b/cloud/apps/relay-ops/src/pre-drain-sample-cli.test.ts deleted file mode 100644 index b997fd60df2..00000000000 --- a/cloud/apps/relay-ops/src/pre-drain-sample-cli.test.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' -import { relayOpsEnvironment } from './environment-config.js' -import { parsePreDrainSampleArgs, runPreDrainSampleCli } from './pre-drain-sample-cli.js' - -const directories: string[] = [] - -afterEach(() => { - for (const directory of directories.splice(0)) { - rmSync(directory, { recursive: true, force: true }) - } -}) - -function membershipFile(): string { - const directory = mkdtempSync(join(tmpdir(), 'relay-pre-drain-selector-')) - directories.push(directory) - const path = join(directory, 'selector.json') - writeFileSync(path, JSON.stringify({ - existingOnly: relayOpsEnvironment('production').cells.map((cell) => cell.cellId), - migrationOnly: [], - general: [] - })) - return path -} - -function args(overrides: Record = {}): string[] { - const values: Record = { - '--target-cell-id': 'production-gce-c25', - '--target-hosts': '1200', - '--expected-selector-generation': '40', - '--selector-membership-file': membershipFile(), - '--wave-index': '2', - '--selector-wave-delta': '2', - ...overrides - } - return Object.entries(values).flatMap(([name, value]) => [name, value]) -} - -describe('pre-drain sample CLI', () => { - it('requires every option exactly once and nothing else', () => { - expect(() => parsePreDrainSampleArgs(['--', ...args()])).not.toThrow() - const full = args() - for (let index = 0; index < full.length; index += 2) { - const missing = [...full.slice(0, index), ...full.slice(index + 2)] - expect(() => parsePreDrainSampleArgs(missing)).toThrow('usage') - } - expect(() => parsePreDrainSampleArgs([...full, '--wave-index', '2'])).toThrow('usage') - expect(() => parsePreDrainSampleArgs([...full, '--skip-window', '1'])).toThrow('usage') - expect(() => parsePreDrainSampleArgs(args({ '--target-hosts': '-1' }))).toThrow('usage') - expect(() => parsePreDrainSampleArgs(args({ '--target-hosts': '' }))).toThrow('usage') - }) - - it('rejects a target cell the environment does not configure', async () => { - await expect(runPreDrainSampleCli(args({ '--target-cell-id': 'production-gce-c99' }), { - collect: async () => { throw new Error('must not sample') }, - readHardRules: async () => { throw new Error('must not sample') } - })).rejects.toThrow('target cell is unknown') - }) - - it('rejects a wave index or selector delta no wave produces', async () => { - for (const overrides of [{ '--wave-index': '10' }, { '--selector-wave-delta': '1' }]) { - await expect(runPreDrainSampleCli(args(overrides), { - collect: async () => { throw new Error('must not sample') }, - readHardRules: async () => { throw new Error('must not sample') } - })).rejects.toThrow('wave index or selector wave delta is invalid') - } - }) - - it('sizes the window from the target hosts and reports the verdict', async () => { - let clock = Date.parse('2026-10-01T12:00:00.000Z') - const lines: string[] = [] - let collected = 0 - // A sample that cannot be judged green trips the run, which is enough to see the window. - await expect(runPreDrainSampleCli(args(), { - now: () => clock, - wait: async (ms) => { clock += ms }, - collect: async () => { - collected += 1 - throw new Error('collector down') - }, - readHardRules: async () => ({ - cellProcessExits: 0, - unattributedExitInstances: [], - director503PeakPerMinute: 0, - directorConcurrencyP99: 1 - }), - print: (line) => lines.push(line) - })).rejects.toThrow('relay pre-drain sample tripped') - const events = lines.map((line) => JSON.parse(line)) - expect(events[0]).toEqual({ - event: 'relay_pre_drain_sample_window', - targetCellId: 'production-gce-c25', - targetHosts: 1200, - windowMinutes: 5 - }) - expect(events.at(-1)).toMatchObject({ event: 'relay_pre_drain_sample_tripped' }) - expect(collected).toBe(3) - }) -}) diff --git a/cloud/apps/relay-ops/src/pre-drain-sample-cli.ts b/cloud/apps/relay-ops/src/pre-drain-sample-cli.ts deleted file mode 100644 index 7d871964ce5..00000000000 --- a/cloud/apps/relay-ops/src/pre-drain-sample-cli.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { pathToFileURL } from 'node:url' -import { relayOpsEnvironment } from './environment-config.js' -import { createGcloudClient } from './gcloud-client.js' -import { - livePreflightGcloud, - readDispatchSelector, - waveAdjustedSelector -} from './incident-live-preflight-cli.js' -import { createIncidentSampleCollector } from './incident-monitor-sources.js' -import type { IncidentSample } from './incident-monitor.js' -import { createPreDrainHardRuleReader } from './pre-drain-hard-rule-sources.js' -import { - preDrainSampleWindowMinutes, - PreDrainSampleTripped, - runPreDrainSample, - type PreDrainHardRuleReadings -} from './pre-drain-sample.js' - -const USAGE = - 'usage: --target-cell-id --target-hosts --expected-selector-generation ' + - ' --selector-membership-file --wave-index <0-9> --selector-wave-delta <0|2>' - -const OPTIONS = [ - '--target-cell-id', - '--target-hosts', - '--expected-selector-generation', - '--selector-membership-file', - '--wave-index', - '--selector-wave-delta' -] as const - -type Option = (typeof OPTIONS)[number] - -const COUNT = /^(0|[1-9][0-9]*)$/ - -function isOption(value: string): value is Option { - return (OPTIONS as readonly string[]).includes(value) -} - -// Every option is required exactly once, so a typo can never silently shorten the window. -export function parsePreDrainSampleArgs(argv: string[]): Record { - const args = argv[0] === '--' ? argv.slice(1) : argv - const values = new Map() - for (let index = 0; index < args.length; index += 2) { - const name = args[index] ?? '' - const value = args[index + 1] - if (!isOption(name) || values.has(name) || !value) throw new Error(USAGE) - values.set(name, value) - } - const get = (name: Option): string => { - const value = values.get(name) - if (value === undefined) throw new Error(USAGE) - return value - } - const parsed = { - '--target-cell-id': get('--target-cell-id'), - '--target-hosts': get('--target-hosts'), - '--expected-selector-generation': get('--expected-selector-generation'), - '--selector-membership-file': get('--selector-membership-file'), - '--wave-index': get('--wave-index'), - '--selector-wave-delta': get('--selector-wave-delta') - } - if ( - !COUNT.test(parsed['--target-hosts']) || - !COUNT.test(parsed['--expected-selector-generation']) - ) throw new Error(USAGE) - return parsed -} - -export async function runPreDrainSampleCli( - argv: string[], - dependencies: { - now?: () => number - wait?: (ms: number) => Promise - collect?: () => Promise - readHardRules?: () => Promise - gcloud?: ReturnType - environment?: NodeJS.ProcessEnv - print?: (line: string) => void - } = {} -): Promise { - const options = parsePreDrainSampleArgs(argv) - const environment = relayOpsEnvironment('production') - const configuredCellIds = new Set(environment.cells.map((cell) => cell.cellId)) - const targetCellId = options['--target-cell-id'] - // An unknown target would exempt nothing and make every one of its exits unattributed. - if (!configuredCellIds.has(targetCellId)) throw new Error('pre-drain sample target cell is unknown') - const windowMinutes = preDrainSampleWindowMinutes(Number(options['--target-hosts'])) - const expectedSelector = waveAdjustedSelector( - await readDispatchSelector( - options['--expected-selector-generation'], - options['--selector-membership-file'] - ), - options['--wave-index'], - options['--selector-wave-delta'] - ) - const print = dependencies.print ?? ((line: string) => console.log(line)) - const gcloud = livePreflightGcloud( - dependencies.gcloud ?? createGcloudClient(), - dependencies.environment - ) - const collect = dependencies.collect ?? createIncidentSampleCollector(gcloud, { - environment: 'production', - expectedSelector, - ...(dependencies.now ? { now: dependencies.now } : {}) - }) - const { membership } = expectedSelector - const readHardRules = dependencies.readHardRules ?? createPreDrainHardRuleReader( - environment, - () => gcloud.accessToken(), - { - targetCellId, - placementCellIds: new Set([...membership.general, ...membership.migrationOnly]), - configuredCellIds - }, - dependencies.now ? { now: dependencies.now } : {} - ) - print(JSON.stringify({ - event: 'relay_pre_drain_sample_window', - targetCellId, - targetHosts: Number(options['--target-hosts']), - windowMinutes - })) - try { - const result = await runPreDrainSample({ - windowMinutes, - collect, - readHardRules, - ...(dependencies.now ? { now: dependencies.now } : {}), - ...(dependencies.wait ? { wait: dependencies.wait } : {}), - log: (record) => print(JSON.stringify({ event: 'relay_pre_drain_sample', ...record })) - }) - print(JSON.stringify({ - event: 'relay_pre_drain_sample_passed', - windowMinutes: result.windowMinutes, - startedAt: result.startedAt, - completedAt: result.completedAt, - samples: result.samples.length - })) - } catch (error) { - if (error instanceof PreDrainSampleTripped) { - print(JSON.stringify({ - event: 'relay_pre_drain_sample_tripped', - windowMinutes, - samples: error.samples.length, - failures: error.failures - })) - } - throw error - } -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - runPreDrainSampleCli(process.argv.slice(2)).catch((error: unknown) => { - console.error(error instanceof Error ? error.message : 'relay pre-drain sample failed') - process.exitCode = 1 - }) -} diff --git a/cloud/apps/relay-ops/src/pre-drain-sample.test.ts b/cloud/apps/relay-ops/src/pre-drain-sample.test.ts deleted file mode 100644 index 8e74ddb1a95..00000000000 --- a/cloud/apps/relay-ops/src/pre-drain-sample.test.ts +++ /dev/null @@ -1,358 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - INCIDENT_FRESHNESS_TOLERANCE_SAMPLES, - INCIDENT_MONITOR_THRESHOLDS, - type IncidentSample -} from './incident-monitor.js' -import { - attributeCellExits, - evaluatePreDrainHardRules, - PRE_DRAIN_HARD_RULES, - PRE_DRAIN_MAX_OVERRUN_SAMPLES, - preDrainSampleWindowMinutes, - PreDrainSampleTripped, - runPreDrainSample, - type PreDrainHardRuleReadings -} from './pre-drain-sample.js' - -const selector = { - generation: 1, - membership: { existingOnly: ['production-gce-c1'], migrationOnly: [], general: [] } -} - -function greenSample(at: number): IncidentSample { - const observedAt = new Date(at).toISOString() - const signal = (value: number) => ({ value, observedAt }) - const source = (signals: Record) => ({ - observedAt, - signals: Object.fromEntries( - Object.entries(signals).map(([name, value]) => [name, signal(value)]) - ) - }) - return { - collectedAt: observedAt, - selector, - expectedSelector: selector, - cells: [{ - cellId: 'production-gce-c1', - region: 'us-central1', - runtimeKnown: true, - powered: true, - expectedAdmissionState: 'existing-only' - }], - sources: { - 'active-probe': source({ - 'director.health': 1, - 'director.ready': 1, - 'director.latency_ms': 1, - 'auth.health': 1, - 'auth.ready': 1, - 'auth.latency_ms': 1, - 'cell.production-gce-c1.health': 1, - 'cell.production-gce-c1.ready': 1, - 'cell.production-gce-c1.latency_ms': 1 - }), - 'cloud-monitoring': source({ - 'cloud_sql.cpu': 0.1, - 'cloud_sql.memory': 0.1, - 'cloud_sql.backends': 1, - 'cloud_sql.lock_waits': 0, - 'cloud_sql.deadlocks': 0, - 'director.instances': 5, - 'director.cpu': 0.1, - 'director.memory': 0.1, - 'director.concurrency': 1, - 'director.errors': 0, - 'auth.errors': 0 - }), - 'relay-logs': source({ - 'relay.pool_waiting': 0, - 'relay.pool_wait_ms': 0, - 'relay.postgres_retries': 0, - 'relay.postgres_retry_exhausted': 0, - 'cell.production-gce-c1.connections': 1, - 'cell.production-gce-c1.queued_bytes': 0 - }), - 'director-admin': source({ - 'cell.production-gce-c1.admission_state': 0, - 'cell.production-gce-c1.heartbeat_fresh': 1, - 'cell.production-gce-c1.heartbeat_age_ms': 1, - 'cell.production-gce-c1.migration_blocked': 0, - 'cell.production-gce-c1.migration_target_inactive': 0 - }) - } - } -} - -const calm: PreDrainHardRuleReadings = { - cellProcessExits: 0, - unattributedExitInstances: [], - director503PeakPerMinute: 40, - directorConcurrencyP99: 20 -} - -// A fake clock the sampler's own waits advance, so a window runs in no real time. -function harness(input: { - windowMinutes: number - sampleAt?: (index: number, at: number) => IncidentSample | Error - hardRulesAt?: (index: number) => PreDrainHardRuleReadings | Error -}) { - let clock = Date.parse('2026-10-01T12:00:00.000Z') - let samples = 0 - let reads = 0 - const run = runPreDrainSample({ - windowMinutes: input.windowMinutes, - now: () => clock, - wait: async (ms) => { - clock += ms - }, - collect: async () => { - const result = input.sampleAt?.(samples, clock) ?? greenSample(clock) - samples += 1 - if (result instanceof Error) throw result - return result - }, - readHardRules: async () => { - const result = input.hardRulesAt?.(reads) ?? calm - reads += 1 - if (result instanceof Error) throw result - return result - } - }) - return { run, startedAt: clock, samples: () => samples } -} - -describe('pre-drain sample window', () => { - it('sizes the window to the hosts the drain will re-place', () => { - expect(preDrainSampleWindowMinutes(0)).toBe(3) - expect(preDrainSampleWindowMinutes(150)).toBe(3) - expect(preDrainSampleWindowMinutes(500)).toBe(3) - expect(preDrainSampleWindowMinutes(501)).toBe(5) - expect(preDrainSampleWindowMinutes(1_500)).toBe(5) - expect(preDrainSampleWindowMinutes(1_501)).toBe(8) - expect(preDrainSampleWindowMinutes(2_750)).toBe(8) - }) - - it('refuses a host count it cannot size', () => { - for (const hosts of [-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY]) { - expect(() => preDrainSampleWindowMinutes(hosts)).toThrow('host count is invalid') - } - }) -}) - -describe('cell exit attribution', () => { - const scope = { - targetCellId: 'production-gce-c25', - placementCellIds: new Set(['production-gce-c25', 'production-gce-c26', 'production-gce-c17']), - configuredCellIds: new Set([ - 'production-gce-c5', 'production-gce-c17', 'production-gce-c25', 'production-gce-c26' - ]) - } - const attribute = (exits: Record, cells: Record) => - attributeCellExits({ - exitsByInstance: new Map(Object.entries(exits)), - cellByInstance: new Map(Object.entries(cells)), - ...scope - }) - - it('ignores the target cell rolling the fix for its own crashes', () => { - expect(attribute({ '25': 12 }, { '25': 'production-gce-c25' })) - .toEqual({ counted: 0, unattributed: [] }) - }) - - it('counts an exit on another general or migration-only cell', () => { - expect(attribute({ '26': 1 }, { '26': 'production-gce-c26' })) - .toEqual({ counted: 1, unattributed: [] }) - expect(attribute({ '17': 2 }, { '17': 'production-gce-c17' })) - .toEqual({ counted: 2, unattributed: [] }) - }) - - it('ignores an existing-only legacy cell, which takes no placements', () => { - expect(attribute({ '5': 15 }, { '5': 'production-gce-c5' })) - .toEqual({ counted: 0, unattributed: [] }) - }) - - it('reports an instance with no cell, or an unknown cell, as unattributed', () => { - expect(attribute({ '9': 1, '8': 1 }, { '9': null, '8': 'production-gce-c99' })) - .toEqual({ counted: 0, unattributed: ['8', '9'] }) - expect(attribute({ '7': 1 }, {})).toEqual({ counted: 0, unattributed: ['7'] }) - }) - - it('trips the rule through the evaluator for a counted exit', () => { - const { counted } = attribute({ '26': 1, '25': 3, '5': 4 }, { - '26': 'production-gce-c26', '25': 'production-gce-c25', '5': 'production-gce-c5' - }) - expect(evaluatePreDrainHardRules({ ...calm, cellProcessExits: counted })).toEqual([ - expect.objectContaining({ signal: 'cells.process_exits_10m', observed: 1 }) - ]) - }) -}) - -describe('pre-drain hard rules', () => { - it('passes a calm fleet', () => { - expect(evaluatePreDrainHardRules(calm)).toEqual([]) - }) - - it('trips on any cell container exit in the lookback', () => { - expect(evaluatePreDrainHardRules({ ...calm, cellProcessExits: 1 })).toEqual([ - expect.objectContaining({ signal: 'cells.process_exits_10m', observed: 1, threshold: 0 }) - ]) - }) - - it('trips on an exit no cell could be named for', () => { - expect(evaluatePreDrainHardRules({ ...calm, unattributedExitInstances: ['111', '222'] })) - .toEqual([expect.objectContaining({ - code: 'exit_unattributed', - signal: 'cells.process_exits_10m.instances=111+222', - observed: 2 - })]) - }) - - it('trips on a disconnect pulse, not on a busy healthy minute', () => { - expect(evaluatePreDrainHardRules({ ...calm, director503PeakPerMinute: 500 })).toEqual([]) - expect(evaluatePreDrainHardRules({ ...calm, director503PeakPerMinute: 501 })).toEqual([ - expect.objectContaining({ signal: 'director.503_peak_minute_10m', observed: 501 }) - ]) - }) - - it('holds director concurrency p99 to the monitor threshold', () => { - expect(PRE_DRAIN_HARD_RULES.directorConcurrencyP99Max) - .toBe(INCIDENT_MONITOR_THRESHOLDS.directorConcurrency) - const max = PRE_DRAIN_HARD_RULES.directorConcurrencyP99Max - expect(evaluatePreDrainHardRules({ ...calm, directorConcurrencyP99: max })).toEqual([]) - // The 2026-10-01 13:42Z refusal. - expect(evaluatePreDrainHardRules({ ...calm, directorConcurrencyP99: 66.8 })).toEqual([ - expect.objectContaining({ code: 'threshold_max', signal: 'director.concurrency_p99_4m' }) - ]) - }) - - it('reads an unpublished concurrency series as missing, never as calm', () => { - expect(evaluatePreDrainHardRules({ ...calm, directorConcurrencyP99: null })).toEqual([ - expect.objectContaining({ code: 'signal_missing', signal: 'director.concurrency_p99_4m' }) - ]) - }) -}) - -describe('runPreDrainSample', () => { - it('passes after one sample per minute across the whole window', async () => { - for (const windowMinutes of [3, 5, 8]) { - const { run, startedAt, samples } = harness({ windowMinutes }) - const result = await run - expect(samples()).toBe(windowMinutes + 1) - expect(Date.parse(result.completedAt) - startedAt).toBeGreaterThanOrEqual( - windowMinutes * 60_000 - ) - } - }) - - it('refuses a window that is not one of the sized bands', async () => { - await expect(harness({ windowMinutes: 15 }).run).rejects.toThrow('window is invalid') - }) - - it('trips at once on a hard rule, before the window ends', async () => { - const { run, samples } = harness({ - windowMinutes: 8, - hardRulesAt: (index) => (index === 2 ? { ...calm, cellProcessExits: 1 } : calm) - }) - await expect(run).rejects.toBeInstanceOf(PreDrainSampleTripped) - await expect(run).rejects.toThrow('cells.process_exits_10m') - expect(samples()).toBe(3) - }) - - it('trips at once on a monitor threshold the monitor never tolerates', async () => { - const { run } = harness({ - windowMinutes: 3, - sampleAt: (index, at) => { - const next = greenSample(at) - if (index === 1) { - next.sources['cloud-monitoring']!.signals['director.concurrency'] = { - value: 80, - observedAt: new Date(at).toISOString() - } - } - return next - } - }) - await expect(run).rejects.toThrow('director.concurrency observed=80') - }) - - it('rides out a cell probe within the monitor tolerance and trips past it', async () => { - const failingProbe = (failFor: number) => (index: number, at: number) => { - const next = greenSample(at) - if (index >= 1 && index < 1 + failFor) { - next.sources['active-probe']!.signals['cell.production-gce-c1.health'] = { - value: 0, - observedAt: new Date(at).toISOString() - } - } - return next - } - const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - await expect( - harness({ windowMinutes: 3, sampleAt: failingProbe(tolerance) }).run - ).resolves.toBeDefined() - await expect( - harness({ windowMinutes: 3, sampleAt: failingProbe(tolerance + 1) }).run - ).rejects.toThrow('cell.production-gce-c1.health') - }) - - it('rides out failed reads within the freshness tolerance and trips past it', async () => { - const failingReads = (failFor: number) => (index: number) => - index >= 1 && index < 1 + failFor ? new Error('Cloud Monitoring returned 503') : calm - await expect( - harness({ - windowMinutes: 3, - hardRulesAt: failingReads(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES) - }).run - ).resolves.toBeDefined() - await expect( - harness({ - windowMinutes: 3, - hardRulesAt: failingReads(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1) - }).run - ).rejects.toThrow('collector_failed pre-drain.hard-rules') - }) - - // Why: the drain starts right after this returns, so it must not start on a reading the - // tolerance was still carrying. - it('keeps sampling past the window until a sample is clean', async () => { - const { run, samples } = harness({ - windowMinutes: 3, - hardRulesAt: (index) => (index === 3 ? { ...calm, directorConcurrencyP99: null } : calm) - }) - const result = await run - expect(samples()).toBe(5) - expect(result.samples.at(-1)?.failures).toEqual([]) - }) - - // Why: two tolerated readings that alternate never build a streak, so only the overrun cap - // stops them from holding the step open until its timeout. - it('trips when alternating tolerated readings never leave a clean sample', async () => { - const { run, samples } = harness({ - windowMinutes: 3, - hardRulesAt: (index) => - index >= 3 && index % 2 === 1 ? { ...calm, directorConcurrencyP99: null } : calm, - sampleAt: (index, at) => - index >= 3 && index % 2 === 0 ? new Error('collector down') : greenSample(at) - }) - await expect(run).rejects.toBeInstanceOf(PreDrainSampleTripped) - expect(samples()).toBe(4 + PRE_DRAIN_MAX_OVERRUN_SAMPLES) - }) - - it('records why a read failed', async () => { - const records: string[][] = [] - let clock = 0 - await runPreDrainSample({ - windowMinutes: 3, - now: () => clock, - wait: async (ms) => { clock += ms }, - collect: async () => greenSample(clock), - readHardRules: async () => { - if (records.length === 1) throw new Error('Google telemetry returned 403') - return calm - }, - log: (record) => records.push(record.errors) - }) - expect(records[1]).toEqual(['hard rules: Google telemetry returned 403']) - }) -}) diff --git a/cloud/apps/relay-ops/src/pre-drain-sample.ts b/cloud/apps/relay-ops/src/pre-drain-sample.ts deleted file mode 100644 index 0ebce1d7dae..00000000000 --- a/cloud/apps/relay-ops/src/pre-drain-sample.ts +++ /dev/null @@ -1,259 +0,0 @@ -import { - describeIncidentFailure, - evaluateIncidentSample, - FRESHNESS_FAILURE_CODES, - INCIDENT_FRESHNESS_TOLERANCE_SAMPLES, - INCIDENT_MONITOR_THRESHOLDS, - toleratedStreakKey, - type IncidentFailure, - type IncidentSample -} from './incident-monitor.js' - -// Inline fleet-health sample a same-cap wave takes right before it isolates its cell. It judges -// every sample with the monitor's own evaluator and tolerances; what it adds is a window sized to -// the drain and three lookback rules no single sample can see. - -export const PRE_DRAIN_SAMPLE_INTERVAL_MS = 60_000 - -// A bigger drain re-places more hosts through the director's single sticky lane, so it gets a -// longer quiet window before it starts. Ordered; the first band whose ceiling covers the cell wins. -export const PRE_DRAIN_SAMPLE_WINDOWS = [ - { maxHosts: 500, minutes: 3 }, - { maxHosts: 1_500, minutes: 5 }, - { maxHosts: Number.POSITIVE_INFINITY, minutes: 8 } -] as const - -export const PRE_DRAIN_HARD_RULES = { - // Any container exit on a cell that takes placements, other than the cell being rolled; a drain - // into a crash loop re-places its hosts twice. - cellProcessExitLookbackMs: 10 * 60_000, - cellProcessExitsMax: 0, - // A disconnect pulse lands as thousands of director 503s in one minute (7,750 and 13,000 on - // 2026-09-20 and 2026-10-01) against a healthy 20-70. - director503LookbackMs: 10 * 60_000, - director503PerMinuteMax: 500, - // The live preflight reads only the newest concurrency point; this holds the last four minutes. - directorConcurrencyLookbackMs: 4 * 60_000, - directorConcurrencyP99Max: INCIDENT_MONITOR_THRESHOLDS.directorConcurrency -} as const - -export type PreDrainHardRuleReadings = { - // Exits on general or migration-only cells other than the target. - cellProcessExits: number - // Exits on an instance no cell could be resolved for; never assumed harmless. - unattributedExitInstances: string[] - director503PeakPerMinute: number - // Null when Cloud Monitoring published no point in the lookback. - directorConcurrencyP99: number | null -} - -// Which exits count. The target's own exits are why it is being rolled (c25 crashed 12 times in -// a week and is the first US roll), and existing-only legacy cells take no placements (c5 alone -// exited 15 times in that week), so neither can stop a drain. An instance with no known cell -// counts as unattributed and trips the rule. -export function attributeCellExits(input: { - exitsByInstance: ReadonlyMap - cellByInstance: ReadonlyMap - targetCellId: string - placementCellIds: ReadonlySet - configuredCellIds: ReadonlySet -}): { counted: number; unattributed: string[] } { - let counted = 0 - const unattributed: string[] = [] - for (const [instanceId, exits] of input.exitsByInstance) { - if (exits <= 0) continue - const cellId = input.cellByInstance.get(instanceId) ?? null - if (cellId === null || !input.configuredCellIds.has(cellId)) { - unattributed.push(instanceId) - continue - } - if (cellId === input.targetCellId || !input.placementCellIds.has(cellId)) continue - counted += exits - } - return { counted, unattributed: unattributed.sort() } -} - -export function preDrainSampleWindowMinutes(hostCount: number): number { - if (!Number.isSafeInteger(hostCount) || hostCount < 0) { - throw new Error('pre-drain sample host count is invalid') - } - const band = PRE_DRAIN_SAMPLE_WINDOWS.find((entry) => hostCount <= entry.maxHosts) - if (!band) throw new Error('pre-drain sample host count is invalid') - return band.minutes -} - -export function evaluatePreDrainHardRules(readings: PreDrainHardRuleReadings): IncidentFailure[] { - const failures: IncidentFailure[] = [] - if (readings.cellProcessExits > PRE_DRAIN_HARD_RULES.cellProcessExitsMax) { - failures.push({ - code: 'threshold_max', - source: 'cloud-monitoring', - signal: 'cells.process_exits_10m', - observed: readings.cellProcessExits, - threshold: PRE_DRAIN_HARD_RULES.cellProcessExitsMax - }) - } - if (readings.unattributedExitInstances.length > 0) { - failures.push({ - code: 'exit_unattributed', - source: 'cloud-monitoring', - signal: `cells.process_exits_10m.instances=${readings.unattributedExitInstances.join('+')}`, - observed: readings.unattributedExitInstances.length, - threshold: 0 - }) - } - if (readings.director503PeakPerMinute > PRE_DRAIN_HARD_RULES.director503PerMinuteMax) { - failures.push({ - code: 'threshold_max', - source: 'cloud-monitoring', - signal: 'director.503_peak_minute_10m', - observed: readings.director503PeakPerMinute, - threshold: PRE_DRAIN_HARD_RULES.director503PerMinuteMax - }) - } - if (readings.directorConcurrencyP99 === null) { - failures.push({ - code: 'signal_missing', - source: 'cloud-monitoring', - signal: 'director.concurrency_p99_4m' - }) - } else if (readings.directorConcurrencyP99 > PRE_DRAIN_HARD_RULES.directorConcurrencyP99Max) { - failures.push({ - code: 'threshold_max', - source: 'cloud-monitoring', - signal: 'director.concurrency_p99_4m', - observed: readings.directorConcurrencyP99, - threshold: PRE_DRAIN_HARD_RULES.directorConcurrencyP99Max - }) - } - return failures -} - -export type PreDrainSampleRecord = { - at: string - failures: IncidentFailure[] - // Why a read failed; a failure code alone hides a permission error or a bad filter. - errors: string[] -} - -// Samples past the window that may still carry a tolerated failure; alternating tolerated readings -// (cell X, then cell Y) never trip a streak, so this is what bounds the overrun. -export const PRE_DRAIN_MAX_OVERRUN_SAMPLES = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 - -export type PreDrainSampleResult = { - windowMinutes: number - startedAt: string - completedAt: string - samples: PreDrainSampleRecord[] -} - -export class PreDrainSampleTripped extends Error { - constructor(readonly failures: IncidentFailure[], readonly samples: PreDrainSampleRecord[]) { - super(`relay pre-drain sample tripped: ${failures.map(describeIncidentFailure).join(',')}`) - } -} - -const CONTINUITY_CODES = new Set(['collector_failed', ...FRESHNESS_FAILURE_CODES]) - -function continuityKey(failure: IncidentFailure): string { - return `${failure.source}/${failure.signal ?? '*'}` -} - -function bumpStreaks(streaks: Map, keys: Set): Map { - const next = new Map() - for (const key of keys) next.set(key, (streaks.get(key) ?? 0) + 1) - return next -} - -/** - * Samples once a minute until the window has elapsed, then keeps going until a sample comes back - * with no failure at all, so the drain never starts on an open reading. A breach of any - * non-tolerated threshold or hard rule trips at once; readings the monitor tolerates (a cell probe - * or the director instance count, an unread signal, a failed collector round trip) trip only past - * the monitor's own consecutive-sample budget. The overrun is capped separately. - */ -export async function runPreDrainSample(input: { - windowMinutes: number - collect: () => Promise - readHardRules: () => Promise - now?: () => number - wait?: (ms: number) => Promise - log?: (record: PreDrainSampleRecord) => void -}): Promise { - if (!PRE_DRAIN_SAMPLE_WINDOWS.some((band) => band.minutes === input.windowMinutes)) { - throw new Error('pre-drain sample window is invalid') - } - const now = input.now ?? Date.now - const wait = input.wait ?? ((ms: number) => new Promise((resolveWait) => { - setTimeout(resolveWait, ms) - })) - const startedMs = now() - const windowMs = input.windowMinutes * 60_000 - const samples: PreDrainSampleRecord[] = [] - let continuityStreaks = new Map() - let probeStreaks = new Map() - let overrunSamples = 0 - for (;;) { - const sampleStartedMs = now() - const failures: IncidentFailure[] = [] - const errors: string[] = [] - const reason = (error: unknown) => error instanceof Error ? error.message : String(error) - try { - failures.push( - ...evaluateIncidentSample(await input.collect(), now(), 'strict', null, null).failures - ) - } catch (error) { - failures.push({ code: 'collector_failed', source: 'cloud-monitoring' }) - errors.push(`collector: ${reason(error)}`) - } - try { - failures.push(...evaluatePreDrainHardRules(await input.readHardRules())) - } catch (error) { - failures.push({ - code: 'collector_failed', - source: 'cloud-monitoring', - signal: 'pre-drain.hard-rules' - }) - errors.push(`hard rules: ${reason(error)}`) - } - const record = { at: new Date(now()).toISOString(), failures, errors } - samples.push(record) - input.log?.(record) - const continuity = failures.filter((failure) => CONTINUITY_CODES.has(failure.code)) - const breaches = failures.filter((failure) => !CONTINUITY_CODES.has(failure.code)) - continuityStreaks = bumpStreaks(continuityStreaks, new Set(continuity.map(continuityKey))) - const probeKeys = new Set() - for (const failure of breaches) { - const key = toleratedStreakKey(failure) - if (key !== null) probeKeys.add(key) - } - probeStreaks = bumpStreaks(probeStreaks, probeKeys) - const tripped = [ - ...breaches.filter((failure) => { - const key = toleratedStreakKey(failure) - return key === null || - (probeStreaks.get(key) ?? 0) > INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples - }), - ...continuity.filter((failure) => - (continuityStreaks.get(continuityKey(failure)) ?? 0) > - INCIDENT_FRESHNESS_TOLERANCE_SAMPLES) - ] - if (tripped.length > 0) throw new PreDrainSampleTripped(tripped, samples) - const windowDone = now() - startedMs >= windowMs - if (windowDone && failures.length > 0) { - overrunSamples += 1 - if (overrunSamples > PRE_DRAIN_MAX_OVERRUN_SAMPLES) { - throw new PreDrainSampleTripped(failures, samples) - } - } - if (windowDone && failures.length === 0) { - return { - windowMinutes: input.windowMinutes, - startedAt: new Date(startedMs).toISOString(), - completedAt: record.at, - samples - } - } - await wait(Math.max(0, sampleStartedMs + PRE_DRAIN_SAMPLE_INTERVAL_MS - now())) - } -} diff --git a/cloud/apps/relay-ops/src/relay-repository.test.ts b/cloud/apps/relay-ops/src/relay-repository.test.ts deleted file mode 100644 index e6a87c35e82..00000000000 --- a/cloud/apps/relay-ops/src/relay-repository.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { readdirSync, readFileSync } from 'node:fs' -import { fileURLToPath } from 'node:url' -import { describe, expect, it } from 'vitest' -import { - RELAY_GITHUB_REPOSITORY, - RELAY_WORKFLOW_FILE_PREFIX, - relayRepositoryApiPath, - relayWorkflowFile -} from './relay-repository.js' - -const sourceDir = fileURLToPath(new URL('.', import.meta.url)) -const sources = readdirSync(sourceDir) - .filter((name) => name.endsWith('.ts') && name !== 'relay-repository.ts') - .map((name) => ({ name, text: readFileSync(`${sourceDir}${name}`, 'utf8') })) - -describe('relay repository identity', () => { - it('builds API paths and workflow filenames from the one repository name', () => { - expect(relayRepositoryApiPath('actions/runs')).toBe(`repos/${RELAY_GITHUB_REPOSITORY}/actions/runs`) - expect(relayWorkflowFile('power-relay-staging.yml')).toBe( - `${RELAY_WORKFLOW_FILE_PREFIX}power-relay-staging.yml` - ) - }) - - // Why: the public-repo copy renames the repository and prefixes every workflow file. Both have to - // be one edit, so no other module may restate either. - it('is the only module naming a GitHub repository', () => { - for (const { name, text } of sources) { - expect(text, `${name} restates a GitHub repository`).not.toMatch(/stablyai\//) - } - }) - - it('is the only module naming a workflow file', () => { - for (const { name, text } of sources) { - for (const match of text.matchAll(/'([^']*\.yml)'/g)) { - const file = match[1] ?? '' - expect(text, `${name} names ${file} outside relayWorkflowFile`).toMatch( - new RegExp(`relayWorkflowFile\\('${file.replaceAll('.', '\\.')}'\\)`) - ) - } - } - }) -}) diff --git a/cloud/apps/relay-ops/src/relay-repository.ts b/cloud/apps/relay-ops/src/relay-repository.ts deleted file mode 100644 index 85158670adb..00000000000 --- a/cloud/apps/relay-ops/src/relay-repository.ts +++ /dev/null @@ -1,14 +0,0 @@ -// Single place naming the GitHub repository that holds the Relay workflows. When the Relay tree is -// copied to its public repository, only this file changes: the repository moves and every workflow -// file gains a prefix, while the workflow display names stay as they are. -export const RELAY_GITHUB_REPOSITORY = 'stablyai/orca-cloud' - -export const RELAY_WORKFLOW_FILE_PREFIX = '' - -export function relayWorkflowFile(name: string): string { - return `${RELAY_WORKFLOW_FILE_PREFIX}${name}` -} - -export function relayRepositoryApiPath(resource: string): string { - return `repos/${RELAY_GITHUB_REPOSITORY}/${resource}` -} diff --git a/cloud/apps/relay-ops/src/resource-inventory.test.ts b/cloud/apps/relay-ops/src/resource-inventory.test.ts deleted file mode 100644 index e2cfa13dccb..00000000000 --- a/cloud/apps/relay-ops/src/resource-inventory.test.ts +++ /dev/null @@ -1,370 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js' -import type { GcloudClient } from './gcloud-client.js' -import { probeEndpointHealth, readResourceInventory } from './resource-inventory.js' - -const digest = `sha256:${'a'.repeat(64)}` -const runService = { - template: { - scaling: { minInstanceCount: 0, maxInstanceCount: 2 }, - containers: [{ image: 'registry/image:tag' }] - }, - conditions: [{ state: 'CONDITION_SUCCEEDED' }], - latestReadyRevision: 'projects/project/revisions/revision-one' -} - -const sleepingStagingGcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) } - -// Staging's Cloud SQL is stopped, so this inventory reads REST only and probes no endpoint. -type MigOutcome = 'ok' | 'throw' | 'missing' -const sleepingStagingFetch = (migOutcome: (migName: string) => MigOutcome): typeof fetch => - async (input) => { - const url = new URL(String(input)) - if (url.hostname === 'run.googleapis.com') return Response.json(runService) - if (url.hostname === 'sqladmin.googleapis.com') return Response.json({ - state: 'STOPPED', - databaseVersion: 'POSTGRES_17', - settings: { activationPolicy: 'NEVER', availabilityType: 'ZONAL', tier: 'db-custom-1-3840' } - }) - if (url.hostname === 'certificatemanager.googleapis.com') return Response.json({ - managed: { domains: ['*.relay-staging.onorca.dev'], state: 'ACTIVE' } - }) - if (url.pathname.includes('/instanceGroupManagers/')) { - const name = url.pathname.split('/').at(-1)! - const outcome = migOutcome(name) - if (outcome === 'throw') throw new TypeError('fetch failed') - if (outcome === 'missing') return new Response(null, { status: 404 }) - return Response.json({ - name, - targetSize: 0, - size: '0', - instanceGroup: `projects/project/zones/zone/instanceGroups/${name}`, - instanceTemplate: `projects/project/global/instanceTemplates/template-${name}`, - status: { isStable: true } - }) - } - if (url.pathname.includes('/instanceTemplates/')) return Response.json({ properties: {} }) - if (url.pathname.endsWith('/getHealth')) return Response.json([]) - throw new Error(`Unexpected request to ${url.hostname}${url.pathname}`) - } - -describe('readResourceInventory', () => { - it('does not delay a healthy endpoint sample', async () => { - let calls = 0 - let waits = 0 - const result = await probeEndpointHealth( - 'https://c9.relay.onorca.dev', - async () => { - calls += 1 - return new Response(null, { status: 200 }) - }, - { - wait: async () => { - waits += 1 - } - } - ) - - expect(result.health).toBe(true) - expect(result.ready).toBe(true) - expect(calls).toBe(2) - expect(waits).toBe(0) - }) - - it('retries one transient endpoint failure within the same sample', async () => { - const calls = new Map() - const waits: number[] = [] - const result = await probeEndpointHealth( - 'https://c9.relay.onorca.dev', - async (input) => { - const path = new URL(String(input)).pathname - const call = (calls.get(path) ?? 0) + 1 - calls.set(path, call) - return new Response(null, { status: path === '/ready' && call === 1 ? 503 : 200 }) - }, - { - wait: async (ms) => { - waits.push(ms) - } - } - ) - - expect(result.health).toBe(true) - expect(result.ready).toBe(true) - expect(calls).toEqual(new Map([['/health', 2], ['/ready', 2]])) - expect(waits).toEqual([11_000]) - }) - - it('fails closed when the endpoint retry is also unhealthy', async () => { - let calls = 0 - const waits: number[] = [] - const result = await probeEndpointHealth( - 'https://c9.relay.onorca.dev', - async () => { - calls += 1 - return new Response(null, { status: 503 }) - }, - { - wait: async (ms) => { - waits.push(ms) - } - } - ) - - expect(result.health).toBe(false) - expect(result.ready).toBe(false) - expect(calls).toBe(4) - // A refusing endpoint is a reading, so only the independent retry runs. - expect(waits).toEqual([11_000]) - }) - - it('treats a thrown fetch as no reading and re-asks that path once', async () => { - const calls: string[] = [] - const waits: number[] = [] - const result = await probeEndpointHealth( - 'https://c9.relay.onorca.dev', - async (input) => { - const path = new URL(String(input)).pathname - calls.push(path) - if (path === '/health' && calls.filter((call) => call === '/health').length === 1) { - throw new TypeError('fetch failed') - } - return new Response(null, { status: 200 }) - }, - { - wait: async (ms) => { - waits.push(ms) - } - } - ) - - expect(result.health).toBe(true) - expect(result.ready).toBe(true) - expect(calls.filter((call) => call === '/health')).toEqual(['/health', '/health']) - expect(waits).toEqual([1_000]) - }) - - it('fails closed when both attempts of a path throw', async () => { - const calls: string[] = [] - const waits: number[] = [] - const result = await probeEndpointHealth( - 'https://c9.relay.onorca.dev', - async (input) => { - const path = new URL(String(input)).pathname - calls.push(path) - if (path === '/health') throw new TypeError('fetch failed') - return new Response(null, { status: 200 }) - }, - { - wait: async (ms) => { - waits.push(ms) - } - } - ) - - expect(result.health).toBe(false) - expect(calls.filter((call) => call === '/health')).toHaveLength(4) - expect(waits).toEqual([1_000, 11_000, 1_000]) - }) - - it('accepts an auth-shaped endpoint that serves no readiness path', async () => { - const calls: string[] = [] - const waits: number[] = [] - const result = await probeEndpointHealth( - 'https://login.onorca.dev', - async (input) => { - const path = new URL(String(input)).pathname - calls.push(path) - return new Response(null, { status: path === '/ready' ? 404 : 200 }) - }, - { - requiresReady: false, - wait: async (ms) => { - waits.push(ms) - } - } - ) - - expect(result.health).toBe(true) - expect(result.ready).toBeNull() - expect(calls).toEqual(['/health']) - expect(waits).toEqual([]) - }) - - it('still requires readiness for the director and cells', async () => { - const waits: number[] = [] - const result = await probeEndpointHealth( - 'https://relay.onorca.dev', - async (input) => new Response(null, { - status: new URL(String(input)).pathname === '/ready' ? 503 : 200 - }), - { - wait: async (ms) => { - waits.push(ms) - } - } - ) - - expect(result.health).toBe(true) - expect(result.ready).toBe(false) - expect(waits).toEqual([11_000]) - }) - - it('measures latency as the answering round trip, not the retry delay', async () => { - let healthCalls = 0 - const result = await probeEndpointHealth( - 'https://c9.relay.onorca.dev', - async (input) => { - if (new URL(String(input)).pathname !== '/health') return new Response(null, { status: 200 }) - healthCalls += 1 - if (healthCalls === 1) throw new TypeError('fetch failed') - return new Response(null, { status: 200 }) - }, - { wait: async (ms) => await new Promise((resolve) => setTimeout(resolve, Math.min(ms, 60))) } - ) - - expect(result.health).toBe(true) - expect(result.latencyMs).not.toBeNull() - expect(result.latencyMs!).toBeLessThan(60) - }) - - it('uses aggregate REST inventory without probing sleeping staging endpoints', async () => { - const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) } - let publicProbeCalls = 0 - const fetchImpl: typeof fetch = async (input) => { - const url = new URL(String(input)) - if (url.hostname.endsWith('onorca.dev')) { - publicProbeCalls += 1 - return Response.json({ status: 'ok' }) - } - if (url.hostname === 'run.googleapis.com') return Response.json(runService) - if (url.hostname === 'sqladmin.googleapis.com') return Response.json({ - state: 'STOPPED', - databaseVersion: 'POSTGRES_17', - settings: { - activationPolicy: 'NEVER', - availabilityType: 'ZONAL', - tier: 'db-custom-1-3840' - } - }) - if (url.hostname === 'certificatemanager.googleapis.com') return Response.json({ - managed: { domains: ['*.relay-staging.onorca.dev'], state: 'ACTIVE' } - }) - if (url.pathname.includes('/instanceGroupManagers/')) { - const name = url.pathname.split('/').at(-1)! - return Response.json({ - name, - targetSize: 0, - size: '0', - instanceGroup: `projects/project/zones/zone/instanceGroups/${name}`, - instanceTemplate: `projects/project/global/instanceTemplates/template-${name}`, - status: { isStable: true } - }) - } - if (url.pathname.includes('/instanceTemplates/')) return Response.json({ - properties: { metadata: { items: [{ - key: 'startup-script', - value: `SECRET_TEXT\nORCA_RELAY_IMAGE_DIGEST=%s\\n' '${digest}'` - }] } } - }) - if (url.pathname.endsWith('/getHealth')) return Response.json([]) - throw new Error(`Unexpected request to ${url.hostname}${url.pathname}`) - } - - const result = await readResourceInventory( - RELAY_OPS_ENVIRONMENTS.staging, - gcloud, - fetchImpl - ) - - expect(publicProbeCalls).toBe(0) - expect(result.cells.every((cell) => cell.targetSize === 0)).toBe(true) - expect(result.cells.every((cell) => cell.endpoint.health === null)).toBe(true) - expect(result.cells.every((cell) => cell.imageDigest === digest)).toBe(true) - expect(JSON.stringify(result)).not.toContain('SECRET_TEXT') - }) - - it('re-asks a MIG read that failed once before calling a cell powered-unknown', async () => { - const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]! - const waits: number[] = [] - let parkedMigCalls = 0 - const result = await readResourceInventory( - RELAY_OPS_ENVIRONMENTS.staging, - sleepingStagingGcloud, - sleepingStagingFetch((migName) => { - if (!migName.endsWith(parkedCell.hostname)) return 'ok' - parkedMigCalls += 1 - return parkedMigCalls === 1 ? 'throw' : 'ok' - }), - { wait: async (ms) => { waits.push(ms) } } - ) - - const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)! - // The MIG was fine and parked at zero; one transient read must not erase that reading. - expect(parked.targetSize).toBe(0) - expect(parkedMigCalls).toBe(2) - expect(waits).toEqual([1_000]) - expect(result.warnings).toEqual([]) - }) - - it('reports a MIG unavailable only when the retry fails too', async () => { - const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]! - const waits: number[] = [] - let parkedMigCalls = 0 - const result = await readResourceInventory( - RELAY_OPS_ENVIRONMENTS.staging, - sleepingStagingGcloud, - sleepingStagingFetch((migName) => { - if (!migName.endsWith(parkedCell.hostname)) return 'ok' - parkedMigCalls += 1 - return 'throw' - }), - { wait: async (ms) => { waits.push(ms) } } - ) - - const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)! - expect(parked.targetSize).toBeNull() - expect(parked.backendHealth).toBe('unknown') - expect(parkedMigCalls).toBe(2) - expect(waits).toEqual([1_000]) - expect(result.warnings).toEqual([ - `${parkedCell.hostname.toUpperCase()} MIG inventory is unavailable.` - ]) - }) - - it('does not re-ask a MIG read the API answered with 404', async () => { - const missingCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]! - const waits: number[] = [] - let missingMigCalls = 0 - const result = await readResourceInventory( - RELAY_OPS_ENVIRONMENTS.staging, - sleepingStagingGcloud, - sleepingStagingFetch((migName) => { - if (!migName.endsWith(missingCell.hostname)) return 'ok' - missingMigCalls += 1 - return 'missing' - }), - { wait: async (ms) => { waits.push(ms) } } - ) - - expect(result.cells.find((cell) => cell.cellId === missingCell.cellId)!.targetSize).toBeNull() - expect(missingMigCalls).toBe(1) - expect(waits).toEqual([]) - }) - - it('represents missing credentials as unknown inventory, never sleeping', async () => { - const gcloud: GcloudClient = { - accessToken: async () => { throw new Error('sensitive context') } - } - let fetchCalls = 0 - const result = await readResourceInventory( - RELAY_OPS_ENVIRONMENTS.production, - gcloud, - async () => { fetchCalls += 1; return Response.json({}) } - ) - expect(fetchCalls).toBe(0) - expect(result.cells.every((cell) => cell.targetSize === null)).toBe(true) - expect(result.cells.every((cell) => cell.backendHealth === 'unknown')).toBe(true) - expect(JSON.stringify(result)).not.toContain('sensitive context') - }) -}) diff --git a/cloud/apps/relay-ops/src/resource-inventory.ts b/cloud/apps/relay-ops/src/resource-inventory.ts deleted file mode 100644 index da490685198..00000000000 --- a/cloud/apps/relay-ops/src/resource-inventory.ts +++ /dev/null @@ -1,437 +0,0 @@ -import { z } from 'zod' -import type { RelayOpsEnvironment, RelayOpsCellConfig } from './environment-config.js' -import type { GcloudClient } from './gcloud-client.js' -import { INCIDENT_MONITOR_THRESHOLDS } from './incident-monitor.js' - -const RunServiceSchema = z.object({ - template: z.object({ - scaling: z.object({ - minInstanceCount: z.number().optional(), - maxInstanceCount: z.number().optional() - }).optional(), - containers: z.array(z.object({ image: z.string() })).min(1) - }), - conditions: z.array(z.object({ state: z.string() })).default([]), - latestReadyRevision: z.string().optional() -}) - -const SqlInstanceSchema = z.object({ - state: z.string(), - databaseVersion: z.string(), - settings: z.object({ - activationPolicy: z.string(), - availabilityType: z.string().optional(), - tier: z.string() - }) -}) - -const MigSchema = z.object({ - name: z.string(), - targetSize: z.number(), - size: z.union([z.string(), z.number()]).transform(Number).optional(), - instanceGroup: z.string(), - instanceTemplate: z.string(), - status: z.object({ isStable: z.boolean().default(false) }).default({ isStable: false }) -}) - -const TemplateSchema = z.object({ - properties: z.object({ - metadata: z.object({ - items: z.array(z.object({ key: z.string(), value: z.string().optional() })).default([]) - }).optional() - }) -}) - -const BackendHealthGroupSchema = z.object({ - healthStatus: z.array(z.object({ healthState: z.string() })).default([]) -}) -const BackendHealthSchema = z.union([ - BackendHealthGroupSchema, - z.array(z.object({ status: BackendHealthGroupSchema })) -]) - -const CertificateSchema = z.object({ - expireTime: z.string().optional(), - managed: z.object({ - domains: z.array(z.string()).default([]), - state: z.string() - }) -}) - -export type EndpointHealth = { - health: boolean | null - ready: boolean | null - latencyMs: number | null -} - -export type ServiceInventory = { - ready: boolean - revision: string | null - image: string - minInstances: number - maxInstances: number -} - -export type CellInventory = RelayOpsCellConfig & { - migName: string - targetSize: number | null - runningInstances: number | null - stable: boolean | null - template: string | null - imageDigest: string | null - backendHealth: 'healthy' | 'unhealthy' | 'empty' | 'unknown' - endpoint: EndpointHealth -} - -export type ResourceInventory = { - director: ServiceInventory | null - auth: ServiceInventory | null - sql: { - state: string - activationPolicy: string - tier: string - availabilityType: string - databaseVersion: string - } | null - certificate: { state: string; domains: string[]; expireTime: string | null } | null - directorEndpoint: EndpointHealth - authEndpoint: EndpointHealth - cells: CellInventory[] - warnings: string[] -} - -const unavailableEndpoint = (): EndpointHealth => ({ health: null, ready: null, latencyMs: null }) -const independentEndpointRetryDelayMs = 11_000 -const transientProbeRetryDelayMs = 1_000 -const sleep = async (ms: number): Promise => - await new Promise((resolvePromise) => setTimeout(resolvePromise, ms)) - -function finalSegment(value: string): string { - return value.split('/').at(-1) ?? value -} - -function parseService(value: unknown): ServiceInventory { - const service = RunServiceSchema.parse(value) - return { - ready: service.conditions.length > 0 && service.conditions.every( - (condition) => condition.state === 'CONDITION_SUCCEEDED' - ), - revision: service.latestReadyRevision ? finalSegment(service.latestReadyRevision) : null, - image: service.template.containers[0]!.image, - minInstances: service.template.scaling?.minInstanceCount ?? 0, - maxInstances: service.template.scaling?.maxInstanceCount ?? 0 - } -} - -class GoogleApiError extends Error { - constructor(readonly status: number) { - super(`Google API returned ${status}`) - } -} - -async function googleRequest( - fetchImpl: typeof fetch, - token: string, - url: string, - init: RequestInit = {} -): Promise { - const response = await fetchImpl(url, { - ...init, - headers: { - authorization: `Bearer ${token}`, - ...(init.body ? { 'content-type': 'application/json' } : {}) - }, - signal: AbortSignal.timeout(30_000) - }) - if (!response.ok) throw new GoogleApiError(response.status) - return await response.json() -} - -// A 404 is the API's answer about the resource; anything else is the absence of a reading, so re-ask. -async function readOnceMore( - read: () => Promise, - wait: (ms: number) => Promise -): Promise { - try { - return await read() - } catch (error) { - if (error instanceof GoogleApiError && error.status === 404) throw error - await wait(transientProbeRetryDelayMs) - return await read() - } -} - -// A reading the endpoint actually produced: ok is its answer, latencyMs is that answer's round trip. -type PathReading = { ok: boolean; latencyMs: number | null } - -async function probePath( - origin: string, - path: '/health' | '/ready', - fetchImpl: typeof fetch, - wait: (ms: number) => Promise -): Promise { - // null means the request never produced an answer (DNS/TCP/TLS failure or the 8s abort). - const attempt = async (): Promise => { - const startedAt = performance.now() - try { - const response = await fetchImpl(`${origin}${path}`, { - redirect: 'error', - signal: AbortSignal.timeout(8_000) - }) - return { ok: response.ok, latencyMs: Math.round(performance.now() - startedAt) } - } catch { - return null - } - } - const first = await attempt() - if (first) return first - // A thrown fetch is the absence of a reading, not an unhealthy answer, so re-ask before concluding. - await wait(transientProbeRetryDelayMs) - return (await attempt()) ?? { ok: false, latencyMs: null } -} - -async function endpointProbe( - origin: string, - fetchImpl: typeof fetch, - requiresReady: boolean, - wait: (ms: number) => Promise -): Promise { - const [health, ready] = await Promise.all([ - probePath(origin, '/health', fetchImpl, wait), - requiresReady ? probePath(origin, '/ready', fetchImpl, wait) : null - ]) - // Latency is the slowest answering round trip in this probe; retry delays are not serving latency. - const latencies = [health.latencyMs, ready?.latencyMs ?? null].filter( - (value): value is number => value !== null - ) - return { - health: health.ok, - ready: ready ? ready.ok : null, - latencyMs: latencies.length > 0 ? Math.max(...latencies) : null - } -} - -export type EndpointProbeOptions = { - // Auth serves no /ready by design, so it is judged on /health and latency alone. - requiresReady?: boolean - wait?: (ms: number) => Promise -} - -export async function probeEndpointHealth( - origin: string, - fetchImpl: typeof fetch, - options: EndpointProbeOptions = {} -): Promise { - const requiresReady = options.requiresReady ?? true - const wait = options.wait ?? sleep - const accepted = (probe: EndpointHealth): boolean => - probe.health === true && - (!requiresReady || probe.ready === true) && - probe.latencyMs !== null && - probe.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs - const first = await endpointProbe(origin, fetchImpl, requiresReady, wait) - if (accepted(first)) return first - // Outwait Relay's ten-second readiness cache before treating the retry as independent. - await wait(independentEndpointRetryDelayMs) - return await endpointProbe(origin, fetchImpl, requiresReady, wait) -} - -function imageDigest(template: z.infer): string | null { - const startupScript = template.properties.metadata?.items.find( - (item) => item.key === 'startup-script' - )?.value - // Return only the immutable digest; startup metadata contains secret names and operational detail. - return startupScript?.match(/ORCA_RELAY_IMAGE_DIGEST=%s\\n' '(sha256:[a-f0-9]{64})'/)?.[1] ?? null -} - -function backendState(value: unknown): CellInventory['backendHealth'] { - const parsedHealth = BackendHealthSchema.parse(value) - const groups = Array.isArray(parsedHealth) - ? parsedHealth.map((group) => group.status) - : [parsedHealth] - const states = groups.flatMap((group) => group.healthStatus.map((status) => status.healthState)) - if (states.length === 0) return 'empty' - if (states.every((state) => state === 'HEALTHY')) return 'healthy' - return 'unhealthy' -} - -function unavailableCell(cell: RelayOpsCellConfig, migName: string): CellInventory { - return { - ...cell, - migName, - targetSize: null, - runningInstances: null, - stable: null, - template: null, - imageDigest: null, - backendHealth: 'unknown', - endpoint: unavailableEndpoint() - } -} - -async function readCell( - environment: RelayOpsEnvironment, - cell: RelayOpsCellConfig, - mig: z.infer | null, - token: string, - fetchImpl: typeof fetch -): Promise { - const migName = `${environment.migPrefix}${cell.hostname}` - if (!mig) return unavailableCell(cell, migName) - // An empty fixed-one MIG cannot serve and must never be woken by observation. - const endpoint = mig.targetSize > 0 - ? await probeEndpointHealth(cell.origin, fetchImpl) - : unavailableEndpoint() - const templateName = finalSegment(mig.instanceTemplate) - const [templateResult, healthResult] = await Promise.allSettled([ - googleRequest( - fetchImpl, - token, - `https://compute.googleapis.com/compute/v1/projects/${environment.project}/global/instanceTemplates/${templateName}` - ), - googleRequest( - fetchImpl, - token, - `https://compute.googleapis.com/compute/v1/projects/${environment.project}/global/backendServices/${migName}/getHealth`, - { method: 'POST', body: JSON.stringify({ group: mig.instanceGroup }) } - ) - ]) - return { - ...cell, - migName, - targetSize: mig.targetSize, - runningInstances: mig.size ?? (mig.status.isStable ? mig.targetSize : null), - stable: mig.status.isStable, - template: templateName, - imageDigest: templateResult.status === 'fulfilled' - ? imageDigest(TemplateSchema.parse(templateResult.value)) - : null, - backendHealth: healthResult.status === 'fulfilled' - ? backendState(healthResult.value) - : 'unknown', - endpoint - } -} - -function parsed( - result: PromiseSettledResult, - schema: S, - warning: string, - warnings: string[] -): z.infer | null { - if (result.status === 'rejected') { - warnings.push(warning) - return null - } - const parsedValue = schema.safeParse(result.value) - if (!parsedValue.success) { - warnings.push(warning) - return null - } - return parsedValue.data -} - -function unavailableInventory(environment: RelayOpsEnvironment, warning: string): ResourceInventory { - return { - director: null, - auth: null, - sql: null, - certificate: null, - directorEndpoint: unavailableEndpoint(), - authEndpoint: unavailableEndpoint(), - cells: environment.cells.map((cell) => - unavailableCell(cell, `${environment.migPrefix}${cell.hostname}`) - ), - warnings: [warning] - } -} - -export type ResourceInventoryOptions = { - wait?: (ms: number) => Promise -} - -export async function readResourceInventory( - environment: RelayOpsEnvironment, - gcloud: GcloudClient, - fetchImpl: typeof fetch = fetch, - options: ResourceInventoryOptions = {} -): Promise { - const wait = options.wait ?? sleep - let token: string - try { - token = await gcloud.accessToken() - } catch { - return unavailableInventory( - environment, - 'Google Cloud credentials are unavailable. Run gcloud auth login.' - ) - } - const runUrl = (service: string) => - `https://run.googleapis.com/v2/projects/${environment.project}/locations/${environment.region}/services/${service}` - const migUrl = (cell: RelayOpsCellConfig) => - `https://compute.googleapis.com/compute/v1/projects/${environment.project}/zones/${cell.zone}/instanceGroupManagers/${environment.migPrefix}${cell.hostname}` - const settled = await Promise.allSettled([ - googleRequest(fetchImpl, token, runUrl(environment.directorService)), - googleRequest(fetchImpl, token, runUrl(environment.authService)), - googleRequest( - fetchImpl, - token, - `https://sqladmin.googleapis.com/sql/v1beta4/projects/${environment.project}/instances/${environment.sqlInstance}` - ), - googleRequest( - fetchImpl, - token, - `https://certificatemanager.googleapis.com/v1/projects/${environment.project}/locations/global/certificates/${environment.certificateName}` - ), - // One transient Compute read must never become a verdict on a cell's power state. - ...environment.cells.map((cell) => - readOnceMore(async () => await googleRequest(fetchImpl, token, migUrl(cell)), wait) - ) - ]) - const warnings: string[] = [] - const directorValue = parsed(settled[0]!, RunServiceSchema, 'Director service inventory is unavailable.', warnings) - const authValue = parsed(settled[1]!, RunServiceSchema, 'Auth service inventory is unavailable.', warnings) - const sqlValue = parsed(settled[2]!, SqlInstanceSchema, 'Cloud SQL inventory is unavailable.', warnings) - const certificateValue = parsed( - settled[3]!, CertificateSchema, 'TLS certificate inventory is unavailable.', warnings - ) - const migValues = environment.cells.map((cell, index) => parsed( - settled[index + 4]!, - MigSchema, - `${cell.hostname.toUpperCase()} MIG inventory is unavailable.`, - warnings - )) - const controlPlaneSleeping = - environment.id === 'staging' && sqlValue?.settings.activationPolicy === 'NEVER' - // Health probes would cold-start scale-to-zero Cloud Run services, so sleeping staging is inventory-only. - const [directorEndpoint, authEndpoint] = controlPlaneSleeping - ? [unavailableEndpoint(), unavailableEndpoint()] - : await Promise.all([ - probeEndpointHealth(environment.directorOrigin, fetchImpl), - // The auth service exposes no /ready, so requiring it would fail every first probe. - probeEndpointHealth(environment.authOrigin, fetchImpl, { requiresReady: false }) - ]) - const cells = await Promise.all(environment.cells.map((cell, index) => - readCell(environment, cell, migValues[index] ?? null, token, fetchImpl) - )) - return { - director: directorValue ? parseService(directorValue) : null, - auth: authValue ? parseService(authValue) : null, - sql: sqlValue ? { - state: sqlValue.state, - activationPolicy: sqlValue.settings.activationPolicy, - tier: sqlValue.settings.tier, - availabilityType: sqlValue.settings.availabilityType ?? 'unknown', - databaseVersion: sqlValue.databaseVersion - } : null, - certificate: certificateValue ? { - state: certificateValue.managed.state, - domains: certificateValue.managed.domains, - expireTime: certificateValue.expireTime ?? null - } : null, - directorEndpoint, - authEndpoint, - cells, - warnings - } -} diff --git a/cloud/apps/relay-ops/src/staging-workflow.test.ts b/cloud/apps/relay-ops/src/staging-workflow.test.ts deleted file mode 100644 index 881ffb2bf9a..00000000000 --- a/cloud/apps/relay-ops/src/staging-workflow.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { parseStagingPowerRequest } from './staging-workflow.js' - -describe('parseStagingPowerRequest', () => { - it('accepts the exact reviewed confirmations', () => { - expect(parseStagingPowerRequest({ mode: 'status', confirmation: '' })).toEqual({ - mode: 'status', confirmation: '' - }) - expect(parseStagingPowerRequest({ mode: 'wake', confirmation: 'WAKE_STAGING' }).mode).toBe('wake') - expect(parseStagingPowerRequest({ mode: 'sleep', confirmation: 'SLEEP_STAGING' }).mode).toBe('sleep') - }) - - it('rejects missing, swapped, or additional fields', () => { - expect(() => parseStagingPowerRequest({ mode: 'wake', confirmation: '' })).toThrow() - expect(() => parseStagingPowerRequest({ mode: 'sleep', confirmation: 'WAKE_STAGING' })).toThrow() - expect(() => parseStagingPowerRequest({ mode: 'production', confirmation: '' })).toThrow() - expect(() => parseStagingPowerRequest({ mode: 'status', confirmation: '', project: 'other' })).toThrow() - }) -}) diff --git a/cloud/apps/relay-ops/src/staging-workflow.ts b/cloud/apps/relay-ops/src/staging-workflow.ts deleted file mode 100644 index 749c6e865bb..00000000000 --- a/cloud/apps/relay-ops/src/staging-workflow.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { execFile } from 'node:child_process' -import { promisify } from 'node:util' -import { z } from 'zod' -import { RELAY_GITHUB_REPOSITORY, relayWorkflowFile } from './relay-repository.js' - -const execFileAsync = promisify(execFile) -const DispatchSchema = z.discriminatedUnion('mode', [ - z.object({ mode: z.literal('status'), confirmation: z.literal('') }).strict(), - z.object({ mode: z.literal('wake'), confirmation: z.literal('WAKE_STAGING') }).strict(), - z.object({ mode: z.literal('sleep'), confirmation: z.literal('SLEEP_STAGING') }).strict() -]) - -export type StagingPowerRequest = z.infer - -export function parseStagingPowerRequest(value: unknown): StagingPowerRequest { - return DispatchSchema.parse(value) -} - -export async function dispatchStagingPowerWorkflow(request: StagingPowerRequest): Promise { - const args = [ - 'workflow', 'run', relayWorkflowFile('power-relay-staging.yml'), - '--repo', RELAY_GITHUB_REPOSITORY, - '-f', `mode=${request.mode}`, - '-f', 'wake-cells=configured' - ] - if (request.confirmation) args.push('-f', `confirmation=${request.confirmation}`) - try { - await execFileAsync('gh', args, { - encoding: 'utf8', - timeout: 30_000, - maxBuffer: 1024 * 1024 - }) - } catch { - throw new Error('Staging power workflow dispatch failed') - } -} diff --git a/cloud/apps/relay-ops/tsconfig.build.json b/cloud/apps/relay-ops/tsconfig.build.json deleted file mode 100644 index 0a11719fef1..00000000000 --- a/cloud/apps/relay-ops/tsconfig.build.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "./tsconfig.json", - "exclude": ["src/**/*.test.ts"] -} diff --git a/cloud/apps/relay-ops/tsconfig.json b/cloud/apps/relay-ops/tsconfig.json deleted file mode 100644 index d266baf8ad6..00000000000 --- a/cloud/apps/relay-ops/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "rootDir": "src", - "outDir": "dist", - "types": ["node", "vitest"], - "exactOptionalPropertyTypes": true, - "verbatimModuleSyntax": true - }, - "include": ["src/**/*.ts"] -} diff --git a/cloud/apps/relay/src/cell-connection-hard-cap-consistency.test.ts b/cloud/apps/relay/src/cell-connection-hard-cap-consistency.test.ts deleted file mode 100644 index 56859c36132..00000000000 --- a/cloud/apps/relay/src/cell-connection-hard-cap-consistency.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { readFileSync } from 'node:fs' -import { fileURLToPath } from 'node:url' -import { - isRelayCellConnectionHardCap, - RELAY_ADMISSION_BUDGETS, - RELAY_CELL_ADMISSION_BOUNDS, - RELAY_CELL_CONNECTION_HARD_CAP, - RELAY_CELL_CONNECTION_HARD_CAPS, - relayCellAdmissionBounds -} from '@orca-cloud/relay-contract' -import { describe, expect, it } from 'vitest' - -// Why: dev scripts run standalone in CI and Terraform cannot read TypeScript, so neither can -// import the constant. Both restate it instead. This asserts every restatement still agrees, so -// raising the cap fails loudly here rather than silently leaving a surface behind. -// Repository root, and every path read below stays inside the Relay tree, so this survives the -// move under cloud/ in the public repository. -const repositoryRoot = fileURLToPath(new URL('../../../', import.meta.url)) -const read = (path: string): string => readFileSync(`${repositoryRoot}${path}`, 'utf8') - -describe('cell connection hard cap stays consistent across surfaces that cannot import it', () => { - it('derives its own bounds from the cap', () => { - expect(RELAY_CELL_ADMISSION_BOUNDS.hardCap).toBe(RELAY_CELL_CONNECTION_HARD_CAP) - expect(RELAY_CELL_ADMISSION_BOUNDS.socketAdmissionCeiling).toBe( - RELAY_CELL_CONNECTION_HARD_CAP - RELAY_ADMISSION_BUDGETS.reservedHostControls - ) - expect(RELAY_CELL_ADMISSION_BOUNDS.maxUnobservedBound).toBe( - RELAY_CELL_ADMISSION_BOUNDS.socketAdmissionCeiling - 1 - ) - for (const hardCap of RELAY_CELL_CONNECTION_HARD_CAPS) { - const bounds = relayCellAdmissionBounds(hardCap) - expect(bounds.socketAdmissionCeiling).toBe( - hardCap - RELAY_ADMISSION_BUDGETS.reservedHostControls - ) - expect(bounds.maxUnobservedBound).toBe(bounds.socketAdmissionCeiling - 1) - } - }) - - it.each([ - ['dev/scripts/relay-recovery-wave-gate.mjs', /targetConnectionCap:\s*(\d+)/], - ['dev/scripts/deploy-relay-gce-multi-target.mjs', /DEFAULT_CONNECTION_CEILING\s*=\s*(\d+)/], - ['dev/scripts/deploy-relay-gce-multi-target.mjs', /CUTOVER_CONNECTION_HARD_CAP\s*=\s*(\d+)/] - ])('%s matches the contract cap', (path, pattern) => { - const match = read(path).match(pattern) - expect(match, `${path} no longer declares ${pattern}`).not.toBeNull() - expect(Number(match![1])).toBe(RELAY_CELL_CONNECTION_HARD_CAP) - }) - - it('every production cell declaring a cap uses a supported contract cap', () => { - const declared = [ - ...read('infra/terraform/environments/production.tfvars').matchAll( - /connection_hard_cap\s*=\s*(\d+)/g - ) - ].map((match) => Number(match[1])) - expect(declared.length).toBeGreaterThan(0) - expect(new Set(declared).has(RELAY_CELL_CONNECTION_HARD_CAP)).toBe(true) - expect(declared.every((hardCap) => isRelayCellConnectionHardCap(hardCap))).toBe(true) - }) - - it('the Terraform cell validation accepts exactly the supported contract caps', () => { - const match = read('infra/terraform/variables.tf').match( - /contains\(\[([^\]]+)\], cell\.connection_hard_cap\)/ - ) - expect(match, 'variables.tf no longer validates connection_hard_cap').not.toBeNull() - expect(match![1]!.split(',').map((value) => Number(value.trim()))).toEqual([ - ...RELAY_CELL_CONNECTION_HARD_CAPS - ]) - }) - - it('the Terraform unobserved bound leaves the contract rebind reserve', () => { - expect(read('infra/terraform/variables.tf')).toContain( - 'cell.connection_unobserved_bound < cell.connection_hard_cap - 100' - ) - }) -}) diff --git a/cloud/apps/relay/src/cell-startup-environment.test.ts b/cloud/apps/relay/src/cell-startup-environment.test.ts deleted file mode 100644 index f85493dd6d0..00000000000 --- a/cloud/apps/relay/src/cell-startup-environment.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { readFileSync } from 'node:fs' -import { describe, expect, it } from 'vitest' -import { loadRelayConfig } from './config.js' - -// A same-cap roll swaps only the image: the cell keeps the env its startup script already -// wrote. So every image must boot on exactly the variables that script renders, and a -// config change that needs a new variable is a template change, not an image-only roll. -const template = readFileSync( - new URL('../../../infra/terraform/relay-gce-startup.sh.tftpl', import.meta.url), - 'utf8' -) - -// One plausible production value per variable the template can write. -const RENDERED: Record = { - DATABASE_URL: 'postgres://relay@127.0.0.1:5432/orca_relay', - ORCA_RELAY_ASSIGNMENT_SIGNING_KEY: 'assignment-key-with-at-least-thirty-two-bytes', - ORCA_RELAY_PUBLIC_URL: 'https://c25.relay.onorca.dev', - ORCA_RELAY_CELL_URL: 'https://c25.relay.onorca.dev', - ORCA_RELAY_AUTH_ISSUER: 'https://auth.onorca.dev', - ORCA_RELAY_AUTH_AUDIENCE: 'orca-relay', - ORCA_RELAY_JWKS_URL: 'https://auth.onorca.dev/.well-known/jwks.json', - ORCA_RELAY_ROLE: 'cell', - ORCA_RELAY_CELL_ID: 'production-gce-c25', - ORCA_RELAY_REGION: 'asia-east2', - ORCA_RELAY_CELL_CAPACITY: '3000', - ORCA_RELAY_DATABASE_POOL_MAX: '16', - ORCA_RELAY_CELL_CONNECTION_HARD_CAP: '3000', - ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND: '60', - ORCA_RELAY_CELLS_JSON: '[]', - ORCA_RELAY_ADMIN_AUDIENCE: 'https://relay.onorca.dev/v1/admin/drain', - ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'deploy@example.iam.gserviceaccount.com', - ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT: 'capacity@example.iam.gserviceaccount.com', - ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT: 'asia-proof@example.iam.gserviceaccount.com', - ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT: 'relay-cell@example.iam.gserviceaccount.com', - ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT: 'relay-director@example.iam.gserviceaccount.com', - ORCA_RELAY_REHOME_AUDIENCE: 'https://relay.onorca.dev/v1/admin/host-drain', - ORCA_RELAY_DIRECTOR_URL: 'https://relay.onorca.dev', - ORCA_RELAY_HEARTBEAT_AUDIENCE: 'https://relay.onorca.dev/v1/admin/cell-heartbeat', - ORCA_RELAY_IMAGE_DIGEST: `sha256:${'a'.repeat(64)}` -} - -const TEMPLATE_VARIABLES = [...template.matchAll(/printf '([A-Z0-9_]+)=/g)].map(([, name]) => name!) - -describe('cell startup environment', () => { - it('boots a cell on exactly the variables the startup template writes', () => { - expect(TEMPLATE_VARIABLES.length).toBeGreaterThan(20) - expect(TEMPLATE_VARIABLES.filter((name) => RENDERED[name] === undefined)).toEqual([]) - const env = Object.fromEntries(TEMPLATE_VARIABLES.map((name) => [name, RENDERED[name]])) - expect(loadRelayConfig(env)).toMatchObject({ - role: 'cell', - cellId: 'production-gce-c25', - databaseUrl: RENDERED.DATABASE_URL - }) - }) -}) diff --git a/cloud/apps/relay/src/regional-rehome-inventory-line-census.test.ts b/cloud/apps/relay/src/regional-rehome-inventory-line-census.test.ts deleted file mode 100644 index 6e0b00a263c..00000000000 --- a/cloud/apps/relay/src/regional-rehome-inventory-line-census.test.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - formatAssignmentInventorySnapshot, - type AssignmentInventorySnapshot -} from './assignment-inventory-snapshot.js' - -// The enable workflow reads the director's rehome inventory line out of Cloud -// Logging with a parser that lives in another language, in another package, and -// is never exercised against the formatter that writes the line. When -// `hostNotArrivedLast24Hours` shipped, the parser read a healthy line as no -// evidence at all and the run failed closed, disabling the durable switch. This -// test is the missing edge: the real formatter's output, through the real -// parser, so a future field fails here instead of in an operator's run. - -type InventoryEvidence = { - active: number - awaitingReceipt: number - targetRegistered: number - completedLast24Hours: number - abortedLast24Hours: number - hostNotArrivedLast24Hours: number | null - oldestActiveAgeMs: number | null -} - -// Imported through a computed URL on purpose: the script is plain ESM outside -// this package's compile scope, so a static import would not resolve. -async function loadParser(): Promise<(entries: unknown[], options: unknown) => InventoryEvidence> { - const source = new URL( - '../../../dev/scripts/relay-rehome-aggregate-evidence.mjs', - import.meta.url - ).href - const loaded: unknown = await import(/* @vite-ignore */ source) - if (!(loaded !== null && typeof loaded === 'object' && 'parseRegionalRehomeInventory' in loaded)) { - throw new Error('relay-rehome-aggregate-evidence.mjs no longer exports its parser') - } - const parse = loaded.parseRegionalRehomeInventory - if (typeof parse !== 'function') throw new Error('parseRegionalRehomeInventory is not callable') - return (entries, options) => readEvidence(parse(entries, options)) -} - -function readEvidence(value: unknown): InventoryEvidence { - if (value === null || typeof value !== 'object') throw new Error('parser returned no evidence') - const counts = ['active', 'awaitingReceipt', 'targetRegistered', 'completedLast24Hours', 'abortedLast24Hours'] as const - const evidence: Record = {} - for (const key of [...counts, 'hostNotArrivedLast24Hours', 'oldestActiveAgeMs'] as const) { - if (!(key in value)) throw new Error(`parser dropped ${key}`) - const read: unknown = Reflect.get(value, key) - if (read !== null && typeof read !== 'number') throw new Error(`${key} is not a count`) - evidence[key] = read - } - for (const key of counts) { - if (evidence[key] === null) throw new Error(`${key} must be a number`) - } - return { - active: Number(evidence['active']), - awaitingReceipt: Number(evidence['awaitingReceipt']), - targetRegistered: Number(evidence['targetRegistered']), - completedLast24Hours: Number(evidence['completedLast24Hours']), - abortedLast24Hours: Number(evidence['abortedLast24Hours']), - hostNotArrivedLast24Hours: evidence['hostNotArrivedLast24Hours'] ?? null, - oldestActiveAgeMs: evidence['oldestActiveAgeMs'] ?? null - } -} - -function snapshot( - regionalRehomes: AssignmentInventorySnapshot['regionalRehomes'] -): AssignmentInventorySnapshot { - return { - cells: [], - activityLeases: { total: 0, expired: 0, requestUnits: 0 }, - connectionReservations: { outstanding: 0, lateArrivalDebt: 0 }, - regionalRehomes - } -} - -function inventoryLine(snapshotValue: AssignmentInventorySnapshot): string { - const line = formatAssignmentInventorySnapshot(snapshotValue).find((candidate) => - candidate.startsWith('[orca-relay] regional rehome inventory ') - ) - if (!line) throw new Error('the formatter no longer emits a rehome inventory line') - return line -} - -describe('regional rehome inventory line census', () => { - it('parses what the director actually prints, field for field', async () => { - const parse = await loadParser() - const regionalRehomes = { - active: 3, - awaitingReceipt: 1, - targetRegistered: 2, - completedLast24Hours: 41, - abortedLast24Hours: 12, - hostNotArrivedLast24Hours: 5, - oldestActiveAgeMs: 77_731_209 - } - const now = Date.parse('2026-09-20T12:00:00Z') - - const evidence = parse( - [{ timestamp: '2026-09-20T11:59:00Z', textPayload: inventoryLine(snapshot(regionalRehomes)) }], - { now, maxAgeMs: 5 * 60_000 } - ) - - expect(evidence).toEqual({ ...regionalRehomes }) - }) - - it('parses the line an idle fleet prints, with no oldest active age', async () => { - const parse = await loadParser() - const now = Date.parse('2026-09-20T12:00:00Z') - - const evidence = parse( - [ - { - timestamp: '2026-09-20T11:59:00Z', - textPayload: inventoryLine( - snapshot({ - active: 0, - awaitingReceipt: 0, - targetRegistered: 0, - completedLast24Hours: 0, - abortedLast24Hours: 0, - hostNotArrivedLast24Hours: 0, - oldestActiveAgeMs: null - }) - ) - } - ], - { now, maxAgeMs: 5 * 60_000 } - ) - - expect(evidence.oldestActiveAgeMs).toBeNull() - expect(evidence.hostNotArrivedLast24Hours).toBe(0) - }) - - it('covers every counter the formatter puts on the line', async () => { - const parse = await loadParser() - // A field the parser ignores is a field the operator never sees, so the - // census fails when the formatter gains one and this test is not updated. - const line = inventoryLine( - snapshot({ - active: 1, - awaitingReceipt: 1, - targetRegistered: 1, - completedLast24Hours: 1, - abortedLast24Hours: 1, - hostNotArrivedLast24Hours: 1, - oldestActiveAgeMs: 1 - }) - ) - const printed = line - .slice('[orca-relay] regional rehome inventory '.length) - .split(' ') - .map((field) => field.split('=')[0]) - - const surfaced = Object.keys( - parse([{ timestamp: '2026-09-20T11:59:00Z', textPayload: line }], { - now: Date.parse('2026-09-20T12:00:00Z'), - maxAgeMs: 5 * 60_000 - }) - ) - - expect([...printed].sort()).toEqual([...surfaced].sort()) - }) -}) diff --git a/cloud/dev/contracts/production-cloud-sql-app-consumers.json b/cloud/dev/contracts/production-cloud-sql-app-consumers.json deleted file mode 100644 index fb89e01d9f1..00000000000 --- a/cloud/dev/contracts/production-cloud-sql-app-consumers.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "comment": "Production Cloud SQL consumers owned by the private orca-cloud application tree (auth and API services). The relay ships without them, so the values the connection budget needs are published here; the private repository binds every field back to its source in its own CI.", - "authInstances": 2, - "authPoolMax": 10, - "apiInstances": 10, - "apiPoolMax": 5, - "maxConnections": 500, - "sources": { - "authInstances": "private apps tfvars: auth service max instances", - "authPoolMax": "private auth service: pg.Pool max", - "apiInstances": "private apps tfvars: API service max instances", - "apiPoolMax": "private API service: pg.Pool max", - "maxConnections": "measured SHOW max_connections = 500 on the live instance 2026-09-16; no max_connections flag is set, so this is the tier default and the previous 400 was an unverified assumption about it" - } -} diff --git a/cloud/dev/fixtures/relay-reserve-watch/2026-10-21-director-stall.json b/cloud/dev/fixtures/relay-reserve-watch/2026-10-21-director-stall.json deleted file mode 100644 index dc3f934d16c..00000000000 --- a/cloud/dev/fixtures/relay-reserve-watch/2026-10-21-director-stall.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "comment": "Production 2026-10-21 07:44-07:58Z, read-only: Cloud Run director 503s per minute (request_count, code 503) and every director runtime-metrics line, instance ids replaced. d5 stalled on its database 07:55-07:57.", - "fleet503PerMinute": {"2026-10-21T07:44":6,"2026-10-21T07:45":7,"2026-10-21T07:46":5,"2026-10-21T07:47":55,"2026-10-21T07:48":11,"2026-10-21T07:49":10,"2026-10-21T07:50":12,"2026-10-21T07:51":4,"2026-10-21T07:52":6,"2026-10-21T07:53":8,"2026-10-21T07:54":6,"2026-10-21T07:55":57,"2026-10-21T07:56":173,"2026-10-21T07:57":45}, - "directorMetricsLines": [ - {"timestamp":"2026-10-21T07:44:05.179655Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:44:08.062180Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-in-flight":1,"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":3}}, - {"timestamp":"2026-10-21T07:44:13.822472Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"relay_assignment_row_busy":1}}, - {"timestamp":"2026-10-21T07:44:14.159472Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:44:25.765825Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:44:35.181579Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:44:38.060162Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:44:43.823368Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:44:44.158999Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:44:55.766172Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:45:05.180654Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:45:08.060467Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:45:13.824498Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:45:14.159451Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:45:25.764958Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"placementRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:45:35.181560Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:45:38.062560Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:45:43.831975Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:45:44.160118Z","instance":"d4","placementRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:45:55.767508Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:46:05.181410Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:46:08.061944Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":3},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":3}}, - {"timestamp":"2026-10-21T07:46:13.824904Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:46:14.160287Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:46:25.766494Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:46:35.180987Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:46:38.061343Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:46:43.824785Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:46:44.161722Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:46:55.768452Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:47:05.182855Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:47:08.066053Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:47:13.825495Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":3},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":3}}, - {"timestamp":"2026-10-21T07:47:14.161928Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"relay_assignment_row_busy":1,"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:47:25.767488Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":4},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":4}}, - {"timestamp":"2026-10-21T07:47:35.183676Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:47:38.064515Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:47:43.827753Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-in-flight":1,"host-rate-limited":1,"wait-timeout":9},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":11}}, - {"timestamp":"2026-10-21T07:47:44.161502Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-in-flight":1,"wait-timeout":7},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":8}}, - {"timestamp":"2026-10-21T07:47:55.769396Z","instance":"d5","stickyRejectionsByReasonDelta":{"wait-timeout":9},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":9}}, - {"timestamp":"2026-10-21T07:48:05.184152Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1,"wait-timeout":10},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":11}}, - {"timestamp":"2026-10-21T07:48:08.069122Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:48:13.826317Z","instance":"d3","stickyRejectionsByReasonDelta":{"wait-timeout":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:48:14.162388Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1,"wait-timeout":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:48:25.770092Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:48:35.183997Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:48:38.072360Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:48:43.826028Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:48:44.162106Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:48:55.769497Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:49:05.183794Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:49:08.070369Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:49:13.826477Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:49:14.162921Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"relay_assignment_row_busy":1,"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:49:25.771495Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:49:35.183472Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:49:38.072836Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:49:43.826252Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:49:44.163115Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:49:55.769846Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:50:05.184917Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:50:08.072488Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:50:13.827130Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":3},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":3}}, - {"timestamp":"2026-10-21T07:50:14.163400Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:50:25.769945Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:50:35.184832Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:50:38.075017Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:50:43.827931Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:50:44.164604Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:50:55.772344Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":3},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":3}}, - {"timestamp":"2026-10-21T07:51:05.184087Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"relay_assignment_row_busy":1}}, - {"timestamp":"2026-10-21T07:51:08.074651Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:51:13.829283Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:51:14.165434Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:51:25.769048Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:51:35.183654Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:51:38.073759Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:51:43.829359Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:51:44.166698Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:51:55.770350Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:52:05.184492Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:52:08.076738Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:52:13.830929Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:52:14.167578Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:52:25.770188Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:52:35.185557Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:52:38.073056Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:52:43.830612Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:52:44.170744Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:52:55.770072Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:53:05.185184Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:53:08.073799Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:53:13.831126Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:53:14.168954Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:53:25.771642Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:53:35.185375Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:53:38.073871Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:53:43.832470Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:53:44.169801Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:53:55.772778Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:54:05.184779Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:54:08.075259Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:54:13.831013Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:54:14.170568Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:54:25.772879Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:54:35.185747Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:54:38.074242Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:54:43.832593Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:54:44.170223Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:54:55.772796Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:55:05.186732Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:55:08.074506Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:55:13.832282Z","instance":"d3","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:55:14.170168Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:55:25.773007Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:55:35.186844Z","instance":"d1","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:55:38.073584Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"placementRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:55:43.832666Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:55:44.171703Z","instance":"d4","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:55:55.774228Z","instance":"d5","stickyRejectionsByReasonDelta":{"wait-timeout":21},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":21}}, - {"timestamp":"2026-10-21T07:56:05.186316Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":4},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":4}}, - {"timestamp":"2026-10-21T07:56:08.074683Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:56:13.836910Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:56:14.171278Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:56:25.774220Z","instance":"d5","stickyRejectionsByReasonDelta":{"wait-timeout":132},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":132}}, - {"timestamp":"2026-10-21T07:56:35.187168Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:56:38.075749Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:56:43.834101Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":4},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":4}}, - {"timestamp":"2026-10-21T07:56:44.171918Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:56:55.775731Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1,"wait-timeout":56},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":57}}, - {"timestamp":"2026-10-21T07:57:05.187236Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:57:08.074119Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:57:13.833738Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:57:14.171816Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:57:25.775798Z","instance":"d5","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:57:35.187493Z","instance":"d1","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}}, - {"timestamp":"2026-10-21T07:57:38.075303Z","instance":"d2","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:57:43.833226Z","instance":"d3","stickyRejectionsByReasonDelta":{"host-rate-limited":2},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":2}}, - {"timestamp":"2026-10-21T07:57:44.170780Z","instance":"d4","drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0}, - {"timestamp":"2026-10-21T07:57:55.775393Z","instance":"d5","stickyRejectionsByReasonDelta":{"wait-timeout":5},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":5}}, - {"timestamp":"2026-10-21T07:58:05.187113Z","instance":"d1","stickyRejectionsByReasonDelta":{"wait-timeout":9},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":9}}, - {"timestamp":"2026-10-21T07:58:08.073793Z","instance":"d2","stickyRejectionsByReasonDelta":{"host-rate-limited":1,"wait-timeout":3},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":4}}, - {"timestamp":"2026-10-21T07:58:13.849059Z","instance":"d3","stickyRejectionsByReasonDelta":{"wait-timeout":11},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":11}}, - {"timestamp":"2026-10-21T07:58:14.171116Z","instance":"d4","stickyRejectionsByReasonDelta":{"wait-timeout":13},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":13}}, - {"timestamp":"2026-10-21T07:58:25.776011Z","instance":"d5","stickyRejectionsByReasonDelta":{"host-rate-limited":1},"drainReturnDeferralsDelta":0,"drainReturnAssignmentsDelta":0,"assign503sByCauseDelta":{"sticky-lane":1}} - ] -} diff --git a/cloud/dev/fixtures/terraform-root-partition/families.json b/cloud/dev/fixtures/terraform-root-partition/families.json deleted file mode 100644 index 9067dda7dbe..00000000000 --- a/cloud/dev/fixtures/terraform-root-partition/families.json +++ /dev/null @@ -1,339 +0,0 @@ -{ - "comment": "Terraform root ownership for every resource family declared under infra/terraform (relay), infra/terraform-foundation, and infra/terraform-apps. env_conditional families live in relay for production and in apps for staging (complementary counts). never_in_relay_state lists foundation/apps families that were introduced with their new root and so must NOT appear in infra/terraform/relay-root-carve-removed.tf. Generated by dev/scripts/terraform-root-partition.mjs --write; the test asserts this file matches the declarations.", - "foundation": [ - "google_artifact_registry_repository.api", - "google_iam_workload_identity_pool.github", - "google_project_service.required", - "google_project_service.sqladmin", - "google_service_account.runtime", - "google_sql_database_instance.auth", - "google_storage_bucket_iam_member.cloud_sql_rollout_lease", - "google_storage_bucket_iam_member.cloud_sql_rollout_lease_bucket_reader" - ], - "apps": [ - "cloudflare_record.artifact_onorca", - "cloudflare_record.artifact_usercontent", - "cloudflare_record.auth", - "google_artifact_registry_repository_iam_member.github_production_app_deploy_writer", - "google_cloud_run_domain_mapping.artifacts", - "google_cloud_run_domain_mapping.auth", - "google_cloud_run_v2_job.skill_storage_monitor", - "google_cloud_run_v2_job_iam_member.github_production_app_skill_monitor_developer", - "google_cloud_run_v2_job_iam_member.skill_storage_scheduler_invoker", - "google_cloud_run_v2_service.api", - "google_cloud_run_v2_service.auth", - "google_cloud_run_v2_service_iam_member.github_production_app_api_developer", - "google_cloud_run_v2_service_iam_member.github_production_app_auth_developer", - "google_cloud_scheduler_job.skill_storage_monitor", - "google_iam_workload_identity_pool_provider.github_production_app_deploy", - "google_logging_metric.skill_api", - "google_logging_metric.skill_archive_rejection", - "google_logging_metric.skill_database", - "google_logging_metric.skill_digest_mismatch", - "google_logging_metric.skill_finalize_saturation", - "google_logging_metric.skill_route_latency", - "google_logging_metric.skill_signing_failure", - "google_logging_metric.skill_storage_inventory", - "google_logging_metric.skill_storage_overdue_quarantine", - "google_logging_project_exclusion.skill_share_bearer_request_urls", - "google_monitoring_alert_policy.skill_api_5xx", - "google_monitoring_alert_policy.skill_cloud_run_resource_pressure", - "google_monitoring_alert_policy.skill_database_migration_failure", - "google_monitoring_alert_policy.skill_digest_mismatch", - "google_monitoring_alert_policy.skill_finalize_failures", - "google_monitoring_alert_policy.skill_route_latency", - "google_monitoring_alert_policy.skill_signing_failure", - "google_monitoring_alert_policy.skill_storage_lifecycle_failure", - "google_monitoring_dashboard.skill_sharing", - "google_project_iam_custom_role.skill_storage_inventory", - "google_project_iam_member.auth_runtime_cloudsql_client", - "google_project_iam_member.github_artifact_writer", - "google_project_iam_member.github_cloud_run_developer", - "google_project_iam_member.runtime_skills_cloudsql_client", - "google_secret_manager_secret.auth", - "google_secret_manager_secret.auth_database_url", - "google_secret_manager_secret.skills_database_url", - "google_secret_manager_secret_iam_member.auth_database_url_accessor", - "google_secret_manager_secret_iam_member.auth_runtime_accessor", - "google_secret_manager_secret_iam_member.runtime_skills_database_url_accessor", - "google_secret_manager_secret_version.auth_database_url", - "google_secret_manager_secret_version.skills_database_url", - "google_service_account.auth_runtime", - "google_service_account.github_production_app_deploy", - "google_service_account.skill_storage_monitor", - "google_service_account.skill_storage_scheduler", - "google_service_account_iam_member.github_auth_runtime_service_account_user", - "google_service_account_iam_member.github_production_app_auth_runtime_user", - "google_service_account_iam_member.github_production_app_deploy_workload_identity_user", - "google_service_account_iam_member.github_production_app_runtime_user", - "google_service_account_iam_member.github_production_app_skill_monitor_user", - "google_service_account_iam_member.github_runtime_service_account_user", - "google_service_account_iam_member.github_skill_storage_monitor_user", - "google_service_account_iam_member.runtime_skill_package_signer", - "google_sql_database.auth", - "google_sql_database.skills", - "google_sql_user.auth", - "google_sql_user.skills", - "google_storage_bucket.artifacts", - "google_storage_bucket.skill_packages", - "google_storage_bucket_iam_member.runtime_artifact_object_admin", - "google_storage_bucket_iam_member.runtime_skill_package_object_user", - "google_storage_bucket_iam_member.skill_storage_monitor_inventory", - "random_password.auth_database", - "random_password.skills_database", - "time_sleep.skill_metric_descriptor_propagation" - ], - "relay": [ - "google_artifact_registry_repository_iam_member.github_production_relay_staging_mirror_writer", - "google_artifact_registry_repository_iam_member.github_production_relay_writer", - "google_artifact_registry_repository_iam_member.github_push_artifact_writer", - "google_artifact_registry_repository_iam_member.github_relay_asia_topology_artifact_reader", - "google_artifact_registry_repository_iam_member.github_staging_relay_deploy_artifact_reader", - "google_certificate_manager_certificate.relay_gce", - "google_certificate_manager_certificate_map.relay_gce", - "google_certificate_manager_certificate_map_entry.relay_gce", - "google_certificate_manager_dns_authorization.relay_gce", - "google_cloud_run_domain_mapping.push", - "google_cloud_run_domain_mapping.relay", - "google_cloud_run_domain_mapping.relay_cell", - "google_cloud_run_v2_service.push", - "google_cloud_run_v2_service.relay", - "google_cloud_run_v2_service.relay_cell", - "google_cloud_run_v2_service.relay_fence_broker", - "google_cloud_run_v2_service_iam_member.github_production_push_developer", - "google_cloud_run_v2_service_iam_member.github_production_relay_director_developer", - "google_cloud_run_v2_service_iam_member.github_production_relay_fence_broker_developer", - "google_cloud_run_v2_service_iam_member.github_staging_relay_capacity_developer", - "google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_auth_developer", - "google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer", - "google_cloud_run_v2_service_iam_member.relay_fence_broker_deploy_invoker", - "google_cloud_run_v2_service_iam_member.relay_fence_broker_invoker", - "google_compute_backend_service.relay_gce_cell", - "google_compute_firewall.relay_gce_iap_ssh", - "google_compute_firewall.relay_gce_load_balancer", - "google_compute_global_address.relay_gce", - "google_compute_global_forwarding_rule.relay_gce", - "google_compute_health_check.relay_gce_liveness", - "google_compute_health_check.relay_gce_readiness", - "google_compute_instance_group_manager.relay_gce_cell", - "google_compute_instance_template.relay_gce_cell", - "google_compute_network.relay_gce", - "google_compute_router.relay_gce", - "google_compute_router.relay_gce_additional", - "google_compute_router_nat.relay_gce", - "google_compute_router_nat.relay_gce_additional", - "google_compute_subnetwork.relay_gce", - "google_compute_subnetwork.relay_gce_additional", - "google_compute_target_https_proxy.relay_gce", - "google_compute_url_map.relay_gce", - "google_iam_workload_identity_pool_provider.github_fence", - "google_iam_workload_identity_pool_provider.github_monitor", - "google_iam_workload_identity_pool_provider.github_production_relay_capacity", - "google_iam_workload_identity_pool_provider.github_push", - "google_iam_workload_identity_pool_provider.github_relay_asia_proof", - "google_iam_workload_identity_pool_provider.github_relay_asia_topology", - "google_iam_workload_identity_pool_provider.github_staging_relay_capacity", - "google_iam_workload_identity_pool_provider.github_staging_relay_deploy", - "google_logging_metric.relay_assignment_lease_shadow", - "google_logging_metric.relay_cell_fix_level", - "google_logging_metric.relay_cell_pool_herd", - "google_logging_metric.relay_cell_reregistration_gave_up", - "google_logging_metric.relay_cell_reregistration_stalled", - "google_logging_metric.relay_cell_reserve_dead_man_tripped", - "google_logging_metric.relay_control_flags_unreadable", - "google_logging_metric.relay_dead_cell_verdict_withheld", - "google_logging_metric.relay_incident", - "google_logging_metric.relay_event_loop_stall", - "google_logging_metric.relay_long_cell_inventory_hold", - "google_logging_metric.relay_reserve_placement_off_with_reserve_cells", - "google_logging_metric.relay_reserve_watch_alarm", - "google_logging_metric.relay_snapshot", - "google_monitoring_alert_policy.relay_assignment_5xx", - "google_monitoring_alert_policy.relay_assignment_edge_429", - "google_monitoring_alert_policy.relay_assignment_lease_bad_signature", - "google_monitoring_alert_policy.relay_cell_control_close_burst", - "google_monitoring_alert_policy.relay_cell_control_rtt", - "google_monitoring_alert_policy.relay_cell_outdated_fix_level", - "google_monitoring_alert_policy.relay_cell_pool_herd", - "google_monitoring_alert_policy.relay_cell_reregistration_gave_up", - "google_monitoring_alert_policy.relay_cell_reregistration_stalled", - "google_monitoring_alert_policy.relay_cell_reserve_dead_man_tripped", - "google_monitoring_alert_policy.relay_cell_process_exit", - "google_monitoring_alert_policy.relay_cloud_nat_port_drops", - "google_monitoring_alert_policy.relay_cloud_sql_backends", - "google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop", - "google_monitoring_alert_policy.relay_cloud_sql_disk", - "google_monitoring_alert_policy.relay_control_flags_unreadable", - "google_monitoring_alert_policy.relay_custom", - "google_monitoring_alert_policy.relay_database_rejection_fenced", - "google_monitoring_alert_policy.relay_dead_cell_verdict_withheld", - "google_monitoring_alert_policy.relay_director_cell_inventory_hold", - "google_monitoring_alert_policy.relay_far_cell_accept_latency", - "google_monitoring_alert_policy.relay_gce_connection_headroom", - "google_monitoring_alert_policy.relay_lock_timeout_burst", - "google_monitoring_alert_policy.relay_event_loop_stall", - "google_monitoring_alert_policy.relay_long_cell_inventory_hold", - "google_monitoring_alert_policy.relay_postgres_retry_exhausted", - "google_monitoring_alert_policy.relay_process_fatal", - "google_monitoring_alert_policy.relay_region_hint_skew", - "google_monitoring_alert_policy.relay_reserve_placement_off_with_reserve_cells", - "google_monitoring_alert_policy.relay_reserve_watch_alarm", - "google_monitoring_dashboard.relay_incident", - "google_project_iam_custom_role.github_production_relay_capacity_mutation", - "google_project_iam_custom_role.github_relay_asia_topology_mutation", - "google_project_iam_custom_role.github_relay_asia_topology_read", - "google_project_iam_custom_role.github_relay_asia_topology_state_list", - "google_project_iam_custom_role.github_staging_relay_capacity_mutation", - "google_project_iam_custom_role.github_staging_relay_power", - "google_project_iam_custom_role.relay_control_writer", - "google_project_iam_custom_role.relay_fence_broker_mutation", - "google_project_iam_member.github_fence_cloudsql_viewer", - "google_project_iam_member.github_fence_compute_viewer", - "google_project_iam_member.github_fence_logging_viewer", - "google_project_iam_member.github_fence_monitoring_viewer", - "google_project_iam_member.github_monitor_compute_viewer", - "google_project_iam_member.github_production_relay_capacity_artifact_reader", - "google_project_iam_member.github_production_relay_capacity_mutation", - "google_project_iam_member.github_production_relay_capacity_viewer", - "google_project_iam_member.github_relay_asia_proof_logging_viewer", - "google_project_iam_member.github_relay_asia_proof_monitoring_viewer", - "google_project_iam_member.github_relay_asia_topology_mutation", - "google_project_iam_member.github_relay_asia_topology_read", - "google_project_iam_member.github_relay_monitor_cloudsql_viewer", - "google_project_iam_member.github_relay_monitor_logging_viewer", - "google_project_iam_member.github_relay_monitor_monitoring_viewer", - "google_project_iam_member.github_staging_relay_capacity_artifact_reader", - "google_project_iam_member.github_staging_relay_capacity_mutation", - "google_project_iam_member.github_staging_relay_capacity_viewer", - "google_project_iam_member.github_staging_relay_deploy_compute_viewer", - "google_project_iam_member.github_staging_relay_deploy_logging_viewer", - "google_project_iam_member.github_staging_relay_power", - "google_project_iam_member.push_runtime_cloudsql_client", - "google_project_iam_member.push_runtime_fcm_admin", - "google_project_iam_member.push_runtime_service_usage_consumer", - "google_project_iam_member.relay_director_runtime_cloudsql_client", - "google_project_iam_member.relay_fence_broker_artifact_reader", - "google_project_iam_member.relay_fence_broker_compute_viewer", - "google_project_iam_member.relay_fence_broker_logging_viewer", - "google_project_iam_member.relay_fence_broker_mutation", - "google_project_iam_member.relay_runtime_artifact_reader", - "google_project_iam_member.relay_runtime_cloudsql_client", - "google_project_iam_member.relay_runtime_log_writer", - "google_secret_manager_secret.push_dedicated_database_url", - "google_secret_manager_secret.push_provider", - "google_secret_manager_secret.relay_assignment_signing_key", - "google_secret_manager_secret.relay_database_url", - "google_secret_manager_secret.relay_director_database_url", - "google_secret_manager_secret.relay_regional_placement_enabled", - "google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor", - "google_secret_manager_secret_iam_member.push_provider_runtime_accessor", - "google_secret_manager_secret_iam_member.relay_assignment_signing_key_accessor", - "google_secret_manager_secret_iam_member.relay_assignment_signing_key_director_accessor", - "google_secret_manager_secret_iam_member.relay_database_url_accessor", - "google_secret_manager_secret_iam_member.relay_database_url_director_accessor", - "google_secret_manager_secret_iam_member.relay_director_database_url_accessor", - "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor", - "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder", - "google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer", - "google_secret_manager_secret_iam_member.relay_regional_placement_director_accessor", - "google_secret_manager_secret_iam_member.relay_regional_placement_runtime_accessor", - "google_secret_manager_secret_version.push_dedicated_database_url", - "google_secret_manager_secret_version.relay_assignment_signing_key", - "google_secret_manager_secret_version.relay_database_url", - "google_secret_manager_secret_version.relay_director_database_url", - "google_secret_manager_secret_version.relay_regional_placement_enabled", - "google_service_account.github_fence", - "google_service_account.github_monitor", - "google_service_account.github_production_relay_capacity", - "google_service_account.github_push_deploy", - "google_service_account.github_relay_asia_proof", - "google_service_account.github_relay_asia_topology", - "google_service_account.github_staging_relay_capacity", - "google_service_account.github_staging_relay_deploy", - "google_service_account.push_runtime", - "google_service_account.relay_director_runtime", - "google_service_account.relay_fence_broker", - "google_service_account.relay_runtime", - "google_service_account_iam_member.github_accepted_repository_workload_identity_user", - "google_service_account_iam_member.github_fence_workload_identity_user", - "google_service_account_iam_member.github_monitor_workload_identity_user", - "google_service_account_iam_member.github_production_push_runtime_token_creator", - "google_service_account_iam_member.github_production_push_runtime_user", - "google_service_account_iam_member.github_production_relay_capacity_runtime_user", - "google_service_account_iam_member.github_production_relay_capacity_workload_identity_user", - "google_service_account_iam_member.github_push_workload_identity_user", - "google_service_account_iam_member.github_relay_asia_proof_workload_identity_user", - "google_service_account_iam_member.github_relay_asia_topology_runtime_user", - "google_service_account_iam_member.github_relay_asia_topology_workload_identity_user", - "google_service_account_iam_member.github_relay_director_runtime_service_account_user", - "google_service_account_iam_member.github_relay_fence_broker_service_account_user", - "google_service_account_iam_member.github_relay_runtime_service_account_user", - "google_service_account_iam_member.github_staging_relay_capacity_runtime_user", - "google_service_account_iam_member.github_staging_relay_capacity_workload_identity_user", - "google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user", - "google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user", - "google_service_account_iam_member.relay_fence_broker_requester_token_creator", - "google_sql_database.push_dedicated", - "google_sql_database.relay", - "google_sql_database_instance.push_dedicated", - "google_sql_user.push_dedicated", - "google_sql_user.relay", - "google_storage_bucket.relay_control", - "google_storage_bucket_iam_member.github_production_relay_capacity_state", - "google_storage_bucket_iam_member.github_push_rollout_lease", - "google_storage_bucket_iam_member.github_relay_asia_topology_state", - "google_storage_bucket_iam_member.github_relay_asia_topology_state_list", - "google_storage_bucket_iam_member.github_staging_relay_capacity_state", - "google_storage_bucket_iam_member.github_staging_relay_deploy_state", - "google_storage_bucket_iam_member.github_staging_relay_deploy_state_list", - "google_storage_bucket_iam_member.relay_control_cell_reader", - "google_storage_bucket_iam_member.relay_control_workflow_writer", - "google_storage_bucket_iam_member.relay_fence_broker_bucket_reader", - "google_storage_bucket_iam_member.relay_fence_broker_state_objects", - "random_password.push_dedicated_database", - "random_password.relay_assignment_signing_key", - "random_password.relay_database" - ], - "env_conditional": { - "google_iam_workload_identity_pool_provider.github": { - "production": "relay", - "staging": "apps" - }, - "google_project_iam_member.github_cloudsql_viewer": { - "production": "relay", - "staging": "apps" - }, - "google_project_iam_member.github_compute_viewer": { - "production": "relay", - "staging": "apps" - }, - "google_project_iam_member.github_log_writer": { - "production": "relay", - "staging": "apps" - }, - "google_project_iam_member.github_logging_viewer": { - "production": "relay", - "staging": "apps" - }, - "google_project_iam_member.github_monitoring_viewer": { - "production": "relay", - "staging": "apps" - }, - "google_service_account.github_deploy": { - "production": "relay", - "staging": "apps" - }, - "google_service_account_iam_member.github_workload_identity_user": { - "production": "relay", - "staging": "apps" - }, - "google_storage_bucket_iam_member.github_terraform_state_reader": { - "production": "relay", - "staging": "apps" - } - }, - "state_orphans": { - "staging": [], - "production": [] - } -} diff --git a/cloud/dev/scripts/capture-terraform-plan-baseline.mjs b/cloud/dev/scripts/capture-terraform-plan-baseline.mjs deleted file mode 100644 index 734d67536c2..00000000000 --- a/cloud/dev/scripts/capture-terraform-plan-baseline.mjs +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env node -import { execFileSync } from 'node:child_process' -import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' -import { join } from 'node:path' - -// Why: the relay root can never plan zero-diff (cell templates roll one at a time by design), -// so the split is gated on plan EQUIVALENCE: the normalized change set for a root's addresses -// must be identical before and after a state move. This captures that set deterministically. -// Read-only: -lock=false, -refresh=false, no apply. Forgets from `removed` blocks are excluded -// because the baseline has none. - -const usage = - 'usage: capture-terraform-plan-baseline.mjs --root --env --out [--tag ]' - -export function normalizePlan(planJson) { - const changes = (planJson.resource_changes ?? []) - .filter((entry) => !entry.change.actions.includes('forget')) - .map((entry) => ({ - address: entry.address, - actions: entry.change.actions, - before: entry.change.before ?? null, - after: entry.change.after ?? null, - after_unknown: entry.change.after_unknown ?? null - })) - .sort((left, right) => (left.address < right.address ? -1 : left.address > right.address ? 1 : 0)) - return changes -} - -export function summarize(changes) { - const counts = { create: 0, update: 0, delete: 0, replace: 0, 'no-op': 0, read: 0 } - for (const change of changes) { - const key = change.actions.join('-') - if (key === 'create') counts.create += 1 - else if (key === 'update') counts.update += 1 - else if (key === 'delete') counts.delete += 1 - else if (key === 'delete-create' || key === 'create-delete') counts.replace += 1 - else if (key === 'read') counts.read += 1 - else counts['no-op'] += 1 - } - return counts -} - -function argument(flag) { - const index = process.argv.indexOf(flag) - return index >= 0 ? process.argv[index + 1] : undefined -} - -if (process.argv[1] && import.meta.url.endsWith(process.argv[1].split('/').pop())) { - const root = argument('--root') - const environment = argument('--env') - const out = argument('--out') - const tag = argument('--tag') ?? `${environment}-${root.replaceAll('/', '_')}` - if (!root || !['staging', 'production'].includes(environment) || !out) { - process.stderr.write(`${usage}\n`) - process.exit(2) - } - // The Cloudflare override only applies to the root that still declares the records; the relay - // root dropped them in the carve and errors on a -var for an undeclared variable. - const declaresArtifactDns = readFileSync(join(root, 'variables.tf'), 'utf8').includes( - 'variable "manage_artifact_dns"' - ) - mkdirSync(out, { recursive: true }) - const planFile = join(out, `${tag}.tfplan`) - execFileSync( - 'terraform', - [ - `-chdir=${root}`, 'plan', '-input=false', '-lock=false', '-refresh=false', '-no-color', - `-var-file=environments/${environment}.tfvars`, - ...(declaresArtifactDns ? ['-var', 'manage_artifact_dns=false'] : []), - `-out=${planFile}` - ], - { stdio: ['ignore', 'inherit', 'inherit'] } - ) - const json = JSON.parse( - execFileSync('terraform', [`-chdir=${root}`, 'show', '-json', planFile], { - encoding: 'utf8', - maxBuffer: 256 * 1024 * 1024 - }) - ) - const normalized = normalizePlan(json) - writeFileSync(join(out, `${tag}.norm.json`), `${JSON.stringify(normalized, null, 1)}\n`) - process.stdout.write(`${tag}: ${JSON.stringify(summarize(normalized))}\n`) -} diff --git a/cloud/dev/scripts/capture-terraform-plan-baseline.test.mjs b/cloud/dev/scripts/capture-terraform-plan-baseline.test.mjs deleted file mode 100644 index b003f3db773..00000000000 --- a/cloud/dev/scripts/capture-terraform-plan-baseline.test.mjs +++ /dev/null @@ -1,15 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { normalizePlan, summarize } from './capture-terraform-plan-baseline.mjs' - -test('normalizes, sorts, and drops forgets so a removed block cannot skew equivalence', () => { - const changes = normalizePlan({ - resource_changes: [ - { address: 'b.two', change: { actions: ['update'], before: { x: 1 }, after: { x: 2 } } }, - { address: 'a.one', change: { actions: ['forget'], before: {}, after: null } }, - { address: 'c.three', change: { actions: ['delete', 'create'], before: {}, after: {}, after_unknown: { id: true } } } - ] - }) - assert.deepEqual(changes.map((change) => change.address), ['b.two', 'c.three']) - assert.deepEqual(summarize(changes), { create: 0, update: 1, delete: 0, replace: 1, 'no-op': 0, read: 0 }) -}) diff --git a/cloud/dev/scripts/check-relay-same-cap-headroom.mjs b/cloud/dev/scripts/check-relay-same-cap-headroom.mjs deleted file mode 100644 index 6cc0bebf64e..00000000000 --- a/cloud/dev/scripts/check-relay-same-cap-headroom.mjs +++ /dev/null @@ -1,167 +0,0 @@ -import { pathToFileURL } from 'node:url' -import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' -import { isReserveCellRefusal, reserveCellRollRefusal } from './relay-reserve-cell-refusal.mjs' - -// Drained hosts that find no free slot keep redialling and pin the drained cell (c28, -// 2026-10-01). Hosts are counted as controls, but each moved host also brings its splices; -// the 20% margin is for those. -export const MAX_HEADROOM_FRACTION = 0.8 - -function count(value, name) { - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} is invalid`) - return value -} - -export function parseHeadroomArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of ['director-origin', 'cell-origin', 'cell-id', 'general-cells']) { - if (!values[key]) throw new Error(`missing --${key}`) - } - for (const key of ['director-origin', 'cell-origin']) { - const url = new URL(values[key]) - if (url.protocol !== 'https:' || url.origin !== values[key]) { - throw new Error('origins must be canonical HTTPS origins') - } - } - const generalCells = values['general-cells'] === 'none' - ? [] - : values['general-cells'].split(',') - if ( - new Set(generalCells).size !== generalCells.length || - generalCells.some((cellId) => !/^[a-z0-9-]+$/.test(cellId)) - ) { - throw new Error('--general-cells is invalid') - } - return { - directorOrigin: values['director-origin'], - cellOrigin: values['cell-origin'], - cellId: values['cell-id'], - generalCells - } -} - -async function adminJson(fetchImpl, url, token, body, wait) { - const response = await fetchAdminOnceMore( - fetchImpl, - url, - { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify(body) - }, - { wait } - ) - const json = await response.json().catch(() => ({})) - // null: the director refused the read because the cell is reserve. - if (isReserveCellRefusal(response.status, json)) return null - if (!response.ok) throw new Error(`${new URL(url).pathname} returned ${response.status}`) - return json -} - -// A cell whose capacity view is stale, absent, or not general offers no slot we can count. -// Placement admits while enforced units plus outstanding reservations stay under the pause. -function freeSlots(status) { - const capacity = status.connectionCapacity - if ( - status.enabled !== true || - status.admissionState !== 'general' || - capacity === null || - capacity === undefined || - capacity.heartbeatFresh !== true - ) { - return 0 - } - const pause = count(capacity.normalAdmissionPause, 'normal admission pause') - const used = Math.max( - count(capacity.observedConnections, 'observed connections'), - count(capacity.enforcedConnectionUnits, 'enforced connection units') - ) - const reserved = count(capacity.pendingControlReservations, 'pending control reservations') - return Math.max(0, pause - used - reserved) -} - -export async function checkSameCapHeadroom(config, overrides = {}) { - const fetchImpl = overrides.fetch ?? fetch - const wait = overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) - const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN - if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') - const cellStatus = async (cellId) => { - const answer = await adminJson( - fetchImpl, - `${config.directorOrigin}/v1/admin/cell-status`, - token, - { v: 1, cellId }, - wait - ) - if (answer === null) return null - if (answer.status?.cellId !== cellId) { - throw new Error('director status does not match the cell') - } - return answer.status - } - const runtime = await adminJson( - fetchImpl, - `${config.cellOrigin}/v1/admin/runtime-status`, - token, - { v: 1 }, - wait - ) - if (runtime?.role !== 'cell' || runtime.cellId !== config.cellId) { - throw new Error('runtime status does not match the cell') - } - const targetHosts = count(runtime.runtime?.controls, 'target host controls') - const targetStatus = await cellStatus(config.cellId) - if (targetStatus === null) throw reserveCellRollRefusal(config.cellId) - const targetRegion = targetStatus.region ?? null - const cells = [] - for (const cellId of config.generalCells) { - if (cellId === config.cellId) continue - const status = await cellStatus(cellId) - // A reserve cell's free seats are not in the database: counted as none, never guessed. - cells.push( - status === null - ? { cellId, region: null, freeSlots: 0, reserve: true } - : { cellId, region: status.region ?? null, freeSlots: freeSlots(status) } - ) - } - const total = (filter) => - cells.filter(filter).reduce((sum, cell) => sum + cell.freeSlots, 0) - const sameRegion = (cell) => targetRegion !== null && cell.region === targetRegion - const free = total(() => true) - const allowedHosts = Math.floor(free * MAX_HEADROOM_FRACTION) - return { - cellId: config.cellId, - region: targetRegion, - targetHosts, - freeSlots: free, - sameRegionFreeSlots: total(sameRegion), - otherRegionFreeSlots: total((cell) => !sameRegion(cell)), - allowedHosts, - sufficient: targetHosts <= allowedHosts, - cells - } -} - -export async function main(argv = process.argv.slice(2)) { - const result = await checkSameCapHeadroom(parseHeadroomArguments(argv)) - process.stdout.write(`${JSON.stringify({ event: 'relay_same_cap_headroom', ...result })}\n`) - if (!result.sufficient) { - throw new Error( - `${result.targetHosts} hosts on ${result.cellId} exceed ` + - `${MAX_HEADROOM_FRACTION * 100}% of ${result.freeSlots} free general-cell slots` - ) - } -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs b/cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs deleted file mode 100644 index de6304c7037..00000000000 --- a/cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs +++ /dev/null @@ -1,230 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { - checkSameCapHeadroom, - main, - parseHeadroomArguments -} from './check-relay-same-cap-headroom.mjs' - -const config = { - directorOrigin: 'https://relay.example.com', - cellOrigin: 'https://c28.relay.example.com', - cellId: 'production-gce-c28', - generalCells: ['production-gce-c28', 'production-gce-c29', 'production-gce-asia-c1'] -} - -function cell({ - region = 'us-central1', - pause = 840, - observed = 0, - enforced = observed, - pending = 0, - ...rest -} = {}) { - return { - enabled: true, - admissionState: 'general', - region, - connectionCapacity: { - normalAdmissionPause: pause, - observedConnections: observed, - enforcedConnectionUnits: enforced, - pendingControlReservations: pending, - heartbeatFresh: true - }, - ...rest - } -} - -function harness({ controls = 500, cells = {} } = {}) { - const statuses = { - 'production-gce-c28': cell({ observed: 800 }), - 'production-gce-c29': cell({ observed: 340 }), - 'production-gce-asia-c1': cell({ region: 'asia-east2', pause: 2_840, observed: 2_640 }), - ...cells - } - return async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') { - return Response.json({ role: 'cell', cellId: config.cellId, runtime: { controls } }) - } - const { cellId } = JSON.parse(options.body) - return Response.json({ v: 1, status: { cellId, ...statuses[cellId] } }) - } -} - -const run = async (fetch) => - await checkSameCapHeadroom(config, { fetch, token: 'masked-token', wait: async () => {} }) - -test('counts free general-cell slots by region and excludes the target', async () => { - const result = await run(harness({ controls: 560 })) - assert.deepEqual( - { ...result, cells: undefined }, - { - cellId: config.cellId, - region: 'us-central1', - targetHosts: 560, - freeSlots: 700, - sameRegionFreeSlots: 500, - otherRegionFreeSlots: 200, - allowedHosts: 560, - sufficient: true, - cells: undefined - } - ) - assert.deepEqual(result.cells.map((entry) => entry.cellId), [ - 'production-gce-c29', - 'production-gce-asia-c1' - ]) -}) - -test('free slots subtract the larger of observed and enforced use plus reservations', async () => { - for (const [state, free] of [ - [{ observed: 340, enforced: 400 }, 640], - [{ observed: 400, enforced: 340 }, 640], - [{ observed: 340, pending: 60 }, 640], - [{ observed: 800, pending: 60 }, 200] - ]) { - const result = await run(harness({ cells: { 'production-gce-c29': cell(state) } })) - assert.equal(result.freeSlots, free) - } -}) - -test('rejects malformed capacity counts', async () => { - await assert.rejects( - run(harness({ - cells: { 'production-gce-c29': cell({ pending: null }) } - })), - /pending control reservations is invalid/ - ) -}) - -test('refuses a cell whose hosts exceed 80% of the free slots', async () => { - const result = await run(harness({ controls: 561 })) - assert.equal(result.sufficient, false) -}) - -test('a stale, full, or non-general cell offers no slots', async () => { - for (const blocked of [ - cell({ - connectionCapacity: { - normalAdmissionPause: 840, - observedConnections: 0, - enforcedConnectionUnits: 0, - pendingControlReservations: 0, - heartbeatFresh: false - } - }), - cell({ connectionCapacity: null }), - cell({ admissionState: 'migration-only' }), - cell({ enabled: false }), - cell({ observed: 900 }) - ]) { - const result = await run(harness({ cells: { 'production-gce-c29': blocked } })) - assert.equal(result.freeSlots, 200) - assert.equal(result.sufficient, false) - } -}) - -test('no general cells means no headroom', async () => { - const result = await checkSameCapHeadroom( - { ...config, generalCells: [] }, - { fetch: harness({ controls: 1 }), token: 'masked-token', wait: async () => {} } - ) - assert.equal(result.freeSlots, 0) - assert.equal(result.sufficient, false) -}) - -const reserveRefusal = () => Response.json({ error: 'cell_in_reserve_mode' }, { status: 404 }) - -test('a reserve cell that is not being rolled offers no slots and is named', async () => { - const base = harness({ controls: 160 }) - const result = await run(async (url, options) => - JSON.parse(options.body).cellId === 'production-gce-c29' - ? reserveRefusal() - : await base(url, options)) - assert.equal(result.freeSlots, 200) - assert.equal(result.sufficient, true) - assert.deepEqual(result.cells.find((entry) => entry.cellId === 'production-gce-c29'), { - cellId: 'production-gce-c29', - region: null, - freeSlots: 0, - reserve: true - }) -}) - -test('refuses to roll a reserve cell, naming the flip back to db', async () => { - const base = harness() - await assert.rejects( - run(async (url, options) => - JSON.parse(options.body ?? '{}').cellId === config.cellId - ? reserveRefusal() - : await base(url, options)), - /production-gce-c28 is in reserve mode .*flip it to admitMode=db/ - ) -}) - -test('any other 404 from cell-status still fails', async () => { - const base = harness() - await assert.rejects( - run(async (url, options) => - JSON.parse(options.body).cellId === 'production-gce-c29' - ? Response.json({ error: 'cell_not_found' }, { status: 404 }) - : await base(url, options)), - /cell-status returned 404/ - ) -}) - -test('rejects a runtime or director answer for another cell', async () => { - const base = harness() - await assert.rejects( - run(async (url, options) => - new URL(url).pathname === '/v1/admin/runtime-status' - ? Response.json({ role: 'cell', cellId: 'production-gce-c1', runtime: { controls: 1 } }) - : await base(url, options)), - /runtime status does not match/ - ) - await assert.rejects( - run(async (url, options) => - new URL(url).pathname === '/v1/admin/cell-status' - ? Response.json({ v: 1, status: { cellId: 'production-gce-c1' } }) - : await base(url, options)), - /director status does not match/ - ) -}) - -test('parses the general cell list, including none', () => { - const base = [ - '--director-origin', 'https://relay.example.com', - '--cell-origin', 'https://c28.relay.example.com', - '--cell-id', 'production-gce-c28' - ] - assert.deepEqual( - parseHeadroomArguments([...base, '--general-cells', 'a-c1,b-c2']).generalCells, - ['a-c1', 'b-c2'] - ) - assert.deepEqual(parseHeadroomArguments([...base, '--general-cells', 'none']).generalCells, []) - assert.throws(() => parseHeadroomArguments([...base, '--general-cells', 'a,a']), /invalid/) - assert.throws(() => parseHeadroomArguments(base), /missing --general-cells/) -}) - -test('main prints the numbers and fails on insufficient headroom', async (t) => { - const writes = [] - t.mock.method(process.stdout, 'write', (text) => writes.push(text)) - t.mock.method(globalThis, 'fetch', harness({ controls: 700 })) - process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'masked-token' - t.after(() => { - delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN - }) - await assert.rejects( - main([ - '--director-origin', config.directorOrigin, - '--cell-origin', config.cellOrigin, - '--cell-id', config.cellId, - '--general-cells', config.generalCells.join(',') - ]), - /700 hosts on production-gce-c28 exceed 80% of 700 free general-cell slots/ - ) - const line = JSON.parse(writes.join('')) - assert.equal(line.event, 'relay_same_cap_headroom') - assert.equal(line.allowedHosts, 560) -}) diff --git a/cloud/dev/scripts/classify-relay-production-capacity-director.mjs b/cloud/dev/scripts/classify-relay-production-capacity-director.mjs deleted file mode 100644 index e24bac7b6e3..00000000000 --- a/cloud/dev/scripts/classify-relay-production-capacity-director.mjs +++ /dev/null @@ -1,109 +0,0 @@ -import { readFileSync } from 'node:fs' -import { pathToFileURL } from 'node:url' -import { isDeepStrictEqual } from 'node:util' - -function parseArguments(argv) { - if (argv.length !== 2 || argv[0] !== '--capacity-service-account' || !argv[1]) { - throw new Error('missing --capacity-service-account') - } - return argv[1] -} - -export function classifyProductionCapacityDirector(state, capacityServiceAccount) { - const currentIdentity = state.currentCapacityServiceAccount - if (currentIdentity !== null && currentIdentity !== capacityServiceAccount) { - throw new Error('director has an unexpected capacity identity') - } - const { - baseCells, - currentCells, - capacityCellIds, - targetCellId, - targetHardCap - } = state - if ( - !Array.isArray(baseCells) || - !Array.isArray(currentCells) || - !Array.isArray(capacityCellIds) || - ![600, 1000].includes(targetHardCap) || - new Set(capacityCellIds).size !== capacityCellIds.length || - !capacityCellIds.includes(targetCellId) || - baseCells.length !== currentCells.length || - new Set(baseCells.map((cell) => cell?.id)).size !== baseCells.length || - new Set(currentCells.map((cell) => cell?.id)).size !== currentCells.length - ) { - throw new Error('director topology transition input is invalid') - } - const capacityCells = new Set(capacityCellIds) - const normalizedCurrent = currentCells.map((current, index) => { - const base = baseCells[index] - if ( - typeof current?.id !== 'string' || - current.id !== base?.id - ) { - throw new Error('director topology cell identity is invalid') - } - if (!capacityCells.has(current.id)) { - if (!isDeepStrictEqual(current, base)) { - throw new Error('director topology changed outside the capacity rollout') - } - return current - } - if ( - base.connectionHardCap !== 1000 || - base.connectionUnobservedBound !== 60 || - ![600, 1000].includes(current.connectionHardCap) || - current.connectionUnobservedBound !== 60 - ) { - throw new Error('director capacity rollout state is invalid') - } - return { - ...current, - connectionHardCap: base.connectionHardCap, - connectionUnobservedBound: base.connectionUnobservedBound - } - }) - if ( - !isDeepStrictEqual(normalizedCurrent, baseCells) || - capacityCellIds.some((cellId) => !baseCells.some((cell) => cell.id === cellId)) - ) { - throw new Error('director topology is outside the reviewed capacity envelope') - } - const withTargetCap = (hardCap) => currentCells.map((cell) => - cell.id === targetCellId - ? { ...cell, connectionHardCap: hardCap, connectionUnobservedBound: 60 } - : cell - ) - const desiredCells = withTargetCap(targetHardCap) - const predecessorCells = withTargetCap(targetHardCap === 600 ? 1000 : 600) - const topologyPhase = isDeepStrictEqual(currentCells, desiredCells) - ? 'desired' - : isDeepStrictEqual(currentCells, predecessorCells) - ? 'predecessor' - : null - if (!topologyPhase) throw new Error('director topology is not a reviewed transition state') - return { - topologyPhase, - directorReady: - topologyPhase === 'desired' && currentIdentity === capacityServiceAccount, - desiredCells - } -} - -export function main(argv = process.argv.slice(2)) { - const capacityServiceAccount = parseArguments(argv) - const state = JSON.parse(readFileSync(0, 'utf8')) - process.stdout.write(`${JSON.stringify({ - event: 'relay_production_capacity_director_classified', - ...classifyProductionCapacityDirector(state, capacityServiceAccount) - })}\n`) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - try { - main() - } catch (error) { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - } -} diff --git a/cloud/dev/scripts/classify-relay-production-capacity-director.test.mjs b/cloud/dev/scripts/classify-relay-production-capacity-director.test.mjs deleted file mode 100644 index 13b188ac042..00000000000 --- a/cloud/dev/scripts/classify-relay-production-capacity-director.test.mjs +++ /dev/null @@ -1,93 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { - classifyProductionCapacityDirector -} from './classify-relay-production-capacity-director.mjs' - -const capacityServiceAccount = - 'orca-cloud-gha-relay-cap@onorca-cloud.iam.gserviceaccount.com' -const capacityCellIds = ['production-gce-c25', 'production-gce-c26'] -const baseCells = [ - { id: 'production-gce-c17', connectionHardCap: 600, connectionUnobservedBound: 60 }, - { id: 'production-gce-c25', connectionHardCap: 1000, connectionUnobservedBound: 60 }, - { id: 'production-gce-c26', connectionHardCap: 1000, connectionUnobservedBound: 60 } -] -const mixedCells = [ - baseCells[0], - { ...baseCells[1], connectionHardCap: 600 }, - baseCells[2] -] - -function state(overrides = {}) { - return { - baseCells, - currentCells: mixedCells, - capacityCellIds, - targetCellId: 'production-gce-c25', - targetHardCap: 1000, - currentCapacityServiceAccount: capacityServiceAccount, - ...overrides - } -} - -test('classifies one target while preserving completed rollout cells', () => { - assert.deepEqual( - classifyProductionCapacityDirector(state(), capacityServiceAccount), - { - topologyPhase: 'predecessor', - directorReady: false, - desiredCells: baseCells - } - ) -}) - -test('skips deployment only for exact topology and identity', () => { - assert.deepEqual( - classifyProductionCapacityDirector(state({ currentCells: baseCells }), capacityServiceAccount), - { topologyPhase: 'desired', directorReady: true, desiredCells: baseCells } - ) - assert.deepEqual( - classifyProductionCapacityDirector(state({ - currentCells: baseCells, - targetHardCap: 600 - }), capacityServiceAccount), - { - topologyPhase: 'predecessor', - directorReady: false, - desiredCells: mixedCells - } - ) -}) - -test('rejects an unknown topology or capacity identity', () => { - assert.throws( - () => classifyProductionCapacityDirector(state({ - currentCells: [baseCells[0], { ...baseCells[1], connectionHardCap: 700 }, baseCells[2]] - }), capacityServiceAccount), - /rollout state is invalid/ - ) - assert.throws( - () => classifyProductionCapacityDirector(state({ - currentCapacityServiceAccount: 'unexpected@onorca-cloud.iam.gserviceaccount.com' - }), capacityServiceAccount), - /unexpected capacity identity/ - ) - assert.throws( - () => classifyProductionCapacityDirector(state({ - currentCells: [{ ...baseCells[0], connectionHardCap: 1000 }, mixedCells[1], mixedCells[2]] - }), capacityServiceAccount), - /outside the capacity rollout/ - ) - assert.throws( - () => classifyProductionCapacityDirector(state({ - currentCells: [baseCells[0], { ...mixedCells[1], url: 'https://wrong.invalid' }, baseCells[2]] - }), capacityServiceAccount), - /outside the reviewed capacity envelope/ - ) - assert.throws( - () => classifyProductionCapacityDirector(state({ - targetCellId: 'production-gce-c17' - }), capacityServiceAccount), - /transition input is invalid/ - ) -}) diff --git a/cloud/dev/scripts/classify-relay-staging-bootstrap.mjs b/cloud/dev/scripts/classify-relay-staging-bootstrap.mjs deleted file mode 100644 index fd50b332c57..00000000000 --- a/cloud/dev/scripts/classify-relay-staging-bootstrap.mjs +++ /dev/null @@ -1,47 +0,0 @@ -import { pathToFileURL } from 'node:url' - -export function classifyStagingBootstrap({ c2Kind, c2Admission, c3Kind, c3Admission }) { - for (const kind of [c2Kind, c3Kind]) { - if (!['legacy', 'modern'].includes(kind)) throw new Error('bootstrap runtime kind is invalid') - } - for (const admission of [c2Admission, c3Admission]) { - if (!['general', 'migration-only'].includes(admission)) { - throw new Error('bootstrap admission is not recoverable') - } - } - if (c2Kind === 'modern' && c3Kind === 'modern') return 'complete' - if (c2Kind === 'modern') return 'roll-c3' - if (c3Kind === 'modern') return 'roll-c2' - if (c2Admission === 'general') return 'normalize-and-roll-both' - if (c3Admission === 'general') return 'resume-c2-then-c3' - throw new Error('bootstrap has no general fallback') -} - -function parseArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - return { - c2Kind: values['c2-kind'], - c2Admission: values['c2-admission'], - c3Kind: values['c3-kind'], - c3Admission: values['c3-admission'] - } -} - -export function main(argv = process.argv.slice(2)) { - process.stdout.write(`${classifyStagingBootstrap(parseArguments(argv))}\n`) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - try { - main() - } catch (error) { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - } -} diff --git a/cloud/dev/scripts/classify-relay-staging-bootstrap.test.mjs b/cloud/dev/scripts/classify-relay-staging-bootstrap.test.mjs deleted file mode 100644 index 66caa4fde4c..00000000000 --- a/cloud/dev/scripts/classify-relay-staging-bootstrap.test.mjs +++ /dev/null @@ -1,60 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { classifyStagingBootstrap } from './classify-relay-staging-bootstrap.mjs' - -const state = (c2Kind, c2Admission, c3Kind, c3Admission) => ({ - c2Kind, - c2Admission, - c3Kind, - c3Admission -}) - -test('classifies the fresh legacy bootstrap', () => { - assert.equal( - classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'general')), - 'normalize-and-roll-both' - ) -}) - -test('retries normalization after a failed C3 restart', () => { - assert.equal( - classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'migration-only')), - 'normalize-and-roll-both' - ) -}) - -test('resumes after C2 isolation', () => { - assert.equal( - classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'general')), - 'resume-c2-then-c3' - ) -}) - -test('resumes after C2 apply or a partial C3 transition', () => { - for (const c2Admission of ['general', 'migration-only']) { - for (const c3Admission of ['general', 'migration-only']) { - assert.equal( - classifyStagingBootstrap(state('modern', c2Admission, 'legacy', c3Admission)), - 'roll-c3' - ) - } - } -}) - -test('repairs an unexpected modern C3 before rolling legacy C2', () => { - assert.equal( - classifyStagingBootstrap(state('legacy', 'migration-only', 'modern', 'general')), - 'roll-c2' - ) -}) - -test('accepts already complete modern cells and rejects no-fallback legacy state', () => { - assert.equal( - classifyStagingBootstrap(state('modern', 'migration-only', 'modern', 'general')), - 'complete' - ) - assert.throws( - () => classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'migration-only')), - /no general fallback/ - ) -}) diff --git a/cloud/dev/scripts/cloud-sql-rollout-lock-census.mjs b/cloud/dev/scripts/cloud-sql-rollout-lock-census.mjs deleted file mode 100644 index 80b86aa4582..00000000000 --- a/cloud/dev/scripts/cloud-sql-rollout-lock-census.mjs +++ /dev/null @@ -1,314 +0,0 @@ -// Derives, from workflow and script content, which workflows roll out against the shared Cloud SQL -// instance. Hand lists go stale silently; everything here is read back off disk. -import { readFileSync, readdirSync } from 'node:fs' -import { - RELAY_WORKFLOW_DIRECTORY, - RELAY_WORKFLOW_FILE_PREFIX, - relayWorkflowFile -} from './relay-repository.mjs' - -export const WORKFLOW_ROOT = RELAY_WORKFLOW_DIRECTORY -export const SCRIPT_ROOT = new URL('./', import.meta.url) - -export const LEASE_ACTION = './.github/actions/cloud-sql-rollout-lease' -export const PRODUCTION_LEASE = { - bucket: 'onorca-cloud-terraform-state', - object: 'terraform/state/cloud-sql-rollout/production.lock' -} -export const STAGING_LEASE = { - bucket: 'onorca-cloud-staging-terraform-state', - object: 'terraform/state/cloud-sql-rollout/staging.lock' -} - -export const PRODUCTION_GROUP = 'production-cloud-sql-rollout' -export const STAGING_GROUP = 'relay-staging-mutation' -export const SELECTABLE_GROUP = - "${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}" -// Prove Relay Asia Staging's load mode drives traffic only and must not hold the staging lock. -export const STAGING_OR_LOAD_GROUP = - "${{ inputs.mode == 'load' && 'relay-staging-load' || 'relay-staging-mutation' }}" - -const selectable = (production, staging) => - `\${{ inputs.environment == 'production' && '${production}' || '${staging}' }}` - -export const SELECTABLE_LEASE = { - bucket: selectable(PRODUCTION_LEASE.bucket, STAGING_LEASE.bucket), - object: selectable(PRODUCTION_LEASE.object, STAGING_LEASE.object) -} - -export const LOCK_GROUPS = new Set([ - PRODUCTION_GROUP, - STAGING_GROUP, - SELECTABLE_GROUP, - STAGING_OR_LOAD_GROUP -]) - -export function readWorkflow(file) { - return readFileSync(new URL(file, WORKFLOW_ROOT), 'utf8') -} - -export function workflowFiles() { - return readdirSync(WORKFLOW_ROOT) - .filter((name) => name.endsWith('.yml') && name.startsWith(RELAY_WORKFLOW_FILE_PREFIX)) - .sort() -} - -// --- YAML-shaped readers (line based; the workflows are hand-written and uniformly indented) --- - -function indentOf(line) { - return line.length - line.trimStart().length -} - -function blockAfter(lines, index) { - const base = indentOf(lines[index]) - const body = [] - for (let i = index + 1; i < lines.length; i += 1) { - if (lines[i].trim() === '') { - body.push(lines[i]) - continue - } - if (indentOf(lines[i]) <= base) break - body.push(lines[i]) - } - return body -} - -export function concurrencyBlocks(text) { - const lines = text.split('\n') - const blocks = [] - lines.forEach((line, index) => { - if (line.trim() !== 'concurrency:') return - const body = blockAfter(lines, index) - blocks.push({ - group: body.find((l) => l.trim().startsWith('group:'))?.trim().slice('group:'.length).trim(), - cancelInProgress: body - .find((l) => l.trim().startsWith('cancel-in-progress:')) - ?.trim() - .slice('cancel-in-progress:'.length) - .trim() - }) - }) - return blocks -} - -export function jobs(text) { - const lines = text.split('\n') - const start = lines.findIndex((line) => line === 'jobs:') - if (start === -1) return [] - const found = [] - for (let i = start + 1; i < lines.length; i += 1) { - const match = /^ {2}([A-Za-z0-9_-]+):\s*$/.exec(lines[i]) - if (!match) continue - found.push({ id: match[1], start: i, body: blockAfter(lines, i) }) - } - return found.map((job) => ({ ...job, text: job.body.join('\n') })) -} - -function scalarField(jobText, key) { - const lines = jobText.split('\n') - const index = lines.findIndex((line) => /^ {4}[A-Za-z-]+:/.test(line) && line.trim().startsWith(`${key}:`)) - if (index === -1) return undefined - const inline = lines[index].trim().slice(`${key}:`.length).trim() - if (inline !== '' && inline !== '>-' && inline !== '|') return inline - return blockAfter(lines, index).join(' ').replace(/\s+/g, ' ').trim() -} - -export function jobNeeds(jobText) { - const raw = scalarField(jobText, 'needs') - if (!raw) return [] - return raw - .replace(/^\[|\]$/g, '') - .split(/[,\n]|\s+-\s+/) - .map((entry) => entry.replace(/^-/, '').trim()) - .filter(Boolean) -} - -export function jobIf(jobText) { - return scalarField(jobText, 'if') ?? '' -} - -export function leaseSteps(text) { - const lines = text.split('\n') - const steps = [] - lines.forEach((line, index) => { - if (line.trim() !== `- uses: ${LEASE_ACTION}`) return - const body = blockAfter(lines, index) - const read = (key) => - body.find((l) => l.trim().startsWith(`${key}:`))?.trim().slice(`${key}:`.length).trim() - steps.push({ - line: index + 1, - bucket: read('bucket'), - object: read('object'), - release: read('release') - }) - }) - return steps -} - -export function leaseStepsByJob(file) { - const text = readWorkflow(file) - const steps = leaseSteps(text) - return jobs(text).map((job) => ({ - id: job.id, - steps: steps.filter((step) => step.line > job.start + 1 && step.line <= job.start + 1 + job.body.length) - })) -} - -// --- trigger and reusable-call graph --- - -export function triggers(text) { - const lines = text.split('\n') - const index = lines.findIndex((line) => line === 'on:') - if (index === -1) return [] - return blockAfter(lines, index) - .map((line) => /^ {2}([a-z_]+):/.exec(line)?.[1]) - .filter(Boolean) -} - -export function isEntrypoint(text) { - return triggers(text).some((trigger) => trigger !== 'workflow_call') -} - -export function reusableCalls(text) { - const counts = new Map() - for (const match of text.matchAll(/uses: \.\/\.github\/workflows\/([A-Za-z0-9._-]+\.yml)/g)) { - counts.set(match[1], (counts.get(match[1]) ?? 0) + 1) - } - return counts -} - -export function entrypointsFor(file, seen = new Set()) { - if (seen.has(file)) return new Set() - seen.add(file) - if (isEntrypoint(readWorkflow(file))) return new Set([file]) - const reached = new Set() - for (const candidate of workflowFiles()) { - if (candidate === file) continue - if (!reusableCalls(readWorkflow(candidate)).has(file)) continue - for (const entry of entrypointsFor(candidate, seen)) reached.add(entry) - } - return reached -} - -// --- what counts as a Cloud SQL connection-budget rollout --- - -const COMMAND_PREFIX = /^(?:-\s+)?(?:run:\s*)?(?:[a-z_]+\s*=\s*"?\$\(\s*)?(?:if\s+|then\s+|else\s+|&&\s+|\|\|\s+|!\s+)*/ - -function commandLines(text) { - return text.split('\n').map((line) => line.trim().replace(COMMAND_PREFIX, '')) -} - -export function appliesTerraform(text) { - return commandLines(text).some((line) => /^terraform\b.*\bapply\b/.test(line)) -} - -export function runsCloudRunMutation(text) { - return commandLines(text).some((line) => - /^gcloud run (?:deploy\b|services (?:update|replace)\b|jobs (?:update|deploy)\b)/.test(line) - ) -} - -// Scripts that mint a Cloud Run revision, plus every script that re-exports one of them. -export function revisionMintingScripts() { - const self = new URL(import.meta.url).pathname.split('/').pop() - const names = readdirSync(SCRIPT_ROOT).filter( - (name) => name.endsWith('.mjs') && !name.endsWith('.test.mjs') && name !== self - ) - const source = new Map( - names.map((name) => [name, readFileSync(new URL(name, SCRIPT_ROOT), 'utf8')]) - ) - const minting = new Set( - names.filter((name) => { - const text = source.get(name) - return ( - text.includes("'--no-traffic'") || - /'run',\s*'services',\s*'update'/.test(text) || - /'run',\s*'deploy'/.test(text) - ) - }) - ) - for (let changed = true; changed; ) { - changed = false - for (const name of names) { - if (minting.has(name)) continue - const imports = [...source.get(name).matchAll(/from '\.\/([A-Za-z0-9._-]+\.mjs)'/g)].map( - (match) => match[1] - ) - if (!imports.some((imported) => minting.has(imported))) continue - minting.add(name) - changed = true - } - } - return minting -} - -export function mutatesSharedInstance(text, minters = revisionMintingScripts()) { - if (appliesTerraform(text)) return 'terraform apply against the reviewed relay cell templates' - if (runsCloudRunMutation(text)) return 'gcloud mints or replaces a Cloud Run revision' - for (const script of minters) { - if (text.includes(`dev/scripts/${script}`)) return `runs ${script}, which mints a Cloud Run revision` - } - return undefined -} - -// --- the declared contract --- - -const production = (extra = {}) => ({ env: 'production', group: PRODUCTION_GROUP, ...extra }) -const staging = (extra = {}) => ({ env: 'staging', group: STAGING_GROUP, ...extra }) -const eitherEnvironment = () => ({ env: 'selectable', group: SELECTABLE_GROUP }) - -// Keys are workflow filenames, which the public copy prefixes; the prefix lives in one place. -const named = (entries) => - Object.fromEntries( - entries.map(([file, entry]) => [ - relayWorkflowFile(file), - entry.leaseFiles - ? { ...entry, leaseFiles: entry.leaseFiles.map((member) => relayWorkflowFile(member)) } - : entry - ]) - ) - -export const LEASED_WORKFLOWS = named([ - ['deploy-relay-fence-broker.yml', production()], - ['deploy-relay-production.yml', production()], - ['deploy-relay-production-director.yml', production()], - ['deploy-relay-production-multi-target.yml', production()], - [ - 'deploy-relay-production-capacity.yml', - production({ - leaseFiles: ['deploy-relay-production-capacity-job.yml'], - reentrant: true - }) - ], - [ - 'deploy-relay-production-same-cap.yml', - production({ - leaseFiles: ['deploy-relay-production-same-cap-job.yml'], - reentrant: true - }) - ], - [ - 'operate-relay-production-rehome.yml', - production({ leaseFiles: ['operate-relay-production-rehome-job.yml'] }) - ], - ['deploy-relay-asia-topology.yml', eitherEnvironment()], - ['operate-relay-asia-admission.yml', eitherEnvironment()], - ['deploy-relay-staging.yml', staging()], - ['deploy-relay-staging-gce-candidate.yml', staging()], - ['bootstrap-relay-staging-capacity.yml', staging()], - ['power-relay-staging.yml', staging()], - ['prove-relay-asia-staging.yml', staging({ group: STAGING_OR_LOAD_GROUP })], - [ - 'prove-relay-staging-capacity.yml', - staging({ exclusiveBy: "inputs.mode == 'refresh-asia-c4-image'" }) - ], - ['recover-relay-staging-c4-image.yml', staging()] -]) - -export const NOT_A_CLOUD_SQL_CANDIDATE = named([ - ['push-deploy.yml', 'Push uses dedicated SQL and its own production-push-rollout group and durable push-rollout lease.'], - [ - 'monitor-relay-production.yml', - 'Read-only. Its identity holds monitoring, logging, Cloud SQL and compute viewer roles only, and it runs `gcloud sql instances describe`, never a mutation. It consumes no connection budget, so the durable lease would only let monitoring block a rollout and a rollout block monitoring.' - ] -]) diff --git a/cloud/dev/scripts/deploy-relay-blue-green.mjs b/cloud/dev/scripts/deploy-relay-blue-green.mjs deleted file mode 100644 index a9ed8ca941e..00000000000 --- a/cloud/dev/scripts/deploy-relay-blue-green.mjs +++ /dev/null @@ -1,1538 +0,0 @@ -import { spawnSync } from 'node:child_process' -import { createHash } from 'node:crypto' -import { pathToFileURL } from 'node:url' -import { isDeepStrictEqual } from 'node:util' - -const POLL_INTERVAL_MS = 5_000 -const MIGRATION_TIMEOUT_MS = 14 * 60 * 1000 -const CONNECTION_CAPACITY_PROTOCOL = 2 -export const DIRECTOR_REGIONAL_PLACEMENT_SECRET = - 'orca-cloud-relay-regional-placement-enabled' -export const DIRECTOR_REGIONAL_PLACEMENT_ENV = - 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED' -export const DIRECTOR_CORRECTION_COHORT_ENV = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT' -// Step 5 placement on the directors; only this deploy changes it, so the reserve guard sees it. -export const DIRECTOR_RESERVE_PLACEMENT_ENV = 'ORCA_RELAY_RESERVE_PLACEMENT' -const RESERVE_PLACEMENT_MODES = ['off', 'dry-run', 'on'] -// The cells whose feeds a director polls; a reserve cell outside it has no placing director. -export const DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV = 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS' -export const DIRECTOR_REHOME_IDENTITY_ENV = - 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT' -export const DIRECTOR_REHOME_AUDIENCE_ENV = 'ORCA_RELAY_REHOME_AUDIENCE' -export const SELECTOR_ROLLBACK_TAG = 'selector-rollback' -export const SELECTOR_REVISION_MARKER = '3' -export const DIRECTOR_ADMISSION_ENVIRONMENT = Object.freeze({ - ORCA_RELAY_DATABASE_POOL_MAX: '3', - ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY: '2', - ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX: '128', - ORCA_RELAY_PUBLIC_ASSIGNMENT_RETRY_AFTER_SECONDS: '5', - ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS: '4000' -}) -const DIRECTOR_STARTUP_PROBE = - 'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1' -// Why: Cloud Run routes a new revision's floor instances their first request up to ~110 s after -// they start, so a one-step move lands every assignment on the one pre-started instance. -export const DIRECTOR_STAGED_TRAFFIC_PERCENT = 20 -// Floor instances have started up to ~220 s after the revision and taken ~110 s more to route. -const DIRECTOR_SPREAD_TIMEOUT_MS = 6 * 60 * 1000 -const DIRECTOR_SPREAD_POLL_INTERVAL_MS = 15_000 -// A hung read must not hold the 20/80 split past the deadline. -const DIRECTOR_SPREAD_READ_TIMEOUT_MS = 30_000 - -export function taggedRevisionOrigin(serviceOrigin, tag) { - const url = new URL(serviceOrigin) - if ( - url.protocol !== 'https:' || - url.pathname !== '/' || - url.search || - url.hash || - !url.hostname.endsWith('.run.app') - ) { - throw new Error('Cloud Run service origin is not canonical') - } - if (!/^[a-z][a-z0-9-]{0,62}$/.test(tag)) throw new Error('invalid Cloud Run tag') - return `https://${tag}---${url.hostname}` -} - -export function activeRevision(service) { - const active = (service.status?.traffic ?? []).filter((entry) => Number(entry.percent ?? 0) > 0) - if (active.length !== 1 || Number(active[0].percent) !== 100 || !active[0].revisionName) { - throw new Error('relay service must have exactly one revision receiving 100% traffic') - } - return active[0].revisionName -} - -export function trafficTags(service) { - return (service.status?.traffic ?? []) - .map((entry) => entry.tag) - .filter((tag) => typeof tag === 'string') -} - -export function taggedTraffic(service, tag) { - const traffic = (service.status?.traffic ?? []).find((entry) => entry.tag === tag) - if (!traffic?.url || !traffic.revisionName) throw new Error(`Cloud Run tag ${tag} is not ready`) - return { origin: traffic.url, revision: traffic.revisionName } -} - -export function revisionEnvironment(revision) { - const entries = revision.spec?.containers?.[0]?.env ?? [] - return Object.fromEntries( - entries - .filter((entry) => entry.name && 'value' in entry) - .map((entry) => [entry.name, entry.value]) - ) -} - -export function revisionSecretEnvironment(revision) { - const entries = revision.spec?.containers?.[0]?.env ?? [] - return Object.fromEntries( - entries - .map((entry) => { - const reference = entry.valueSource?.secretKeyRef ?? entry.valueFrom?.secretKeyRef - return [entry.name, reference && { - secret: reference.secret ?? reference.name, - version: reference.version ?? reference.key - }] - }) - .filter(([name, reference]) => name && reference) - ) -} - -export function revisionMinimumInstances(revision) { - return Number(revision.metadata?.annotations?.['autoscaling.knative.dev/minScale'] ?? 0) -} - -export function revisionMaximumInstances(revision) { - const value = Number(revision.metadata?.annotations?.['autoscaling.knative.dev/maxScale']) - if (!Number.isSafeInteger(value) || value < 1) { - throw new Error('serving revision has no bounded maximum instance count') - } - return value -} - -function hasExpectedEnvironment(environment, expected) { - return Object.entries(expected).every(([key, value]) => environment[key] === value) -} - -function projectServiceAccount(config, argument) { - const value = config[argument] - if (value === undefined) return undefined - const suffix = `@${config.project}.iam.gserviceaccount.com` - const account = value.endsWith(suffix) ? value.slice(0, -suffix.length) : '' - if (!/^[a-z][a-z0-9-]{4,28}[a-z0-9]$/.test(account)) { - throw new Error(`--${argument} must belong to the selected project`) - } - return value -} - -function directorCellsJson(value, { allowMissingRegion = false } = {}) { - if (value === undefined) return undefined - if (value.length > 100_000 || /[\r\n]/.test(value)) { - throw new Error('--director-cells-json is invalid') - } - const cells = JSON.parse(value) - if (!Array.isArray(cells) || cells.length < 1 || cells.length > 100) { - throw new Error('--director-cells-json must contain 1..100 cells') - } - const ids = new Set() - for (const cell of cells) { - const keys = Object.keys(cell ?? {}).sort() - const expectedKeys = [ - 'capacityRequests', - 'id', - 'initiallyEnabled', - ...(allowMissingRegion && cell?.region === undefined ? [] : ['region']), - 'url', - ...(cell?.connectionHardCap === undefined - ? [] - : ['connectionHardCap', 'connectionUnobservedBound']) - ].sort() - let origin - try { - origin = new URL(cell?.url) - } catch { - throw new Error('--director-cells-json contains an invalid cell URL') - } - const cap = cell?.connectionHardCap - const bound = cell?.connectionUnobservedBound - if ( - JSON.stringify(keys) !== JSON.stringify(expectedKeys) || - !/^[a-z][a-z0-9-]{0,39}$/.test(cell.id ?? '') || - ids.has(cell.id) || - origin.protocol !== 'https:' || - origin.origin !== cell.url || - !Number.isSafeInteger(cell.capacityRequests) || - cell.capacityRequests < 1 || - !['us-central1', 'asia-east2'].includes( - cell.region ?? (allowMissingRegion ? 'us-central1' : undefined) - ) || - typeof cell.initiallyEnabled !== 'boolean' || - (cap !== undefined && - (![600, 1_000, 3_000].includes(cap) || - !Number.isSafeInteger(bound) || - bound < 0 || - bound >= cap - 100)) - ) { - throw new Error('--director-cells-json contains an invalid cell') - } - ids.add(cell.id) - } - return JSON.stringify( - cells.map((cell) => ({ - id: cell.id, - url: cell.url, - capacityRequests: cell.capacityRequests, - region: cell.region ?? 'us-central1', - initiallyEnabled: cell.initiallyEnabled, - ...(cell.connectionHardCap === undefined - ? {} - : { - connectionHardCap: cell.connectionHardCap, - connectionUnobservedBound: cell.connectionUnobservedBound - }) - })) - ) -} - -export function directorTopologyChange(currentValue, desiredValue, cellId) { - const current = JSON.parse(directorCellsJson(currentValue, { allowMissingRegion: true })) - const desired = JSON.parse(directorCellsJson(desiredValue)) - if ( - current.length !== desired.length || - desired.some(({ id }, index) => id !== current[index]?.id) - ) { - throw new Error('director topology changes the Relay cell set or order') - } - let changed = false - for (let index = 0; index < desired.length; index += 1) { - const before = structuredClone(current[index]) - const after = structuredClone(desired[index]) - if (after.id === cellId) { - delete before.connectionHardCap - delete before.connectionUnobservedBound - delete after.connectionHardCap - delete after.connectionUnobservedBound - changed = !isDeepStrictEqual(current[index], desired[index]) - } - if (!isDeepStrictEqual(before, after)) { - throw new Error('director topology changes fields outside the reviewed capacity pair') - } - } - if (!desired.some(({ id }) => id === cellId)) { - throw new Error('director topology omits the reviewed capacity cell') - } - return { changed, value: JSON.stringify(desired) } -} - -export function directorCellSetAddition(currentValue, desiredValue) { - const current = JSON.parse(directorCellsJson(currentValue, { allowMissingRegion: true })) - const desired = JSON.parse(directorCellsJson(desiredValue)) - if (desired.length < current.length) { - throw new Error('director topology addition cannot remove cells') - } - const desiredById = new Map(desired.map((cell) => [cell.id, cell])) - if (current.some((cell) => !isDeepStrictEqual(desiredById.get(cell.id), cell))) { - throw new Error('director topology addition changes an existing cell') - } - const currentIds = new Set(current.map((cell) => cell.id)) - const additions = desired.filter((cell) => !currentIds.has(cell.id)) - if (additions.some((cell) => cell.initiallyEnabled !== false)) { - throw new Error('director topology additions must start disabled') - } - return { changed: additions.length > 0, value: JSON.stringify(desired) } -} - -export function correctionCohortPercent(value) { - if (!/^(?:[0-9]|[1-9][0-9]|100)$/.test(String(value))) { - throw new Error('region correction cohort must be an integer from 0 to 100') - } - return String(value) -} - -export function directorDeploymentEnvironment(config) { - const imageDigest = config.image?.match(/@(sha256:[a-f0-9]{64})$/)?.[1] - if (config.image !== undefined && imageDigest === undefined) { - throw new Error('--image must use an immutable digest for director deployments') - } - const environment = { - ...DIRECTOR_ADMISSION_ENVIRONMENT, - ORCA_RELAY_ADMISSION_SELECTOR_VERSION: SELECTOR_REVISION_MARKER, - ...(imageDigest === undefined ? {} : { ORCA_RELAY_IMAGE_DIGEST: imageDigest }) - } - if (config['region-correction-cohort-percent'] !== undefined && - config['region-correction-cohort-percent'] !== 'preserve') { - environment[DIRECTOR_CORRECTION_COHORT_ENV] = correctionCohortPercent(config['region-correction-cohort-percent']) - } - if (config['reserve-placement'] !== undefined && config['reserve-placement'] !== 'preserve') { - environment[DIRECTOR_RESERVE_PLACEMENT_ENV] = config['reserve-placement'] - } - if (config['shadow-seat-feed-cells'] !== undefined && config['shadow-seat-feed-cells'] !== 'preserve') { - environment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] = shadowSeatFeedCellsSetting(config['shadow-seat-feed-cells']) - } - const serviceAccount = projectServiceAccount(config, 'capacity-service-account') - const asiaProofServiceAccount = projectServiceAccount(config, 'asia-proof-service-account') - const deployServiceAccount = projectServiceAccount(config, 'deploy-service-account') - const rehomeDirectorServiceAccount = projectServiceAccount( - config, - 'rehome-director-service-account' - ) - const cellsJson = directorCellsJson(config['director-cells-json']) - if (serviceAccount !== undefined) { - environment.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT = serviceAccount - } - if (asiaProofServiceAccount !== undefined) { - environment.ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT = asiaProofServiceAccount - } - // The admin identity the director trusts; omitted, the serving revision's value carries over. - if (deployServiceAccount !== undefined) { - environment.ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT = deployServiceAccount - } - if (rehomeDirectorServiceAccount !== undefined) { - environment[DIRECTOR_REHOME_IDENTITY_ENV] = rehomeDirectorServiceAccount - environment[DIRECTOR_REHOME_AUDIENCE_ENV] = config['rehome-audience'] - } - if (cellsJson !== undefined) environment.ORCA_RELAY_CELLS_JSON = cellsJson - return environment -} - -// `all` or a comma list of cell ids, as the director's config reads it. -function shadowSeatFeedCellsSetting(value) { - if (value === 'all') return value - const cells = value.split(',').map((cell) => cell.trim()) - if (cells.length > 100 || cells.some((cell) => !/^[a-z][a-z0-9-]{0,39}$/.test(cell))) { - throw new Error('--shadow-seat-feed-cells must be preserve, all, or a comma list of cell ids') - } - return cells.join(',') -} - -export function environmentUpdateValue(environment) { - const entries = Object.entries(environment) - if (entries.every(([key, value]) => !key.includes(',') && !value.includes(','))) { - return entries.map(([key, value]) => `${key}=${value}`).join(',') - } - const delimiter = ['~', '|', '@', '%', ';'].find((candidate) => - entries.every(([key, value]) => !key.includes(candidate) && !value.includes(candidate)) - ) - if (!delimiter) throw new Error('candidate environment has no safe gcloud delimiter') - return `^${delimiter}^${entries.map(([key, value]) => `${key}=${value}`).join(delimiter)}` -} - -export function parseArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`) - values[key.slice(2)] = value - } - const required = ['project', 'region', 'service', 'image', 'role', 'release-id'] - for (const key of required) if (!values[key]) throw new Error(`missing --${key}`) - if (!['director', 'cell'].includes(values.role)) throw new Error('--role must be director or cell') - if (values['min-instances'] !== undefined && !/^(0|[1-9][0-9]*)$/.test(values['min-instances'])) { - throw new Error('--min-instances must be a nonnegative integer') - } - if (values['max-instances'] !== undefined && !/^[1-9][0-9]*$/.test(values['max-instances'])) { - throw new Error('--max-instances must be a positive integer') - } - if (values.role === 'cell') { - for (const key of ['director-origin', 'admin-audience']) { - if (!values[key]) throw new Error(`missing --${key}`) - } - if (!['enabled', 'disabled'].includes(values['final-admission'] ?? 'enabled')) { - throw new Error('--final-admission must be enabled or disabled') - } - if ( - values['capacity-service-account'] !== undefined || - values['deploy-service-account'] !== undefined || - values['director-cells-json'] !== undefined || - values['runtime-service-account'] !== undefined || - values['rehome-director-service-account'] !== undefined || - values['rehome-audience'] !== undefined || - values['expected-rehome-generation'] !== undefined || - values['rehome-control-origin'] !== undefined || - values['region-correction-cohort-percent'] !== undefined || - values['reserve-placement'] !== undefined || - values['shadow-seat-feed-cells'] !== undefined - ) { - throw new Error('director configuration arguments require --role director') - } - } - if ( - values['reserve-placement'] !== undefined && - !['preserve', ...RESERVE_PLACEMENT_MODES].includes(values['reserve-placement']) - ) { - throw new Error('--reserve-placement must be preserve, off, dry-run or on') - } - if ( - values.role === 'director' && - values['capacity-cell-id'] !== undefined && - values['director-cells-json'] === undefined - ) { - throw new Error('--capacity-cell-id requires --director-cells-json') - } - if (values['regional-placement-enabled'] !== undefined) { - throw new Error('regional placement changes use the audited runtime-setting step') - } - if ( - values['regional-placement-secret-version'] !== undefined && - !/^[1-9][0-9]*$/.test(values['regional-placement-secret-version']) - ) { - throw new Error('--regional-placement-secret-version must be a positive integer') - } - if (!['true', 'false'].includes(values['prune-revisions'] ?? 'false')) { - throw new Error('--prune-revisions must be true or false') - } - if (!['true', 'false'].includes(values['bootstrap-runtime-identity'] ?? 'false')) { - throw new Error('--bootstrap-runtime-identity must be true or false') - } - if ( - values['shape-baseline-revision'] !== undefined && - !/^[a-z][a-z0-9-]{0,62}$/.test(values['shape-baseline-revision']) - ) { - throw new Error('--shape-baseline-revision must be a Cloud Run revision name') - } - if (values['shape-baseline-revision'] !== undefined) { - if (values.role !== 'director') { - throw new Error('--shape-baseline-revision requires --role director') - } - // A shape adoption moves only what the reviewed apply changed: no identity, topology or - // pruning rides along, whichever caller builds the arguments. - if (values['bootstrap-runtime-identity'] === 'true') { - throw new Error('--shape-baseline-revision cannot combine with a runtime identity bootstrap') - } - if (values['capacity-cell-id'] !== undefined || values['director-cells-json'] !== undefined) { - throw new Error('--shape-baseline-revision cannot combine with a director topology change') - } - if (values['prune-revisions'] === 'true') { - throw new Error('--shape-baseline-revision cannot combine with revision pruning') - } - } - if (values.role === 'director') { - directorDeploymentEnvironment(values) - projectServiceAccount(values, 'runtime-service-account') - projectServiceAccount(values, 'predecessor-runtime-service-account') - if ( - values['bootstrap-runtime-identity'] === 'true' && - (!values['runtime-service-account'] || - !values['predecessor-runtime-service-account'] || - !values['predecessor-image-digest']) - ) { - throw new Error('runtime identity bootstrap requires exact predecessor digest and identities') - } - if ( - values['predecessor-image-digest'] !== undefined && - !/^sha256:[a-f0-9]{64}$/.test(values['predecessor-image-digest']) - ) { - throw new Error('--predecessor-image-digest must be an immutable digest') - } - const rehomePair = [ - 'rehome-director-service-account', - 'rehome-audience' - ].map((key) => values[key] !== undefined) - if (rehomePair[0] !== rehomePair[1]) { - throw new Error('rehome identity and audience must be configured together') - } - if (values['rehome-audience'] !== undefined) { - const audience = new URL(values['rehome-audience']) - if ( - audience.protocol !== 'https:' || - audience.pathname !== '/v1/admin/host-drain' || - audience.search || - audience.hash - ) { - throw new Error('--rehome-audience must be an exact host-drain HTTPS URL') - } - } - // --admin-audience alone serves the reserve guard's reads; durable verification needs all three. - const controlArguments = ['expected-rehome-generation', 'rehome-control-origin'].map( - (key) => values[key] !== undefined - ) - if ( - controlArguments.some(Boolean) && - (!controlArguments.every(Boolean) || values['admin-audience'] === undefined) - ) { - throw new Error('durable rehome verification arguments must be configured together') - } - if ( - values['expected-rehome-generation'] !== undefined && - !/^(0|[1-9][0-9]*)$/.test(values['expected-rehome-generation']) - ) { - throw new Error('--expected-rehome-generation must be a nonnegative integer') - } - if (values['rehome-control-origin'] !== undefined) { - const origin = new URL(values['rehome-control-origin']) - if (origin.protocol !== 'https:' || origin.origin !== values['rehome-control-origin']) { - throw new Error('--rehome-control-origin must be an HTTPS origin') - } - } - } - return values -} - -function commandJson(args, { timeoutMs } = {}) { - const result = spawnSync('gcloud', args, { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'pipe'], - timeout: timeoutMs - }) - if (result.error) throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.error.message}`) - if (result.status !== 0) { - throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`) - } - return JSON.parse(result.stdout) -} - -function commandText(args, { sensitive = false } = {}) { - const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) - if (result.status !== 0) { - const detail = sensitive ? 'credential command failed' : result.stderr.trim() - throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${detail}`) - } - return result.stdout.trim() -} - -export function suppliedAdminIdentityToken(environment = process.env) { - const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN - if (token === undefined) return null - // The workflow supplies a masked Google ID token because external-account gcloud cannot mint one directly. - if (token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) { - throw new Error('invalid supplied admin identity token') - } - return token -} - -// The unverified `email` claim, only to name the caller in an error; the director verifies it. -export function adminTokenEmail(token) { - try { - const payload = JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString('utf8')) - return typeof payload?.email === 'string' ? payload.email : undefined - } catch { - return undefined - } -} - -// cell-admit-mode answers only the revision's deploy identity, so a drifted one (a revision that -// trusts another account than the workflow's) fails here by name instead of as a bare 401. -export function assertDirectorTrustsAdminCaller(token, environment, label) { - const trusted = environment.ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT - const caller = adminTokenEmail(token) - if (trusted === undefined || caller === undefined || caller === trusted) return - throw new Error( - `the ${label} director trusts ${trusted} for deploy admin reads, but this deploy authenticates as ${caller}; ` + - 'run as that identity or correct the revision\'s ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT' - ) -} - -function adminIdentityToken(config) { - const supplied = suppliedAdminIdentityToken() - if (supplied === null && !config['admin-audience']) { - throw new Error('admin reads need --admin-audience or a supplied ORCA_RELAY_ADMIN_ID_TOKEN') - } - return ( - supplied ?? - commandText(['auth', 'print-identity-token', `--audiences=${config['admin-audience']}`], { - sensitive: true - }) - ) -} - -function serviceArguments(config) { - return ['--project', config.project, '--region', config.region] -} - -export function directorStartupProbeArguments(role) { - return role === 'director' ? ['--startup-probe', DIRECTOR_STARTUP_PROBE] : [] -} - -function describeService(config) { - return commandJson([ - 'run', - 'services', - 'describe', - config.service, - ...serviceArguments(config), - '--format=json' - ]) -} - -function describeRevision(config, revision) { - return commandJson([ - 'run', - 'revisions', - 'describe', - revision, - ...serviceArguments(config), - '--format=json' - ]) -} - -function listRevisions(config) { - return commandJson([ - 'run', - 'revisions', - 'list', - '--service', - config.service, - ...serviceArguments(config), - '--format=json' - ]) -} - -function deleteRevision(config, revision) { - commandText([ - 'run', - 'revisions', - 'delete', - revision, - ...serviceArguments(config), - '--quiet' - ]) -} - -function updateTraffic(config, args) { - commandText([ - 'run', - 'services', - 'update-traffic', - config.service, - ...serviceArguments(config), - ...args, - '--quiet' - ]) -} - -export function serviceMaximumInstances(service) { - const value = service.metadata?.annotations?.['run.googleapis.com/maxScale'] - return value === undefined ? undefined : Number(value) -} - -function clearServiceMaximumInstances(config) { - // `--max=0` resets the service cap to Cloud Run's default without a new revision. - commandText(['run', 'services', 'update', config.service, ...serviceArguments(config), '--max=0', '--quiet']) -} - -// Why: a service cap is divided among the revisions taking traffic, so one below two full -// revisions leaves a 20% candidate a fifth of it while the serving floor holds the rest. -function ensureServiceRoomForOverlap(config, operations, service, revisionMaximum) { - const cap = serviceMaximumInstances(service) - if (cap === undefined || cap >= 2 * revisionMaximum) return - operations.clearServiceMaximumInstances(config) - const cleared = serviceMaximumInstances(operations.describeService(config)) - if (cleared !== undefined && cleared < 2 * revisionMaximum) { - throw new Error(`director service still caps instances at ${cleared}, below two revisions of ${revisionMaximum}`) - } - console.warn(JSON.stringify({ event: 'director_service_cap_cleared', from: cap, to: cleared ?? null })) -} - -function removeDirectorTrafficTags(config, operations, retained = new Set()) { - const tags = trafficTags(operations.describeService(config)).filter( - (tag) => !retained.has(tag) - ) - if (tags.length === 0) return - try { - operations.updateTraffic(config, [`--remove-tags=${tags.join(',')}`]) - } catch (error) { - const remaining = new Set(trafficTags(operations.describeService(config))) - if (tags.some((tag) => remaining.has(tag))) throw error - } -} - -function pruneDirectorRevisions(config, operations) { - const service = operations.describeService(config) - const retained = new Set([ - activeRevision(service), - taggedTraffic(service, SELECTOR_ROLLBACK_TAG).revision - ]) - for (const revision of operations.listRevisions(config)) { - const name = revision.metadata?.name - if (!name) throw new Error('director revision list contains an unnamed revision') - if (!retained.has(name)) operations.deleteRevision(config, name) - } - const remaining = operations - .listRevisions(config) - .map((revision) => revision.metadata?.name) - if ( - remaining.length !== retained.size || - remaining.some((name) => !name || !retained.has(name)) - ) { - throw new Error('old director revisions remain after deployment') - } -} - -function deployCandidate(...candidate) { - commandText(candidateUpdateArguments(...candidate)) -} - -export function candidateUpdateArguments( - config, - tag, - env = {}, - image = config.image, - minInstances = config['min-instances'], - maxInstances, - regionalPlacementVersion -) { - const args = [ - 'run', - 'services', - 'update', - config.service, - ...serviceArguments(config), - '--image', - image, - '--tag', - tag, - '--no-traffic' - ] - // Per revision: gcloud's `--max` is the service cap, which Cloud Run splits by traffic share. - if (maxInstances !== undefined) args.push('--max-instances', String(maxInstances)) - if (config.role === 'director') { - args.push( - '--update-secrets', - `${DIRECTOR_REGIONAL_PLACEMENT_ENV}=${DIRECTOR_REGIONAL_PLACEMENT_SECRET}:${regionalPlacementVersion}` - ) - if (config['runtime-service-account'] !== undefined) { - args.push('--service-account', config['runtime-service-account']) - } - } - args.push(...directorStartupProbeArguments(config.role)) - if (minInstances !== undefined) { - args.push('--min-instances', String(minInstances)) - } - const entries = Object.entries(env) - if (entries.length > 0) { - args.push('--update-env-vars', environmentUpdateValue(env)) - } - args.push('--quiet') - return args -} - -function revisionShape(revision, mutableEnvironment, allowServiceAccountChange = false) { - const spec = structuredClone(revision.spec ?? {}) - if (allowServiceAccountChange) delete spec.serviceAccountName - for (const container of spec.containers ?? []) { - delete container.image - delete container.startupProbe - container.env = (container.env ?? []).filter( - ({ name }) => !(name in mutableEnvironment) - ) - } - const ignoredAnnotations = new Set([ - 'autoscaling.knative.dev/minScale', - 'run.googleapis.com/client-name', - 'run.googleapis.com/client-version', - 'run.googleapis.com/operation-id', - 'serving.knative.dev/creator' - ]) - const annotations = Object.fromEntries( - Object.entries(revision.metadata?.annotations ?? {}).filter( - ([name]) => !ignoredAnnotations.has(name) - ) - ) - return { spec, annotations } -} - -function assertPreservedRevisionShape( - serving, - candidate, - mutableEnvironment, - allowServiceAccountChange = false -) { - if (!isDeepStrictEqual( - revisionShape(serving, mutableEnvironment, allowServiceAccountChange), - revisionShape(candidate, mutableEnvironment, allowServiceAccountChange) - )) { - throw new Error('director candidate changed unrelated revision shape') - } -} - -function releaseLabel(releaseId) { - const normalized = releaseId.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') - if (!normalized) throw new Error('release id has no usable characters') - return normalized.slice(0, 32).replace(/-$/g, '') -} - -export function cloudRunTrafficTag(service, prefix, releaseId) { - if (!/^[a-z][a-z0-9-]*$/.test(service)) throw new Error('invalid Cloud Run service name') - if (!/^[a-z][a-z0-9-]*$/.test(prefix)) throw new Error('invalid Cloud Run tag prefix') - const digest = createHash('sha256').update(releaseLabel(releaseId)).digest('hex').slice(0, 9) - const tag = `${prefix}-${digest}` - // Cloud Run imposes this combined bound in addition to the standalone tag bound. - if (service.length + tag.length > 46) throw new Error('Cloud Run service leaves no safe tag space') - return tag -} - -function cellIdentifier(sourceCellId, releaseId) { - const suffix = releaseLabel(releaseId) - const available = 128 - suffix.length - 2 - return `${sourceCellId.slice(0, available)}--${suffix}` -} - -// Instances that served a request since `since`, read from Cloud Run's own request log. -function servingInstanceIds(config, revision, since) { - const filter = [ - 'resource.type="cloud_run_revision"', - `resource.labels.service_name="${config.service}"`, - `resource.labels.revision_name="${revision}"`, - `logName="projects/${config.project}/logs/run.googleapis.com%2Frequests"`, - `timestamp>="${since}"` - ].join(' AND ') - const entries = commandJson([ - 'logging', - 'read', - filter, - '--project', - config.project, - '--limit', - '1000', - '--format=json(labels.instanceId)' - ], { timeoutMs: DIRECTOR_SPREAD_READ_TIMEOUT_MS }) - return new Set(entries.map((entry) => entry.labels?.instanceId).filter(Boolean)) -} - -export async function waitForServingInstances( - config, - revision, - required, - since, - { - readInstances = servingInstanceIds, - pollIntervalMs = DIRECTOR_SPREAD_POLL_INTERVAL_MS, - timeoutMs = DIRECTOR_SPREAD_TIMEOUT_MS - } = {} -) { - const deadline = Date.now() + timeoutMs - let instances = 0 - for (;;) { - try { - instances = readInstances(config, revision, since).size - console.warn(JSON.stringify({ event: 'director_candidate_spread', revision, instances, required })) - if (instances >= required) return - } catch (error) { - // A throttled log read is not evidence either way; the deadline still bounds the wait. - console.warn(JSON.stringify({ event: 'director_candidate_spread_unread', revision, error: String(error) })) - } - if (Date.now() >= deadline) { - // Fewer than the floor here can mean Cloud Run packs the partial share onto fewer instances. - console.warn(JSON.stringify({ event: 'director_candidate_spread_timeout', revision, instances, required })) - throw new Error(`candidate ${revision} served from ${instances} of ${required} instances before the deadline`) - } - await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)) - } -} - -async function waitForHealth(origin, connectionCapacityProtocol) { - const response = await fetch(`${origin}/health`, { signal: AbortSignal.timeout(15_000) }) - const body = await response.json() - if ( - !response.ok || - body.ok !== true || - (connectionCapacityProtocol !== undefined && - body.connectionCapacityProtocol !== connectionCapacityProtocol) - ) { - throw new Error(`candidate health failed at ${origin}`) - } -} - -export async function waitForEvacuationCapacity( - adminPost, - sourceCellId, - targetCellId, - { pollIntervalMs = POLL_INTERVAL_MS, timeoutMs = MIGRATION_TIMEOUT_MS } = {} -) { - const deadline = Date.now() + timeoutMs - while (Date.now() < deadline) { - try { - return await adminPost('/v1/admin/evacuation-capacity', { - v: 1, - sourceCellId, - targetCellId - }) - } catch (error) { - if (!(error instanceof Error) || !error.message.endsWith(': target_cell_unavailable')) throw error - await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)) - } - } - throw new Error('timed out waiting for candidate cell readiness') -} - -async function adminClient(config) { - const token = adminIdentityToken(config) - return async (path, body) => { - const response = await fetch(`${config['director-origin']}${path}`, { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(30_000) - }) - const result = await response.json().catch(() => ({ error: `http_${response.status}` })) - if (!response.ok) throw new Error(`${path} failed: ${result.error ?? response.status}`) - return result - } -} - -export async function assertRegionalRehomeDisabled( - config, - origin, - fetchImpl = fetch -) { - const response = await fetchImpl(`${origin}/v1/admin/regional-rehome-control`, { - method: 'POST', - headers: { - authorization: `Bearer ${adminIdentityToken(config)}`, - 'content-type': 'application/json' - }, - body: JSON.stringify({ v: 1, action: 'inspect' }), - signal: AbortSignal.timeout(30_000) - }) - const result = await response.json().catch(() => ({ error: `http_${response.status}` })) - if (!response.ok) { - throw new Error(`regional rehome inspection failed: ${result.error ?? response.status}`) - } - const generation = Number(config['expected-rehome-generation']) - if ( - result.v !== 1 || - result.control?.enabled !== false || - result.control?.generation !== generation - ) { - throw new Error('regional rehome control is not durably disabled at the expected generation') - } - return result.control -} - -// `undefined` when the director predates the reserve census (its router 404s the route). -export async function readReserveCells(config, origin, cellIds, fetchImpl = fetch) { - const reserve = [] - for (const cellId of cellIds) { - const response = await fetchImpl(`${origin}/v1/admin/cell-admit-mode`, { - method: 'POST', - headers: { - authorization: `Bearer ${adminIdentityToken(config)}`, - 'content-type': 'application/json' - }, - body: JSON.stringify({ v: 1, cellId }), - signal: AbortSignal.timeout(30_000) - }) - const result = await response.json().catch(() => null) - if (response.status === 404 && result === null) return undefined - if (!response.ok || result?.v !== 1 || !['db', 'reserve'].includes(result.admitMode)) { - throw new Error(`cell admit mode read failed for ${cellId}: ${result?.error ?? response.status}`) - } - if (result.admitMode === 'reserve') reserve.push(cellId) - } - return reserve -} - -// A director without the census places hosts on reserve cells from stale rows, so traffic may -// move between revisions that differ in it only while every cell is db. -export function assertReserveCompatibleTrafficMove(servingReserve, candidateReserve) { - if ((servingReserve === undefined) === (candidateReserve === undefined)) return - const reserve = servingReserve ?? candidateReserve - if (reserve.length > 0) { - throw new Error( - `director revisions differ in reserve-mode support while cells are reserve (${reserve.join(', ')}); flip them to db first` - ) - } -} - -// With placement off no director books a reserve cell, so each one's dead-man trips (60-120 s) -// and every host re-registers at once: the cells go to db first, on purpose. -export function assertReservePlacementKept(servingPlacement, candidatePlacement, reserve) { - if (servingPlacement !== 'on' || candidatePlacement === 'on') return - if (reserve !== undefined && reserve.length > 0) { - throw new Error( - `reserve placement would turn ${candidatePlacement} while cells are reserve (${reserve.join(', ')}); flip them to db first` - ) - } -} - -// Placement as the director built it (runtime-status), not as its env asks; an image from before -// the field reports nothing, and only its env can say. -export async function readDirectorPlacement(config, origin, fetchImpl = fetch) { - const response = await fetchImpl(`${origin}/v1/admin/runtime-status`, { - method: 'POST', - headers: { - authorization: `Bearer ${adminIdentityToken(config)}`, - 'content-type': 'application/json' - }, - body: JSON.stringify({ v: 1 }), - signal: AbortSignal.timeout(30_000) - }) - const result = await response.json().catch(() => null) - if (!response.ok || result?.v !== 1 || result.role !== 'director') { - throw new Error(`director runtime status failed at ${origin}: ${result?.error ?? response.status}`) - } - if (result.reservePlacement === undefined) return 'unreported' - if (!RESERVE_PLACEMENT_MODES.includes(result.reservePlacement)) { - throw new Error(`director at ${origin} reports reserve placement ${result.reservePlacement}`) - } - return result.reservePlacement -} - -const shadowCellSet = (value) => - (value ?? '').trim() === 'all' - ? 'all' - : new Set( - (value ?? '') - .split(',') - .map((cell) => cell.trim()) - .filter(Boolean) - ) - -// A reserve cell the directors stop polling gets no booking and no reserver poll: its dead-man -// trips. Shrinking the shadow-cell list past one is refused until that cell is db. -export function assertShadowCellsKeepReserveCells(servingCells, candidateCells, reserve) { - if (reserve === undefined || reserve.length === 0) return - const before = shadowCellSet(servingCells) - const after = shadowCellSet(candidateCells) - if (after === 'all') return - const dropped = reserve.filter((cell) => (before === 'all' || before.has(cell)) && !after.has(cell)) - if (dropped.length > 0) { - throw new Error( - `the shadow seat-feed cells would drop reserve cells (${dropped.join(', ')}); flip them to db first` - ) - } -} - -function directorCellIds(...environments) { - const ids = new Set() - for (const environment of environments) { - for (const cell of JSON.parse(environment.ORCA_RELAY_CELLS_JSON ?? '[]')) ids.add(cell.id) - } - return [...ids] -} - -function assertDirectorRevisionIdentity(revision, config) { - const expectedRuntimeServiceAccount = config['runtime-service-account'] - if ( - expectedRuntimeServiceAccount !== undefined && - revision.spec?.serviceAccountName !== expectedRuntimeServiceAccount - ) { - throw new Error('director revision uses an unexpected runtime service account') - } -} - -// The reviewed shape a deploy must preserve. Normally the serving revision's; after a Terraform -// apply changes the template (a sidecar, a volume, a secret reference), the apply's own 0% -// revision, named by the operator, is the reviewed shape instead. -export function directorShapeBaseline(config, service, servingRevision, describe) { - const named = config['shape-baseline-revision'] - if (named === undefined) return servingRevision - if (named === activeRevision(service)) { - throw new Error('shape baseline revision is already serving; deploy without it') - } - if (named !== service.status?.latestCreatedRevisionName) { - throw new Error('shape baseline revision is not the latest created revision') - } - const servingImage = servingRevision.spec?.containers?.[0]?.image - // A shape change and an image change never ride together: each is reviewed on its own. - if (config.image?.split('@').at(-1) !== servingImage?.split('@').at(-1)) { - throw new Error('a shape baseline deploy must keep the serving image') - } - const baseline = describe(config, named) - // Terraform ignores the image, so its revision runs what is serving; anything else was not an apply. - if (baseline.spec?.containers?.[0]?.image !== servingImage) { - throw new Error('shape baseline revision does not run the serving image') - } - return baseline -} - -export async function deployDirector(config, tag, overrides = {}) { - const operations = { - deployCandidate, - describeService, - describeRevision, - clearServiceMaximumInstances, - listRevisions, - deleteRevision, - updateTraffic, - waitForHealth, - waitForServingInstances, - assertRegionalRehomeDisabled, - readReserveCells, - readDirectorPlacement, - adminIdentityToken, - ...overrides - } - // Why: gcloud does not carry minScale onto a new revision, and the candidate below takes - // 100% of traffic. Inheriting the serving floor keeps one owner for the number instead of - // restating it here; public admission is per-instance, so losing it shrinks fleet capacity. - const initialService = operations.describeService(config) - const servingRevisionName = activeRevision(initialService) - const servingRevision = operations.describeRevision(config, servingRevisionName) - const shapeBaseline = directorShapeBaseline( - config, - initialService, - servingRevision, - operations.describeRevision - ) - const bootstrapRuntimeIdentity = config['bootstrap-runtime-identity'] === 'true' - if (bootstrapRuntimeIdentity) { - if ( - servingRevision.spec?.serviceAccountName !== - config['predecessor-runtime-service-account'] - ) { - throw new Error('director predecessor runtime service account does not match') - } - if ( - config['predecessor-runtime-service-account'] === config['runtime-service-account'] - ) { - throw new Error('director runtime identity bootstrap has already completed') - } - const predecessorDigest = servingRevision.spec?.containers?.[0]?.image?.split('@').at(-1) - if (predecessorDigest !== config['predecessor-image-digest']) { - throw new Error('director predecessor image digest does not match') - } - } else { - assertDirectorRevisionIdentity(servingRevision, config) - } - const servingMinimumInstances = revisionMinimumInstances(servingRevision) - const servingMaximumInstances = revisionMaximumInstances(servingRevision) - const requiredMaximumInstances = config['max-instances'] === undefined - ? servingMaximumInstances - : Number(config['max-instances']) - if (servingMaximumInstances !== requiredMaximumInstances) { - throw new Error( - `serving revision holds ${servingMaximumInstances} maximum instances, expected ${requiredMaximumInstances}` - ) - } - const servingSecrets = revisionSecretEnvironment(servingRevision) - const servingRegionalPlacementVersion = - servingSecrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.version - const targetRegionalPlacementVersion = - config['regional-placement-secret-version'] ?? servingRegionalPlacementVersion - if (!/^[1-9][0-9]*$/.test(targetRegionalPlacementVersion ?? '')) { - throw new Error('director deployment requires an exact regional placement secret version') - } - const rollbackRegionalPlacementVersion = - /^[1-9][0-9]*$/.test(servingRegionalPlacementVersion ?? '') - ? servingRegionalPlacementVersion - : targetRegionalPlacementVersion - const requiredMinimumInstances = - config['min-instances'] === undefined - ? servingMinimumInstances - : Number(config['min-instances']) - const requiredCapacityProtocol = - config['prune-revisions'] === 'true' ? CONNECTION_CAPACITY_PROTOCOL : undefined - const currentEnvironment = revisionEnvironment(servingRevision) - const deploymentEnvironment = directorDeploymentEnvironment(config) - deploymentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ??= correctionCohortPercent( - currentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ?? '0' - ) - if (currentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV] !== undefined) { - deploymentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV] ??= currentEnvironment[DIRECTOR_RESERVE_PLACEMENT_ENV] - } - if (currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] !== undefined) { - deploymentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] ??= currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV] - } - if (config['region-correction-cohort-percent'] !== undefined && - config['region-correction-cohort-percent'] !== 'preserve' && - config['expected-rehome-generation'] === undefined) { - throw new Error('cohort changes require an exact disabled regional-rehome generation') - } - const mutableEnvironment = { - ...deploymentEnvironment, - [DIRECTOR_REGIONAL_PLACEMENT_ENV]: '' - } - const topology = config['director-cells-json'] === undefined - ? { changed: false } - : config['capacity-cell-id'] === undefined - ? directorCellSetAddition( - currentEnvironment.ORCA_RELAY_CELLS_JSON, - deploymentEnvironment.ORCA_RELAY_CELLS_JSON - ) - : directorTopologyChange( - currentEnvironment.ORCA_RELAY_CELLS_JSON, - deploymentEnvironment.ORCA_RELAY_CELLS_JSON, - config['capacity-cell-id'] - ) - const verifyRehomeDisabled = async (origin) => { - if (config['expected-rehome-generation'] === undefined) return - await operations.assertRegionalRehomeDisabled(config, origin) - } - if (!bootstrapRuntimeIdentity) { - await verifyRehomeDisabled(config['rehome-control-origin']) - } - ensureServiceRoomForOverlap(config, operations, initialService, requiredMaximumInstances) - removeDirectorTrafficTags(config, operations) - let deployed - let staged = false - let promoted = false - try { - operations.deployCandidate( - config, - SELECTOR_ROLLBACK_TAG, - deploymentEnvironment, - config.image, - 0, - requiredMaximumInstances, - rollbackRegionalPlacementVersion - ) - const rollback = taggedTraffic( - operations.describeService(config), - SELECTOR_ROLLBACK_TAG - ) - const rollbackRevision = operations.describeRevision(config, rollback.revision) - assertDirectorRevisionIdentity(rollbackRevision, config) - assertPreservedRevisionShape( - shapeBaseline, - rollbackRevision, - mutableEnvironment, - bootstrapRuntimeIdentity - ) - const rollbackEnvironment = revisionEnvironment(rollbackRevision) - const rollbackSecrets = revisionSecretEnvironment(rollbackRevision) - if ( - rollbackEnvironment.ORCA_RELAY_ROLE !== 'director' || - rollbackEnvironment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== - SELECTOR_REVISION_MARKER || - !hasExpectedEnvironment(rollbackEnvironment, deploymentEnvironment) || - rollbackSecrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.secret !== - DIRECTOR_REGIONAL_PLACEMENT_SECRET || - rollbackSecrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.version !== - rollbackRegionalPlacementVersion || - revisionMinimumInstances(rollbackRevision) !== 0 - ) { - throw new Error('rollback revision is not selector-compatible') - } - await operations.waitForHealth(rollback.origin, requiredCapacityProtocol) - await verifyRehomeDisabled(rollback.origin) - operations.deployCandidate( - config, - tag, - deploymentEnvironment, - config.image, - requiredMinimumInstances, - requiredMaximumInstances, - targetRegionalPlacementVersion - ) - const candidate = taggedTraffic(operations.describeService(config), tag) - const candidateRevision = operations.describeRevision(config, candidate.revision) - assertDirectorRevisionIdentity(candidateRevision, config) - assertPreservedRevisionShape( - shapeBaseline, - candidateRevision, - mutableEnvironment, - bootstrapRuntimeIdentity - ) - const environment = revisionEnvironment(candidateRevision) - const secrets = revisionSecretEnvironment(candidateRevision) - if ( - environment.ORCA_RELAY_ROLE !== 'director' || - environment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== SELECTOR_REVISION_MARKER || - !hasExpectedEnvironment(environment, deploymentEnvironment) || - secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.secret !== - DIRECTOR_REGIONAL_PLACEMENT_SECRET || - secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV]?.version !== - targetRegionalPlacementVersion - ) { - throw new Error('stable relay service is not selector-compatible') - } - // Fail before the traffic move, so a candidate that lost the floor never serves. - const candidateMinimumInstances = revisionMinimumInstances(candidateRevision) - if (candidateMinimumInstances !== requiredMinimumInstances) { - throw new Error( - `candidate holds ${candidateMinimumInstances} minimum instances, expected ${requiredMinimumInstances}` - ) - } - await operations.waitForHealth(candidate.origin, requiredCapacityProtocol) - await verifyRehomeDisabled(candidate.origin) - // The same image and the same placement and shadow-cell settings support the same things; - // anything else needs every cell's admit mode (and the admin token that reads it). - const sameImage = - servingRevision.spec?.containers?.[0]?.image?.split('@').at(-1) === config.image?.split('@').at(-1) - const sameSetting = (name) => (currentEnvironment[name] ?? '') === (environment[name] ?? '') - if (sameImage && sameSetting(DIRECTOR_RESERVE_PLACEMENT_ENV) && sameSetting(DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV)) { - console.warn(JSON.stringify({ event: 'director_reserve_support_checked', skipped: 'same image and settings' })) - } else { - const servingOrigin = initialService.status?.url - if (!servingOrigin) throw new Error('relay service reports no URL for the serving revision') - const token = operations.adminIdentityToken(config) - assertDirectorTrustsAdminCaller(token, currentEnvironment, 'serving') - assertDirectorTrustsAdminCaller(token, environment, 'candidate') - const cellIds = directorCellIds(currentEnvironment, environment) - const servingReserve = await operations.readReserveCells(config, servingOrigin, cellIds) - const candidateReserve = await operations.readReserveCells(config, candidate.origin, cellIds) - const reserve = candidateReserve ?? servingReserve - // Placement as each director built it, which an env string alone does not say. - const placementAt = async (origin, env) => { - const placement = await operations.readDirectorPlacement(config, origin) - return placement === 'unreported' ? (env[DIRECTOR_RESERVE_PLACEMENT_ENV] ?? 'off') : placement - } - const servingPlacement = await placementAt(servingOrigin, currentEnvironment) - const candidatePlacement = await placementAt(candidate.origin, environment) - assertReserveCompatibleTrafficMove(servingReserve, candidateReserve) - assertReservePlacementKept(servingPlacement, candidatePlacement, reserve) - assertShadowCellsKeepReserveCells( - currentEnvironment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], - environment[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], - reserve - ) - console.warn( - JSON.stringify({ - event: 'director_reserve_support_checked', - reservePlacement: { serving: servingPlacement, candidate: candidatePlacement }, - serving: servingReserve === undefined ? 'unsupported' : servingReserve, - candidate: candidateReserve === undefined ? 'unsupported' : candidateReserve - }) - ) - } - // A floor of one is the pre-started instance itself; nothing else needs to warm. - if (requiredMinimumInstances > 1) { - const since = new Date().toISOString() - staged = true - operations.updateTraffic(config, [`--to-tags=${tag}=${DIRECTOR_STAGED_TRAFFIC_PERCENT}`]) - await operations.waitForServingInstances( - config, - candidate.revision, - requiredMinimumInstances, - since - ) - } - operations.updateTraffic(config, [`--to-tags=${tag}=100`]) - promoted = true - removeDirectorTrafficTags(config, operations, new Set([SELECTOR_ROLLBACK_TAG])) - deployed = { - event: 'director_deployed', - revision: candidate.revision, - rollbackRevision: rollback.revision, - topologyChanged: topology.changed - } - } catch (error) { - const recoveryErrors = [error] - if (promoted) { - try { - operations.updateTraffic(config, [`--to-tags=${SELECTOR_ROLLBACK_TAG}=100`]) - } catch (rollbackError) { - recoveryErrors.push(rollbackError) - } - } else if (staged) { - // The serving revision still holds most traffic and its warm floor; the cold rollback has neither. - try { - operations.updateTraffic(config, [`--to-revisions=${servingRevisionName}=100`]) - } catch (rollbackError) { - recoveryErrors.push(rollbackError) - } - } - try { - removeDirectorTrafficTags( - config, - operations, - promoted ? new Set([SELECTOR_ROLLBACK_TAG]) : new Set() - ) - } catch (cleanupError) { - recoveryErrors.push(cleanupError) - } - if (recoveryErrors.length > 1) { - const details = recoveryErrors - .map((failure) => failure instanceof Error ? failure.message : String(failure)) - .join('; ') - throw new AggregateError(recoveryErrors, `director deploy recovery failed: ${details}`) - } - throw error - } - if (config['prune-revisions'] === 'true') pruneDirectorRevisions(config, operations) - process.stdout.write(`${JSON.stringify(deployed)}\n`) -} - -async function waitForTargetRegistration(adminPost, sourceCellId, targetCellId) { - const deadline = Date.now() + MIGRATION_TIMEOUT_MS - while (Date.now() < deadline) { - const status = await adminPost('/v1/admin/evacuation-status', { - v: 1, - sourceCellId, - targetCellId, - completeReady: false - }) - process.stdout.write( - `${JSON.stringify({ event: 'migration_registration', ...status })}\n` - ) - if (status.inProgress === status.targetRegistered) return - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)) - } - throw new Error('timed out waiting for target control registrations') -} - -async function waitForCompletion(adminPost, sourceCellId, targetCellId) { - const deadline = Date.now() + MIGRATION_TIMEOUT_MS - while (Date.now() < deadline) { - const status = await adminPost('/v1/admin/evacuation-status', { - v: 1, - sourceCellId, - targetCellId, - completeReady: true - }) - process.stdout.write(`${JSON.stringify({ event: 'migration_completion', ...status })}\n`) - if (status.inProgress === 0) return - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)) - } - throw new Error('timed out waiting for source activity to drain') -} - -async function startAllEvacuations(adminPost, sourceCellId, targetCellId) { - for (;;) { - const result = await adminPost('/v1/admin/evacuate-cell', { - v: 1, - sourceCellId, - targetCellId, - limit: 100 - }) - process.stdout.write(`${JSON.stringify({ event: 'migration_batch', started: result.started })}\n`) - if (result.started === 0) return - } -} - -async function deployCell(config, tag, oldTag, drainTag) { - const initialService = describeService(config) - const currentRevision = activeRevision(initialService) - const currentRevisionState = describeRevision(config, currentRevision) - const currentEnv = revisionEnvironment(currentRevisionState) - const currentImage = currentRevisionState.spec?.containers?.[0]?.image - const sourceCellId = currentEnv.ORCA_RELAY_CELL_ID - const sourceOrigin = currentEnv.ORCA_RELAY_CELL_URL - const capacityRequests = Number(currentEnv.ORCA_RELAY_CELL_CAPACITY) - if ( - currentEnv.ORCA_RELAY_ROLE !== 'cell' || - !sourceCellId || - !sourceOrigin || - !currentImage || - !Number.isInteger(capacityRequests) || - capacityRequests <= 0 - ) { - throw new Error('active cell revision has invalid role, identity, image, or capacity') - } - updateTraffic(config, [`--update-tags=${drainTag}=${currentRevision}`]) - const drainRevision = taggedTraffic(describeService(config), drainTag) - const previousOrigin = taggedRevisionOrigin(initialService.status.url, oldTag) - const candidateOrigin = taggedRevisionOrigin(initialService.status.url, tag) - const targetCellId = cellIdentifier(sourceCellId, config['release-id']) - const adminPost = await adminClient(config) - - await adminPost('/v1/admin/cell-config', { - v: 1, - cellId: targetCellId, - cellUrl: candidateOrigin, - capacityRequests, - enabled: false - }) - deployCandidate(config, tag, { - ORCA_RELAY_CELL_ID: targetCellId, - ORCA_RELAY_CELL_URL: candidateOrigin, - ORCA_RELAY_PUBLIC_URL: candidateOrigin - }) - const candidate = taggedTraffic(describeService(config), tag) - if (candidate.origin !== candidateOrigin) { - throw new Error('queried candidate tag URL mismatches its configured origin') - } - await waitForHealth(candidate.origin) - deployCandidate( - config, - oldTag, - { - ORCA_RELAY_CELL_ID: sourceCellId, - ORCA_RELAY_CELL_URL: previousOrigin, - ORCA_RELAY_PUBLIC_URL: previousOrigin - }, - currentImage - ) - const previous = taggedTraffic(describeService(config), oldTag) - if (previous.origin !== previousOrigin) { - throw new Error('queried previous tag URL mismatches its configured origin') - } - await waitForHealth(previous.origin) - - // HTTP health precedes the authenticated heartbeat that makes a migration target eligible. - await waitForEvacuationCapacity(adminPost, sourceCellId, targetCellId) - await adminPost('/v1/admin/cell-state', { v: 1, cellId: sourceCellId, enabled: false }) - let sourceReconfigured = false - try { - const capacity = await adminPost('/v1/admin/evacuation-capacity', { - v: 1, - sourceCellId, - targetCellId - }) - process.stdout.write(`${JSON.stringify({ event: 'migration_capacity', ...capacity })}\n`) - if (capacity.requiredTargetUnits > capacity.availableTargetUnits) { - throw new Error('candidate lacks durable reservation headroom for target-first migration') - } - // The keeper uses the old image and cell identity without competing with live controls. - await adminPost('/v1/admin/cell-config', { - v: 1, - cellId: sourceCellId, - cellUrl: previous.origin, - capacityRequests, - enabled: false - }) - sourceReconfigured = true - await adminPost('/v1/admin/cell-config', { - v: 1, - cellId: targetCellId, - cellUrl: candidate.origin, - capacityRequests, - enabled: true - }) - } catch (error) { - if (sourceReconfigured) { - await adminPost('/v1/admin/cell-config', { - v: 1, - cellId: sourceCellId, - cellUrl: sourceOrigin, - capacityRequests, - enabled: true - }) - } else { - await adminPost('/v1/admin/cell-state', { v: 1, cellId: sourceCellId, enabled: true }) - } - throw error - } - await startAllEvacuations(adminPost, sourceCellId, targetCellId) - - await fetch(`${drainRevision.origin}/v1/admin/drain`, { - method: 'POST', - headers: { - authorization: `Bearer ${adminIdentityToken(config)}`, - 'content-type': 'application/json' - }, - body: JSON.stringify({ v: 1, graceMs: 120_000 }), - signal: AbortSignal.timeout(30_000) - }).then(async (response) => { - if (!response.ok) throw new Error(`old revision drain failed: ${response.status}`) - }) - - await waitForTargetRegistration(adminPost, sourceCellId, targetCellId) - updateTraffic(config, [`--to-tags=${tag}=100`]) - await waitForCompletion(adminPost, sourceCellId, targetCellId) - const finalEnabled = (config['final-admission'] ?? 'enabled') === 'enabled' - if (!finalEnabled) { - // GCE-backed staging keeps stamped cells runnable for regression without assigning normal traffic. - await adminPost('/v1/admin/cell-state', { v: 1, cellId: targetCellId, enabled: false }) - } - process.stdout.write( - `${JSON.stringify({ - event: 'cell_deployed', - service: config.service, - sourceCellId, - targetCellId, - enabled: finalEnabled, - drainRevision: drainRevision.revision, - previousRevision: previous.revision, - candidateRevision: candidate.revision - })}\n` - ) -} - -export async function main(argv = process.argv.slice(2)) { - const config = parseArguments(argv) - const tag = cloudRunTrafficTag(config.service, 'candidate', config['release-id']) - const oldTag = cloudRunTrafficTag(config.service, 'previous', config['release-id']) - const drainTag = cloudRunTrafficTag(config.service, 'drain', config['release-id']) - if (config.role === 'director') await deployDirector(config, tag) - else await deployCell(config, tag, oldTag, drainTag) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs deleted file mode 100644 index f5b4e87997f..00000000000 --- a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs +++ /dev/null @@ -1,1343 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import { test } from 'node:test' -import { fileURLToPath } from 'node:url' -import { - activeRevision, - correctionCohortPercent, - DIRECTOR_CORRECTION_COHORT_ENV, - cloudRunTrafficTag, - DIRECTOR_ADMISSION_ENVIRONMENT, - DIRECTOR_REGIONAL_PLACEMENT_ENV, - DIRECTOR_REGIONAL_PLACEMENT_SECRET, - DIRECTOR_REHOME_AUDIENCE_ENV, - DIRECTOR_REHOME_IDENTITY_ENV, - assertRegionalRehomeDisabled, - assertReserveCompatibleTrafficMove, - assertShadowCellsKeepReserveCells, - deployDirector, - DIRECTOR_RESERVE_PLACEMENT_ENV, - directorDeploymentEnvironment, - DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV, - directorCellSetAddition, - directorStartupProbeArguments, - directorTopologyChange, - environmentUpdateValue, - parseArguments, - readDirectorPlacement, - readReserveCells, - revisionEnvironment, - revisionSecretEnvironment, - suppliedAdminIdentityToken, - taggedRevisionOrigin, - taggedTraffic, - trafficTags, - candidateUpdateArguments, - waitForEvacuationCapacity, - waitForServingInstances -} from './deploy-relay-blue-green.mjs' - -// Terraform declares these values; audited blue/green stamps the same values without targeting -// the drifted service, so this contract must fail before either side can silently diverge. -function terraformDirectorEnvironment(names) { - const read = (name) => - readFileSync(fileURLToPath(new URL(`../../infra/terraform/${name}`, import.meta.url)), 'utf8') - const relay = read('relay.tf') - const variables = read('variables.tf') - const production = read('environments/production.tfvars') - return Object.fromEntries( - names.map((name) => { - const block = new RegExp(`name\\s*=\\s*"${name}"\\s*\\n\\s*value\\s*=\\s*([^\\n]+)`).exec(relay) - assert.ok(block, `${name} is not set by relay.tf`) - const variable = /var\.([a-z_]+)/.exec(block[1]) - assert.ok(variable, `${name} is not sourced from a Terraform variable`) - // An environment override wins over the variable default, as Terraform resolves it. - const override = new RegExp(`^${variable[1]}\\s*=\\s*(\\S+)`, 'm').exec(production) - const fallback = new RegExp( - `variable\\s+"${variable[1]}"[\\s\\S]*?default\\s*=\\s*(\\S+)` - ).exec(variables) - assert.ok(override || fallback, `${variable[1]} has neither an override nor a default`) - return [name, String((override ?? fallback)[1]).replace(/"/g, '')] - }) - ) -} - -test('director admission environment matches what Terraform deploys', () => { - const names = Object.keys(DIRECTOR_ADMISSION_ENVIRONMENT) - assert.deepEqual(terraformDirectorEnvironment(names), { ...DIRECTOR_ADMISSION_ENVIRONMENT }) - - const relay = readFileSync( - fileURLToPath(new URL('../../infra/terraform/relay.tf', import.meta.url)), - 'utf8' - ) - assert.match( - relay, - /name = "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED"[\s\S]*?secret\s+= google_secret_manager_secret\.relay_regional_placement_enabled\.secret_id[\s\S]*?version = data\.external\.relay_serving_regional_placement_version\.result\.version/ - ) - assert.match(relay, /data "external" "relay_serving_regional_placement_version"/) - assert.match(relay, /read-relay-serving-regional-placement-version\.mjs/) - assert.doesNotMatch(relay, /template\[0\]\.containers\[0\]\.env/) -}) - -test('validates the final stamped-cell admission state', () => { - const required = [ - '--project', - 'project', - '--region', - 'region', - '--service', - 'service', - '--image', - 'image', - '--role', - 'cell', - '--release-id', - 'release', - '--director-origin', - 'https://relay.example.com', - '--admin-audience', - 'https://relay.example.com/v1/admin/drain' - ] - - assert.equal(parseArguments(required)['final-admission'], undefined) - assert.equal( - parseArguments([...required, '--final-admission', 'disabled'])['final-admission'], - 'disabled' - ) - assert.throws(() => parseArguments([...required, '--final-admission', 'sometimes'])) - assert.equal(parseArguments([...required, '--min-instances', '0'])['min-instances'], '0') - assert.throws(() => parseArguments([...required, '--min-instances', '-1'])) - assert.throws(() => - parseArguments([...required, '--capacity-service-account', 'relay@example.com']) - ) -}) - -test('validates optional director capacity configuration', () => { - const cells = [ - { - id: 'staging-gce-c3', - url: 'https://c3.relay-staging.onorca.dev', - capacityRequests: 4_000, - initiallyEnabled: false, - region: 'us-central1', - connectionHardCap: 600, - connectionUnobservedBound: 60 - } - ] - const config = { - project: 'onorca-cloud-staging', - 'capacity-service-account': - 'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com', - 'director-cells-json': JSON.stringify(cells) - } - assert.deepEqual(directorDeploymentEnvironment(config), { - ...DIRECTOR_ADMISSION_ENVIRONMENT, - ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3', - ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT: - 'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com', - ORCA_RELAY_CELLS_JSON: JSON.stringify([ - { - id: cells[0].id, - url: cells[0].url, - capacityRequests: cells[0].capacityRequests, - region: cells[0].region, - initiallyEnabled: cells[0].initiallyEnabled, - connectionHardCap: cells[0].connectionHardCap, - connectionUnobservedBound: cells[0].connectionUnobservedBound - } - ]) - }) - assert.throws( - () => - directorDeploymentEnvironment({ - ...config, - 'capacity-service-account': 'foreign@other-project.iam.gserviceaccount.com' - }), - /selected project/ - ) - assert.throws( - () => - directorDeploymentEnvironment({ - ...config, - 'director-cells-json': JSON.stringify([{ ...cells[0], unexpected: true }]) - }), - /invalid cell/ - ) - assert.match( - environmentUpdateValue(directorDeploymentEnvironment(config)), - /^\^~\^ORCA_RELAY_DATABASE_POOL_MAX=/ - ) - assert.equal(environmentUpdateValue({ FIRST: 'one', SECOND: 'two' }), 'FIRST=one,SECOND=two') - assert.deepEqual( - directorTopologyChange( - JSON.stringify([ - { - capacityRequests: 4_000, - connectionHardCap: 600, - connectionUnobservedBound: 60, - id: 'staging-gce-c3', - initiallyEnabled: false, - region: 'us-central1', - url: 'https://c3.relay-staging.onorca.dev' - } - ]), - JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }]), - 'staging-gce-c3' - ), - { - changed: true, - value: directorDeploymentEnvironment({ - 'director-cells-json': JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }]) - }).ORCA_RELAY_CELLS_JSON - } - ) - assert.throws( - () => - directorTopologyChange( - JSON.stringify(cells), - JSON.stringify([{ ...cells[0], url: 'https://wrong.relay-staging.onorca.dev' }]), - 'staging-gce-c3' - ), - /outside the reviewed capacity pair/ - ) -}) - -test('validates exact director runtime and regional rehome identities', () => { - const base = [ - '--project', - 'onorca-cloud', - '--region', - 'us-central1', - '--service', - 'orca-cloud-relay', - '--image', - `relay@sha256:${'a'.repeat(64)}`, - '--role', - 'director', - '--release-id', - 'rehome', - '--runtime-service-account', - 'relay-director@onorca-cloud.iam.gserviceaccount.com', - '--rehome-director-service-account', - 'relay-director@onorca-cloud.iam.gserviceaccount.com', - '--rehome-audience', - 'https://relay.onorca.dev/v1/admin/host-drain', - '--expected-rehome-generation', - '7', - '--rehome-control-origin', - 'https://relay.onorca.dev', - '--admin-audience', - 'https://relay.onorca.dev/v1/admin/drain' - ] - const config = parseArguments(base) - assert.deepEqual(directorDeploymentEnvironment(config), { - ...DIRECTOR_ADMISSION_ENVIRONMENT, - ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3', - ORCA_RELAY_IMAGE_DIGEST: `sha256:${'a'.repeat(64)}`, - [DIRECTOR_REHOME_IDENTITY_ENV]: - 'relay-director@onorca-cloud.iam.gserviceaccount.com', - [DIRECTOR_REHOME_AUDIENCE_ENV]: - 'https://relay.onorca.dev/v1/admin/host-drain' - }) - const missingAudience = [...base] - missingAudience.splice(missingAudience.indexOf('--rehome-audience'), 2) - assert.throws(() => parseArguments(missingAudience), /configured together/) - const invalidOrigin = [...base] - invalidOrigin[invalidOrigin.indexOf('--rehome-control-origin') + 1] = - 'http://relay.onorca.dev' - assert.throws( - () => parseArguments(invalidOrigin), - /HTTPS origin/ - ) - const mutableImage = [...base] - mutableImage[mutableImage.indexOf('--image') + 1] = 'relay:latest' - assert.throws(() => parseArguments(mutableImage), /immutable digest/) -}) - -test('requires durable regional rehome control to be disabled at the exact generation', async () => { - const config = { - 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain', - 'expected-rehome-generation': '7' - } - const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN - process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc' - try { - const control = await assertRegionalRehomeDisabled( - config, - 'https://candidate.example.test', - async (url, init) => { - assert.equal(url, 'https://candidate.example.test/v1/admin/regional-rehome-control') - assert.equal(init.headers.authorization, 'Bearer aaa.bbb.ccc') - return new Response(JSON.stringify({ - v: 1, - control: { generation: 7, enabled: false } - })) - } - ) - assert.equal(control.generation, 7) - await assert.rejects( - assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () => - new Response(JSON.stringify({ - v: 1, - control: { generation: 8, enabled: false } - })) - ), - /expected generation/ - ) - await assert.rejects( - assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () => - new Response(JSON.stringify({ - v: 1, - control: { generation: 7, enabled: true } - })) - ), - /durably disabled/ - ) - } finally { - if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN - else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment - } -}) - -test('rejects literal regional placement changes outside the runtime-setting step', () => { - const base = { - project: 'onorca-cloud', - region: 'us-central1', - service: 'orca-cloud-relay', - image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:${'a'.repeat(64)}`, - role: 'director', - 'release-id': 'regional-kill-switch' - } - const args = Object.entries(base).flatMap(([key, value]) => [`--${key}`, value]) - - assert.doesNotThrow(() => parseArguments(args)) - assert.throws( - () => parseArguments([...args, '--regional-placement-enabled', 'false']), - /audited runtime-setting step/ - ) -}) - -test('appends disabled Asia cells without changing the existing director topology', () => { - const current = [ - { - id: 'production-gce-c26', - url: 'https://c26.relay.onorca.dev', - capacityRequests: 4_000, - initiallyEnabled: true, - connectionHardCap: 1_000, - connectionUnobservedBound: 60 - } - ] - const asia = { - id: 'production-gce-c27', - url: 'https://c27.relay.onorca.dev', - region: 'asia-east2', - capacityRequests: 6_000, - initiallyEnabled: false, - connectionHardCap: 3_000, - connectionUnobservedBound: 60 - } - - assert.deepEqual( - directorCellSetAddition( - JSON.stringify(current), - JSON.stringify([asia, { ...current[0], region: 'us-central1' }]) - ), - { - changed: true, - value: directorDeploymentEnvironment({ - 'director-cells-json': JSON.stringify([asia, { ...current[0], region: 'us-central1' }]) - }).ORCA_RELAY_CELLS_JSON - } - ) - const exact = JSON.stringify([{ ...current[0], region: 'us-central1' }, asia]) - assert.deepEqual(directorCellSetAddition(exact, exact), { - changed: false, - value: directorDeploymentEnvironment({ 'director-cells-json': exact }) - .ORCA_RELAY_CELLS_JSON - }) - assert.throws( - () => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1', capacityRequests: 6_000 }, asia])), - /changes an existing cell/ - ) - assert.throws( - () => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1' }, { ...asia, initiallyEnabled: true }])), - /must start disabled/ - ) -}) - -test('pins a director startup probe above the bounded reconciliation window', () => { - assert.deepEqual(directorStartupProbeArguments('director'), [ - '--startup-probe', - 'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1' - ]) - assert.deepEqual(directorStartupProbeArguments('cell'), []) -}) - -test('bounds traffic tags by the Cloud Run service-plus-tag contract', () => { - const service = 'orca-cloud-relay-staging-c1' - const candidate = cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a') - assert.match(candidate, /^candidate-[a-f0-9]{9}$/) - assert.equal(service.length + candidate.length, 46) - assert.equal(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a')) - assert.notEqual(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-2-19cc312a')) - assert.throws(() => cloudRunTrafficTag(`${service}-too-long`, 'candidate', 'release')) -}) - -test('derives and validates a Cloud Run tagged revision origin', () => { - assert.equal( - taggedRevisionOrigin( - 'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app', - 'candidate-123' - ), - 'https://candidate-123---orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app' - ) - assert.throws(() => taggedRevisionOrigin('https://relay-staging.onorca.dev', 'candidate-123')) - assert.throws(() => - taggedRevisionOrigin( - 'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app', - '123-invalid' - ) - ) -}) - -test('requires exactly one active revision and reads queried tag metadata', () => { - const service = { - status: { - traffic: [ - { percent: 100, revisionName: 'relay-00001-old' }, - { - percent: 0, - revisionName: 'relay-00002-new', - tag: 'candidate-123', - url: 'https://candidate-123---relay-hash-uc.a.run.app' - } - ] - } - } - assert.equal(activeRevision(service), 'relay-00001-old') - assert.deepEqual(taggedTraffic(service, 'candidate-123'), { - origin: 'https://candidate-123---relay-hash-uc.a.run.app', - revision: 'relay-00002-new' - }) - assert.throws(() => - activeRevision({ status: { traffic: [{ percent: 50 }, { percent: 50 }] } }) - ) - assert.deepEqual(trafficTags(service), ['candidate-123']) -}) - -const adminToken = (email) => - `e30.${Buffer.from(JSON.stringify(email ? { email } : {})).toString('base64url')}.sig` - -function directorHarness({ - deployFailure, - deleteFailure, - cleanupReportsFailure = false, - cleanupFailsBeforeRemoval = false, - servingMinimum = 1, - servingMaximum = 5, - // Reproduces gcloud dropping minScale from a newly created revision. - dropRequestedMinimum = false, - servingServiceAccount, - servingImageDigest = `sha256:${'f'.repeat(64)}`, - reserveCells = {}, - servingEnvironment = {}, - placements = {}, - callerEmail, - // The revision template's Cloud SQL transport; a Terraform apply changes it under the serving one. - templateTransport, - spreadFailure, - // The service-level cap gcloud's `--max` sets; Cloud Run divides it by traffic share. - serviceMaximum, - serviceCapSticks = false -} = {}) { - const state = { - activeRevision: 'relay-00001-old', - tags: new Map([['candidate-old', 'relay-00000-stale']]), - revisions: new Map([ - ['relay-00000-stale', { env: { ORCA_RELAY_ROLE: 'director' }, minimum: 1, maximum: 5 }], - [ - 'relay-00001-old', - { - env: { - ORCA_RELAY_ROLE: 'director', - ...servingEnvironment - }, - secrets: { - [DIRECTOR_REGIONAL_PLACEMENT_ENV]: { - secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, - version: '1' - } - }, - minimum: servingMinimum, - maximum: servingMaximum, - serviceAccount: servingServiceAccount, - image: `relay@${servingImageDigest}` - } - ] - ]), - nextRevision: 2, - latestCreated: 'relay-00001-old', - serviceMaximum, - serviceCapClears: 0, - templateTransport - } - const removed = [] - const healthProtocols = [] - const trafficMoves = [] - const spreadWaits = [] - let pendingCleanupFailure = cleanupFailsBeforeRemoval - const operations = { - describeService: () => ({ - metadata: { - annotations: state.serviceMaximum === undefined - ? {} - : { 'run.googleapis.com/maxScale': String(state.serviceMaximum) } - }, - status: { - url: 'https://relay-hash-uc.a.run.app', - latestCreatedRevisionName: state.latestCreated, - traffic: [ - { percent: 100, revisionName: state.activeRevision }, - ...[...state.tags].map(([tag, revisionName]) => ({ - tag, - revisionName, - url: `https://${tag}---relay-hash-uc.a.run.app` - })) - ] - } - }), - describeRevision: (_config, revision) => { - const value = state.revisions.get(revision) - return { - metadata: { - annotations: { - 'autoscaling.knative.dev/minScale': String(value?.minimum ?? 0), - 'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5), - ...value?.transport?.annotations - } - }, - spec: { - serviceAccountName: value?.serviceAccount, - containers: [ - { - name: value?.transport?.name, - image: value?.image, - env: [ - ...Object.entries(value?.env ?? {}).map(([name, value]) => ({ name, value })), - ...Object.entries(value?.secrets ?? {}).map(([name, secretKeyRef]) => ({ - name, - valueSource: { secretKeyRef } - })) - ] - }, - ...value?.transport?.sidecars ?? [] - ] - } - } - }, - deployCandidate: (config, tag, env, image, minimum, maximum, regionalVersion) => { - const revision = `relay-${String(state.nextRevision++).padStart(5, '0')}-new` - state.revisions.set(revision, { - // gcloud --update-env-vars keeps every variable the deploy does not name. - env: { ORCA_RELAY_ROLE: 'director', ...servingEnvironment, ...env }, - secrets: { - [DIRECTOR_REGIONAL_PLACEMENT_ENV]: { - secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, - version: regionalVersion - } - }, - minimum: dropRequestedMinimum ? 0 : Number(minimum ?? 1), - maximum, - serviceAccount: - config['runtime-service-account'] ?? - state.revisions.get(state.activeRevision)?.serviceAccount, - image: image ?? state.revisions.get(state.activeRevision)?.image, - transport: state.templateTransport - }) - state.latestCreated = revision - state.tags.set(tag, revision) - if (deployFailure) throw deployFailure - }, - clearServiceMaximumInstances: () => { - state.serviceCapClears += 1 - if (!serviceCapSticks) state.serviceMaximum = 100 - }, - listRevisions: () => - [...state.revisions].map(([name]) => ({ metadata: { name } })), - deleteRevision: (_config, revision) => { - if (deleteFailure) throw deleteFailure - state.revisions.delete(revision) - }, - updateTraffic: (_config, args) => { - const remove = args.find((argument) => argument.startsWith('--remove-tags=')) - if (remove) { - const tags = remove.slice('--remove-tags='.length).split(',') - removed.push(tags) - if (pendingCleanupFailure && tags.includes('candidate-new')) { - pendingCleanupFailure = false - throw new Error('failed to remove promoted candidate tag') - } - for (const tag of tags) state.tags.delete(tag) - if (cleanupReportsFailure) throw new Error('gcloud reported failed latest revision') - return - } - const move = args.find((argument) => /^--to-(tags|revisions)=/.test(argument)) - assert.ok(move) - trafficMoves.push(move) - const [target, percent] = move.slice(move.indexOf('=') + 1).split('=') - if (percent !== '100') return - state.activeRevision = move.startsWith('--to-tags=') ? state.tags.get(target) : target - }, - waitForHealth: async (_origin, connectionCapacityProtocol) => { - healthProtocols.push(connectionCapacityProtocol) - }, - waitForServingInstances: async (_config, revision, required) => { - spreadWaits.push({ revision, required }) - if (spreadFailure) throw spreadFailure - }, - readReserveCells: async (_config, origin) => reserveCells[origin], - readDirectorPlacement: async (_config, origin) => placements[origin] ?? 'unreported', - adminIdentityToken: () => adminToken(callerEmail) - } - return { state, removed, healthProtocols, trafficMoves, spreadWaits, operations } -} - -test('director deploy removes stale and promoted Cloud Run tags', async () => { - const harness = directorHarness() - const config = { - project: 'onorca-cloud-staging', - 'capacity-service-account': - 'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com' - } - await deployDirector(config, 'candidate-new', harness.operations) - assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']]) - assert.equal(harness.state.activeRevision, 'relay-00003-new') - assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) - assert.deepEqual([...harness.state.revisions.keys()], [ - 'relay-00000-stale', - 'relay-00001-old', - 'relay-00002-new', - 'relay-00003-new' - ]) - for (const revision of ['relay-00002-new', 'relay-00003-new']) { - assert.deepEqual( - Object.fromEntries( - Object.entries(harness.state.revisions.get(revision).env).filter(([key]) => - key in DIRECTOR_ADMISSION_ENVIRONMENT - ) - ), - DIRECTOR_ADMISSION_ENVIRONMENT - ) - assert.equal( - harness.state.revisions.get(revision).env.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT, - config['capacity-service-account'] - ) - } - assert.deepEqual(harness.healthProtocols, [undefined, undefined]) -}) - -test('director deploy stamps the durable regional placement secret reference', async () => { - const harness = directorHarness() - await deployDirector({}, 'candidate-new', harness.operations) - for (const revision of ['relay-00002-new', 'relay-00003-new']) { - assert.deepEqual( - harness.state.revisions.get(revision).secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV], - { secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, version: '1' } - ) - } -}) - -test('bootstraps both rollback and candidate onto the distinct director identity', async () => { - const predecessor = 'relay-runtime@onorca-cloud.iam.gserviceaccount.com' - const director = 'relay-director@onorca-cloud.iam.gserviceaccount.com' - const harness = directorHarness({ servingServiceAccount: predecessor }) - const inspected = [] - harness.operations.assertRegionalRehomeDisabled = async (_config, origin) => { - inspected.push(origin) - } - await deployDirector({ - project: 'onorca-cloud', - 'runtime-service-account': director, - 'predecessor-runtime-service-account': predecessor, - 'predecessor-image-digest': `sha256:${'f'.repeat(64)}`, - 'bootstrap-runtime-identity': 'true', - 'expected-rehome-generation': '0', - 'rehome-control-origin': 'https://relay.onorca.dev' - }, 'candidate-new', harness.operations) - assert.equal( - harness.state.revisions.get(harness.state.activeRevision).serviceAccount, - director - ) - assert.equal( - harness.state.revisions.get(harness.state.tags.get('selector-rollback')).serviceAccount, - director - ) - assert.deepEqual(inspected, [ - 'https://selector-rollback---relay-hash-uc.a.run.app', - 'https://candidate-new---relay-hash-uc.a.run.app' - ]) -}) - -test('steady-state director deploy rejects the predecessor identity', async () => { - const harness = directorHarness({ - servingServiceAccount: 'relay-runtime@onorca-cloud.iam.gserviceaccount.com' - }) - await assert.rejects(deployDirector({ - 'runtime-service-account': 'relay-director@onorca-cloud.iam.gserviceaccount.com' - }, 'candidate-new', harness.operations), /unexpected runtime service account/) -}) - -test('director deploy prunes old revisions when requested', async () => { - const harness = directorHarness() - await deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations) - assert.deepEqual([...harness.state.revisions.keys()], [ - 'relay-00002-new', - 'relay-00003-new' - ]) - assert.deepEqual(harness.healthProtocols, [2, 2]) -}) - -test('a prune failure preserves the active and rollback traffic pair', async () => { - const harness = directorHarness({ deleteFailure: new Error('injected delete failure') }) - await assert.rejects( - deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations), - /injected delete failure/ - ) - assert.equal(harness.state.activeRevision, 'relay-00003-new') - assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) -}) - -test('director deploy removes a candidate tag left by a failed update', async () => { - const failure = new Error('candidate failed to become ready') - const harness = directorHarness({ deployFailure: failure }) - await assert.rejects(deployDirector({}, 'candidate-new', harness.operations), failure) - assert.deepEqual(harness.removed, [['candidate-old'], ['selector-rollback']]) - assert.equal(harness.state.tags.size, 0) -}) - -test('director candidate inherits the serving warm-instance floor', async () => { - const harness = directorHarness({ servingMinimum: 5 }) - await deployDirector({}, 'candidate-new', harness.operations) - const serving = harness.state.revisions.get(harness.state.activeRevision) - assert.equal(serving.minimum, 5) - // The standby rollback revision must stay cold. - const rollback = harness.state.revisions.get(harness.state.tags.get('selector-rollback')) - assert.equal(rollback.minimum, 0) -}) - -test('a multi-instance floor moves part of the traffic until every floor instance serves', async () => { - const harness = directorHarness({ servingMinimum: 5 }) - await deployDirector({}, 'candidate-new', harness.operations) - assert.deepEqual(harness.trafficMoves, ['--to-tags=candidate-new=20', '--to-tags=candidate-new=100']) - assert.deepEqual(harness.spreadWaits, [{ revision: 'relay-00003-new', required: 5 }]) - assert.equal(harness.state.activeRevision, 'relay-00003-new') -}) - -test('a candidate pins its own maximum, never the service-wide cap', () => { - const args = candidateUpdateArguments( - { service: 'relay', project: 'p', region: 'r', role: 'director' }, - 'candidate-new', - {}, - 'relay@sha256:abc', - 5, - 5, - '1' - ) - assert.equal(args[args.indexOf('--max-instances') + 1], '5') - assert.ok(!args.includes('--max')) -}) - -test('a service cap below two full revisions is cleared before any candidate exists', async () => { - const harness = directorHarness({ servingMinimum: 5, serviceMaximum: 5 }) - await deployDirector({ 'max-instances': '5' }, 'candidate-new', harness.operations) - assert.equal(harness.state.serviceCapClears, 1) - assert.equal(harness.state.serviceMaximum, 100) - assert.equal(harness.state.activeRevision, 'relay-00003-new') -}) - -test('a service cap with room for two revisions, or none, is left alone', async () => { - for (const serviceMaximum of [undefined, 10, 100]) { - const harness = directorHarness({ servingMinimum: 5, serviceMaximum }) - await deployDirector({ 'max-instances': '5' }, 'candidate-new', harness.operations) - assert.equal(harness.state.serviceCapClears, 0) - } -}) - -test('a service cap that will not clear stops the deploy before traffic moves', async () => { - const harness = directorHarness({ servingMinimum: 5, serviceMaximum: 5, serviceCapSticks: true }) - await assert.rejects( - deployDirector({ 'max-instances': '5' }, 'candidate-new', harness.operations), - /still caps instances at 5, below two revisions of 5/ - ) - assert.equal(harness.state.activeRevision, 'relay-00001-old') - assert.deepEqual(harness.trafficMoves, []) -}) - -test('a floor of one moves all traffic at once', async () => { - const harness = directorHarness() - await deployDirector({}, 'candidate-new', harness.operations) - assert.deepEqual(harness.trafficMoves, ['--to-tags=candidate-new=100']) - assert.deepEqual(harness.spreadWaits, []) -}) - -test('a candidate that never spreads returns traffic to the warm serving revision', async () => { - const failure = new Error('candidate served from 1 of 5 instances before the deadline') - const harness = directorHarness({ servingMinimum: 5, spreadFailure: failure }) - await assert.rejects(deployDirector({}, 'candidate-new', harness.operations), failure) - assert.deepEqual(harness.trafficMoves, [ - '--to-tags=candidate-new=20', - '--to-revisions=relay-00001-old=100' - ]) - assert.equal(harness.state.activeRevision, 'relay-00001-old') - assert.equal(harness.state.tags.size, 0) -}) - -test('the spread wait counts distinct instances that served since the partial move', async () => { - const reads = [new Set(['a']), new Set(['a', 'b']), new Set(['a', 'b', 'c'])] - const seen = [] - await waitForServingInstances({}, 'relay-00003-new', 3, '2026-10-20T05:25:00.000Z', { - readInstances: (_config, revision, since) => { - seen.push([revision, since]) - return reads.shift() - }, - pollIntervalMs: 0, - timeoutMs: 60_000 - }) - assert.equal(seen.length, 3) - assert.deepEqual(seen[0], ['relay-00003-new', '2026-10-20T05:25:00.000Z']) -}) - -test('the spread wait retries a failed log read', async () => { - let reads = 0 - await waitForServingInstances({}, 'relay-00003-new', 2, '2026-10-20T05:25:00.000Z', { - readInstances: () => { - reads += 1 - if (reads === 1) throw new Error('RESOURCE_EXHAUSTED') - return new Set(['a', 'b']) - }, - pollIntervalMs: 0, - timeoutMs: 60_000 - }) - assert.equal(reads, 2) -}) - -test('the spread wait fails at its deadline', async () => { - await assert.rejects( - waitForServingInstances({}, 'relay-00003-new', 5, '2026-10-20T05:25:00.000Z', { - readInstances: () => new Set(['a']), - pollIntervalMs: 0, - timeoutMs: 0 - }), - /served from 1 of 5 instances before the deadline/ - ) -}) - -test('director deploy rejects a serving maximum above the checked budget', async () => { - const harness = directorHarness({ servingMaximum: 6 }) - await assert.rejects( - deployDirector({ 'max-instances': '5' }, 'candidate-new', harness.operations), - /holds 6 maximum instances, expected 5/ - ) -}) - -test('an explicit --min-instances still overrides the serving floor', async () => { - const harness = directorHarness({ servingMinimum: 5 }) - await deployDirector({ 'min-instances': '0' }, 'candidate-new', harness.operations) - assert.equal(harness.state.revisions.get(harness.state.activeRevision).minimum, 0) -}) - -test('director deploy refuses to move traffic onto a candidate that lost the floor', async () => { - const harness = directorHarness({ servingMinimum: 5, dropRequestedMinimum: true }) - await assert.rejects( - deployDirector({}, 'candidate-new', harness.operations), - /candidate holds 0 minimum instances, expected 5/ - ) - // Traffic never moved, so the original revision still serves. - assert.equal(harness.state.activeRevision, 'relay-00001-old') -}) - -const SERVING_IMAGE = `relay@sha256:${'f'.repeat(64)}` - -// What a Terraform apply of relay_director_cloud_sql_proxy_sidecar puts in the template. -const SIDECAR_TRANSPORT = { - name: 'relay', - annotations: { 'run.googleapis.com/container-dependencies': '{"relay":["cloud-sql-proxy"]}' }, - sidecars: [{ name: 'cloud-sql-proxy', image: 'gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:1' }] -} - -// The apply leaves its revision at 0%, as the director service declares no traffic block. -function applySidecarTemplate(harness) { - const serving = harness.state.revisions.get(harness.state.activeRevision) - harness.state.templateTransport = SIDECAR_TRANSPORT - harness.state.revisions.set('relay-00009-terraform', { ...serving, transport: SIDECAR_TRANSPORT }) - harness.state.latestCreated = 'relay-00009-terraform' -} - -test('a deploy after a sidecar apply refuses the new shape unless the apply revision is named', async () => { - const harness = directorHarness() - applySidecarTemplate(harness) - await assert.rejects( - deployDirector({}, 'candidate-new', harness.operations), - /unrelated revision shape/ - ) - assert.equal(harness.state.activeRevision, 'relay-00001-old') -}) - -test('a deploy adopts the shape of the named Terraform apply revision', async () => { - const harness = directorHarness() - applySidecarTemplate(harness) - await deployDirector( - { image: SERVING_IMAGE, 'shape-baseline-revision': 'relay-00009-terraform' }, - 'candidate-new', - harness.operations - ) - const active = harness.state.revisions.get(harness.state.activeRevision) - assert.equal(active.transport, SIDECAR_TRANSPORT) - assert.equal( - harness.state.revisions.get(harness.state.tags.get('selector-rollback')).transport, - SIDECAR_TRANSPORT - ) -}) - -test('the named shape baseline must be the unserved latest revision on the serving image', async () => { - const stale = directorHarness() - applySidecarTemplate(stale) - await assert.rejects( - deployDirector({ 'shape-baseline-revision': 'relay-00000-stale' }, 'candidate-new', stale.operations), - /not the latest created revision/ - ) - const serving = directorHarness() - await assert.rejects( - deployDirector({ 'shape-baseline-revision': 'relay-00001-old' }, 'candidate-new', serving.operations), - /already serving/ - ) - const otherImage = directorHarness() - applySidecarTemplate(otherImage) - otherImage.state.revisions.get('relay-00009-terraform').image = `relay@sha256:${'e'.repeat(64)}` - await assert.rejects( - deployDirector({ image: SERVING_IMAGE, 'shape-baseline-revision': 'relay-00009-terraform' }, 'candidate-new', otherImage.operations), - /does not run the serving image/ - ) - const newImage = directorHarness() - applySidecarTemplate(newImage) - await assert.rejects( - deployDirector( - { image: `relay@sha256:${'e'.repeat(64)}`, 'shape-baseline-revision': 'relay-00009-terraform' }, - 'candidate-new', - newImage.operations - ), - /must keep the serving image/ - ) - for (const harness of [stale, serving, otherImage, newImage]) { - assert.equal(harness.state.activeRevision, 'relay-00001-old') - } -}) - -test('--shape-baseline-revision takes only a revision name, and only for a director', () => { - const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', 'i@sha256:' + 'a'.repeat(64), '--release-id', 'x'] - assert.throws( - () => parseArguments([...base, '--role', 'director', '--shape-baseline-revision', 'Bad_Name']), - /must be a Cloud Run revision name/ - ) - assert.throws( - () => parseArguments([...base, '--role', 'cell', '--director-origin', 'https://d', '--admin-audience', 'a', '--shape-baseline-revision', 'relay-1']), - /--shape-baseline-revision requires --role director/ - ) - const director = [...base, '--role', 'director', '--shape-baseline-revision', 'relay-1'] - assert.equal(parseArguments(director)['shape-baseline-revision'], 'relay-1') - for (const [extra, refusal] of [ - [['--bootstrap-runtime-identity', 'true'], /cannot combine with a runtime identity bootstrap/], - [['--capacity-cell-id', 'c1', '--director-cells-json', '[]'], /cannot combine with a director topology change/], - [['--director-cells-json', '[]'], /cannot combine with a director topology change/], - [['--prune-revisions', 'true'], /cannot combine with revision pruning/] - ]) { - assert.throws(() => parseArguments([...director, ...extra]), refusal) - } -}) - -test('director deploy rejects unrelated revision-shape drift', async () => { - const harness = directorHarness() - const describeRevision = harness.operations.describeRevision - harness.operations.describeRevision = (config, revision) => { - const described = describeRevision(config, revision) - described.spec.containerConcurrency = revision === 'relay-00001-old' ? 80 : 1_000 - return described - } - await assert.rejects( - deployDirector({}, 'candidate-new', harness.operations), - /unrelated revision shape/ - ) - assert.equal(harness.state.activeRevision, 'relay-00001-old') -}) - -test('director cleanup verifies success when gcloud reports a stale revision failure', async () => { - const harness = directorHarness({ cleanupReportsFailure: true }) - await deployDirector({}, 'candidate-new', harness.operations) - assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']]) - assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) -}) - -test('director deploy restores rollback traffic after promoted-tag cleanup fails', async () => { - const harness = directorHarness({ cleanupFailsBeforeRemoval: true }) - await assert.rejects( - deployDirector({}, 'candidate-new', harness.operations), - /failed to remove promoted candidate tag/ - ) - assert.equal( - harness.state.activeRevision, - harness.state.tags.get('selector-rollback') - ) - assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback']) -}) - -test('reads only literal revision environment values', () => { - assert.deepEqual( - revisionEnvironment({ - spec: { - containers: [ - { - env: [ - { name: 'ORCA_RELAY_CELL_ID', value: 'staging-c1' }, - { name: 'ORCA_RELAY_CELL_CAPACITY', value: '900' }, - { name: 'DATABASE_URL', valueFrom: { secretKeyRef: { name: 'database' } } } - ] - } - ] - } - }), - { ORCA_RELAY_CELL_ID: 'staging-c1', ORCA_RELAY_CELL_CAPACITY: '900' } - ) -}) - -test('reads only Secret Manager revision environment references', () => { - assert.deepEqual( - revisionSecretEnvironment({ - spec: { - containers: [{ env: [ - { name: 'LITERAL', value: 'true' }, - { - name: DIRECTOR_REGIONAL_PLACEMENT_ENV, - valueSource: { secretKeyRef: { - secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, - version: 'latest' - } } - }, - { - name: 'GCP_SECRET_SHAPE', - valueFrom: { secretKeyRef: { name: 'gcp-secret', key: '2' } } - } - ] }] - } - }), - { - [DIRECTOR_REGIONAL_PLACEMENT_ENV]: { - secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, - version: 'latest' - }, - GCP_SECRET_SHAPE: { - secret: 'gcp-secret', - version: '2' - } - } - ) -}) - -test('accepts only a bounded JWT-shaped supplied admin identity token', () => { - assert.equal(suppliedAdminIdentityToken({}), null) - assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc') - assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' })) - assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' })) - assert.throws(() => - suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` }) - ) -}) - -test('waits for authenticated target readiness without hiding other capacity errors', async () => { - let attempts = 0 - const capacity = await waitForEvacuationCapacity( - async () => { - attempts += 1 - if (attempts < 3) throw new Error('/v1/admin/evacuation-capacity failed: target_cell_unavailable') - return { requiredTargetUnits: 2, availableTargetUnits: 4_000 } - }, - 'source', - 'target', - { pollIntervalMs: 1, timeoutMs: 100 } - ) - assert.equal(attempts, 3) - assert.equal(capacity.availableTargetUnits, 4_000) - await assert.rejects( - waitForEvacuationCapacity(async () => { - throw new Error('/v1/admin/evacuation-capacity failed: forbidden') - }, 'source', 'target'), - /forbidden/ - ) -}) - - -test('validates bounded correction cohorts and leaves unspecified values to serving inheritance', () => { - for (const value of ['0', '1', '100']) assert.equal(correctionCohortPercent(value), value) - for (const value of ['-1', '101', '1.5', '', '01', 'true', '1\n']) { - assert.throws(() => correctionCohortPercent(value), /integer from 0 to 100/) - } - assert.equal(directorDeploymentEnvironment({})[DIRECTOR_CORRECTION_COHORT_ENV], undefined) - assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': 'preserve' })[DIRECTOR_CORRECTION_COHORT_ENV], undefined) - assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': '1' })[DIRECTOR_CORRECTION_COHORT_ENV], '1') -}) - -test('inherits the cohort on candidate and rollback revisions without resetting an enabled cohort', async () => { - const harness = directorHarness() - harness.state.revisions.get('relay-00001-old').env[DIRECTOR_CORRECTION_COHORT_ENV] = '3' - await deployDirector({}, 'candidate-new', harness.operations) - for (const revision of ['relay-00002-new', 'relay-00003-new']) { - assert.equal(harness.state.revisions.get(revision).env[DIRECTOR_CORRECTION_COHORT_ENV], '3') - } -}) - -test('starts an unstamped cohort at zero and rejects a cohort change without disabled-control proof', async () => { - const harness = directorHarness() - await assert.rejects(deployDirector({ 'region-correction-cohort-percent': '1' }, - 'candidate-new', harness.operations), /exact disabled regional-rehome generation/) - assert.equal(harness.state.activeRevision, 'relay-00001-old') - assert.equal(harness.state.nextRevision, 2) - await deployDirector({}, 'candidate-new', harness.operations) - assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '0') -}) - -test('sets a reviewed cohort only behind repeated disabled-control verification', async () => { - const harness = directorHarness() - let verified = 0 - const config = { 'region-correction-cohort-percent': '1', 'expected-rehome-generation': '7' } - await deployDirector(config, 'candidate-new', { ...harness.operations, - assertRegionalRehomeDisabled: async () => { verified++ } }) - assert.ok(verified >= 2) - assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '1') -}) - -test('director deploy refuses a traffic move between reserve-support revisions while a cell is reserve', async () => { - const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app' - const oldServing = directorHarness({ - reserveCells: { [candidateOrigin]: ['production-gce-c3'] } - }) - await assert.rejects( - deployDirector({}, 'candidate-new', oldServing.operations), - /differ in reserve-mode support while cells are reserve \(production-gce-c3\)/ - ) - assert.equal(oldServing.state.activeRevision, 'relay-00001-old') - const rollbackToOld = directorHarness({ - reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'] } - }) - await assert.rejects(deployDirector({}, 'candidate-new', rollbackToOld.operations), /flip them to db first/) - assert.equal(rollbackToOld.state.activeRevision, 'relay-00001-old') - const allDb = directorHarness({ reserveCells: { [candidateOrigin]: [] } }) - await deployDirector({}, 'candidate-new', allDb.operations) - assert.equal(allDb.state.activeRevision, 'relay-00003-new') -}) - -test('two revisions that both read the census may move traffic with reserve cells', () => { - assert.doesNotThrow(() => assertReserveCompatibleTrafficMove(['c3'], ['c3'])) - assert.doesNotThrow(() => assertReserveCompatibleTrafficMove(undefined, undefined)) - assert.throws(() => assertReserveCompatibleTrafficMove(undefined, ['c3']), /flip them to db first/) -}) - -test('reads a census-less director as unsupported and fails closed on any other refusal', async () => { - const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN - process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc' - const config = { 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain' } - try { - assert.equal( - await readReserveCells(config, 'https://old.example.test', ['c1'], async () => - new Response('404 Not Found', { status: 404 }) - ), - undefined - ) - const modes = { c1: 'db', c2: 'reserve' } - assert.deepEqual( - await readReserveCells(config, 'https://new.example.test', ['c1', 'c2'], async (url, init) => { - assert.equal(url, 'https://new.example.test/v1/admin/cell-admit-mode') - const { cellId } = JSON.parse(init.body) - return new Response(JSON.stringify({ v: 1, cellId, admitMode: modes[cellId], updatedAt: 1 })) - }), - ['c2'] - ) - await assert.rejects( - readReserveCells(config, 'https://new.example.test', ['c1'], async () => - new Response(JSON.stringify({ error: 'invalid_token' }), { status: 401 }) - ), - /cell admit mode read failed for c1: invalid_token/ - ) - } finally { - if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN - else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment - } -}) - -test('a same-image deploy reads no admit modes, and keeps the serving reserve placement', async () => { - const servingImageDigest = `sha256:${'f'.repeat(64)}` - const harness = directorHarness({ servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' } }) - let reads = 0 - harness.operations.readReserveCells = async () => { - reads += 1 - return [] - } - await deployDirector({ image: `relay@${servingImageDigest}` }, 'candidate-new', harness.operations) - assert.equal(reads, 0) - assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'on') -}) - -test('turning reserve placement off is refused while any cell is reserve, even on the same image', async () => { - const servingImageDigest = `sha256:${'f'.repeat(64)}` - const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app' - const reserved = directorHarness({ - servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' }, - reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'], [candidateOrigin]: ['production-gce-c3'] }, - placements: { 'https://relay-hash-uc.a.run.app': 'on', [candidateOrigin]: 'off' } - }) - await assert.rejects( - deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', reserved.operations), - /reserve placement would turn off while cells are reserve \(production-gce-c3\); flip them to db first/ - ) - assert.equal(reserved.state.activeRevision, 'relay-00001-old') - const allDb = directorHarness({ - servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' }, - reserveCells: { 'https://relay-hash-uc.a.run.app': [], [candidateOrigin]: [] } - }) - await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', allDb.operations) - assert.equal(allDb.state.revisions.get(allDb.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'off') -}) - -test('--reserve-placement takes only a known mode, and only for a director', () => { - const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', `relay@sha256:${'a'.repeat(64)}`, '--release-id', '1'] - assert.equal(parseArguments([...base, '--role', 'director', '--reserve-placement', 'on'])['reserve-placement'], 'on') - assert.throws(() => parseArguments([...base, '--role', 'director', '--reserve-placement', 'yes']), /must be preserve, off, dry-run or on/) - assert.throws( - () => parseArguments([...base, '--role', 'cell', '--director-origin', 'https://d', '--admin-audience', 'a', '--reserve-placement', 'on']), - /require --role director/ - ) - assert.equal(directorDeploymentEnvironment({ 'reserve-placement': 'preserve' })[DIRECTOR_RESERVE_PLACEMENT_ENV], undefined) - assert.equal(directorDeploymentEnvironment({ 'reserve-placement': 'dry-run' })[DIRECTOR_RESERVE_PLACEMENT_ENV], 'dry-run') -}) - -test('the guard reads placement as built, so a director that never placed does not block turning it off', async () => { - const servingImageDigest = `sha256:${'f'.repeat(64)}` - const candidateOrigin = 'https://candidate-new---relay-hash-uc.a.run.app' - // Configured on, but the serving director reports off: it was never feeding the dead-man. - const harness = directorHarness({ - servingEnvironment: { [DIRECTOR_RESERVE_PLACEMENT_ENV]: 'on' }, - reserveCells: { 'https://relay-hash-uc.a.run.app': ['production-gce-c3'], [candidateOrigin]: ['production-gce-c3'] }, - placements: { 'https://relay-hash-uc.a.run.app': 'off', [candidateOrigin]: 'off' } - }) - await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', harness.operations) - assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_RESERVE_PLACEMENT_ENV], 'off') -}) - -test('a shadow-cell change on the same image is checked, and a shrink past a reserve cell is refused', () => { - assert.throws( - () => assertShadowCellsKeepReserveCells('production-gce-c1,production-gce-c3', 'production-gce-c1', ['production-gce-c3']), - /would drop reserve cells \(production-gce-c3\); flip them to db first/ - ) - assert.throws(() => assertShadowCellsKeepReserveCells('all', 'production-gce-c1', ['production-gce-c3']), /production-gce-c3/) - assert.throws(() => assertShadowCellsKeepReserveCells('all', undefined, ['production-gce-c3']), /production-gce-c3/) - assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1', 'all', ['production-gce-c3'])) - assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1,production-gce-c3', 'production-gce-c1', [])) - // A reserve cell the serving list never polled is not being dropped by this deploy. - assert.doesNotThrow(() => assertShadowCellsKeepReserveCells('production-gce-c1', 'production-gce-c1', ['production-gce-c3'])) -}) - -test('reads a director placement from runtime-status, and reports an image without the field', async () => { - const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN - process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc' - const config = { 'admin-audience': 'https://relay.onorca.dev/v1/admin/drain' } - try { - const answer = (body) => async () => new Response(JSON.stringify({ v: 1, role: 'director', ...body })) - assert.equal(await readDirectorPlacement(config, 'https://d.example.test', answer({ reservePlacement: 'on' })), 'on') - assert.equal(await readDirectorPlacement(config, 'https://d.example.test', answer({})), 'unreported') - await assert.rejects(readDirectorPlacement(config, 'https://d.example.test', answer({ reservePlacement: 'maybe' })), /reports reserve placement maybe/) - await assert.rejects( - readDirectorPlacement(config, 'https://d.example.test', async () => new Response('{}', { status: 401 })), - /runtime status failed/ - ) - } finally { - if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN - else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment - } -}) - -// Why: staging drifted to a director that trusts gha-deploy while every staging workflow -// authenticates as gha-relay; the deploy, not Terraform, owns the live value. -test('a director deploy sets the trusted deploy identity only when asked', () => { - const config = { project: 'onorca-cloud-staging' } - const relay = 'orca-cloud-staging-gha-relay@onorca-cloud-staging.iam.gserviceaccount.com' - assert.equal( - directorDeploymentEnvironment({ ...config, 'deploy-service-account': relay }) - .ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT, - relay - ) - assert.equal( - 'ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT' in directorDeploymentEnvironment(config), - false - ) - assert.throws( - () => - directorDeploymentEnvironment({ - ...config, - 'deploy-service-account': 'foreign@other-project.iam.gserviceaccount.com' - }), - /selected project/ - ) -}) - -// Staging's director trusted gha-deploy while the workflow authenticated as gha-relay: a bare 401. -test('the guard names the deploy identity a drifted director trusts instead of reading into a 401', async () => { - const servingImageDigest = `sha256:${'f'.repeat(64)}` - const harness = directorHarness({ - servingEnvironment: { ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'gha-deploy@p.iam.gserviceaccount.com' }, - callerEmail: 'gha-relay@p.iam.gserviceaccount.com' - }) - await assert.rejects( - deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', harness.operations), - /serving director trusts gha-deploy@p\.iam\.gserviceaccount\.com for deploy admin reads, but this deploy authenticates as gha-relay@p\.iam\.gserviceaccount\.com/ - ) - const trusted = directorHarness({ - servingEnvironment: { ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT: 'gha-relay@p.iam.gserviceaccount.com' }, - callerEmail: 'gha-relay@p.iam.gserviceaccount.com' - }) - await deployDirector({ image: `relay@${servingImageDigest}`, 'reserve-placement': 'off' }, 'candidate-new', trusted.operations) -}) - -test('a director deploy takes --admin-audience alone for the reserve guard', () => { - const base = ['--project', 'p', '--region', 'r', '--service', 's', '--image', `relay@sha256:${'e'.repeat(64)}`, '--role', 'director', '--release-id', 'x'] - const audience = ['--admin-audience', 'https://relay.example.test'] - assert.equal(parseArguments([...base, ...audience])['admin-audience'], 'https://relay.example.test') - assert.throws( - () => parseArguments([...base, '--expected-rehome-generation', '1', '--rehome-control-origin', 'https://relay.example.test']), - /durable rehome verification arguments must be configured together/ - ) - assert.throws( - () => parseArguments([...base, ...audience, '--expected-rehome-generation', '1']), - /durable rehome verification arguments must be configured together/ - ) -}) - -test('--shadow-seat-feed-cells sets the director setting, and preserve keeps the serving one', async () => { - assert.equal( - directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'c1, c2' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], - 'c1,c2' - ) - assert.equal(directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'all' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], 'all') - assert.equal(directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'preserve' })[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], undefined) - assert.throws(() => directorDeploymentEnvironment({ 'shadow-seat-feed-cells': 'C1;x' }), /must be preserve, all, or a comma list/) - const servingImageDigest = `sha256:${'f'.repeat(64)}` - const harness = directorHarness({ servingEnvironment: { [DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]: 'c1,c3' } }) - await deployDirector({ image: `relay@${servingImageDigest}`, 'shadow-seat-feed-cells': 'preserve' }, 'candidate-new', harness.operations) - assert.equal(harness.state.revisions.get(harness.state.activeRevision).env[DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV], 'c1,c3') - const shrink = directorHarness({ - servingEnvironment: { [DIRECTOR_SHADOW_SEAT_FEED_CELLS_ENV]: 'c1,c3' }, - reserveCells: { 'https://relay-hash-uc.a.run.app': ['c3'], 'https://candidate-new---relay-hash-uc.a.run.app': ['c3'] } - }) - await assert.rejects( - deployDirector({ image: `relay@${servingImageDigest}`, 'shadow-seat-feed-cells': 'c1' }, 'candidate-new', shrink.operations), - /would drop reserve cells \(c3\)/ - ) -}) diff --git a/cloud/dev/scripts/deploy-relay-gce-candidate.mjs b/cloud/dev/scripts/deploy-relay-gce-candidate.mjs deleted file mode 100644 index 0c324dda5ea..00000000000 --- a/cloud/dev/scripts/deploy-relay-gce-candidate.mjs +++ /dev/null @@ -1,876 +0,0 @@ -import { readFileSync } from 'node:fs' -import { spawnSync } from 'node:child_process' -import { pathToFileURL } from 'node:url' -import { - inspectAdmissionSelector, - selectorCellState, - transitionAdmissionSelector -} from './relay-admission-selector.mjs' -import { isReserveCellRefusal, reserveCellRollRefusal } from './relay-reserve-cell-refusal.mjs' - -const DEFAULT_POLL_INTERVAL_MS = 5_000 -const DEFAULT_TIMEOUT_MS = 14 * 60 * 1_000 -const ADMIN_RETRY_ATTEMPTS = 3 -const ADMIN_RETRY_BASE_MS = 250 -const CONNECTION_CONTROL_REBIND_RESERVE = 100 -const SUPPORTED_CONNECTION_HARD_CAPS = new Set([600, 1_000, 3_000]) -const RETRYABLE_ADMIN_PATHS = new Set([ - '/v1/admin/runtime-status', - '/v1/admin/cell-status', - '/v1/admin/evacuation-capacity', - '/v1/admin/evacuation-status' -]) - -function canonicalOrigin(value, name) { - const url = new URL(value) - if (url.protocol !== 'https:' || url.origin !== value || url.pathname !== '/') { - throw new Error(`${name} must be a canonical HTTPS origin`) - } - return value -} - -function adminAudience(value) { - const url = new URL(value) - if ( - url.protocol !== 'https:' || - url.pathname !== '/v1/admin/drain' || - url.search || - url.hash || - url.toString() !== value - ) { - throw new Error('--admin-audience must be the canonical HTTPS director drain URL') - } - return value -} - -function positiveInteger(value, name, maximum = Number.MAX_SAFE_INTEGER) { - const parsed = Number(value) - if (!Number.isInteger(parsed) || parsed <= 0 || parsed > maximum) { - throw new Error(`${name} must be a positive integer`) - } - return parsed -} - -function nonnegativeInteger(value, name) { - const parsed = Number(value) - if (!Number.isInteger(parsed) || parsed < 0) { - throw new Error(`${name} must be a nonnegative integer`) - } - return parsed -} - -export function parseArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`) - values[key.slice(2)] = value - } - for (const key of [ - 'project', - 'director-origin', - 'admin-audience', - 'topology-file', - 'source-cell-id', - 'target-cell-id', - 'runtime-service-account', - 'mode' - ]) { - if (!values[key]) throw new Error(`missing --${key}`) - } - if ( - ![ - 'audit', - 'preflight', - 'recover-forward', - 'continue-evacuation', - 'disable-cell', - 'execute', - 'reset-empty-candidate', - 'enable-empty-cell' - ].includes(values.mode) - ) { - throw new Error( - '--mode must be audit, preflight, recover-forward, continue-evacuation, disable-cell, execute, reset-empty-candidate, or enable-empty-cell' - ) - } - return { - project: values.project, - directorOrigin: canonicalOrigin(values['director-origin'], '--director-origin'), - adminAudience: adminAudience(values['admin-audience']), - topologyFile: values['topology-file'], - sourceCellId: values['source-cell-id'], - targetCellId: values['target-cell-id'], - runtimeServiceAccount: values['runtime-service-account'], - mode: values.mode, - batchSize: positiveInteger(values['batch-size'] ?? 100, '--batch-size', 100), - drainGraceMs: positiveInteger( - values['drain-grace-ms'] ?? 120_000, - '--drain-grace-ms', - 60 * 60 * 1_000 - ), - pollIntervalMs: positiveInteger( - values['poll-interval-ms'] ?? DEFAULT_POLL_INTERVAL_MS, - '--poll-interval-ms', - 60_000 - ), - timeoutMs: positiveInteger( - values['timeout-ms'] ?? DEFAULT_TIMEOUT_MS, - '--timeout-ms', - 60 * 60 * 1_000 - ) - } -} - -export function deployment(value, cellId) { - if (!value || typeof value !== 'object') throw new Error(`missing topology for ${cellId}`) - const expected = { - cellId, - origin: canonicalOrigin(value.origin, `${cellId} origin`), - region: String(value.region ?? 'us-central1'), - zone: String(value.zone ?? ''), - migName: String(value.mig_name ?? ''), - instanceGroup: String(value.instance_group ?? ''), - backendName: String(value.backend_name ?? ''), - backendId: String(value.backend_id ?? ''), - urlMapName: String(value.url_map_name ?? ''), - generationIdentity: String(value.generation_identity ?? ''), - image: String(value.image ?? ''), - imageDigest: String(value.image ?? '').split('@')[1] ?? '', - capacityRequests: positiveInteger(value.capacity_requests, `${cellId} capacity`), - databasePoolMax: positiveInteger( - value.database_pool_max ?? 10, - `${cellId} database pool maximum`, - 100 - ), - connectionHardCap: - value.connection_hard_cap === null || value.connection_hard_cap === undefined - ? undefined - : positiveInteger(value.connection_hard_cap, `${cellId} connection hard cap`), - connectionUnobservedBound: - value.connection_unobserved_bound === null || - value.connection_unobserved_bound === undefined - ? undefined - : nonnegativeInteger( - value.connection_unobserved_bound, - `${cellId} unobserved connection bound` - ), - initiallyEnabled: value.initially_enabled, - fenced: value.fenced, - desiredTargetSize: value.desired_target_size - } - if (!/^[a-z0-9-]+$/.test(expected.zone)) throw new Error(`${cellId} has an invalid zone`) - if (!['us-central1', 'asia-east2'].includes(expected.region) || !expected.zone.startsWith(`${expected.region}-`)) { - throw new Error(`${cellId} has an invalid region`) - } - for (const [name, resource] of [ - ['MIG', expected.migName], - ['instance group', expected.instanceGroup], - ['backend', expected.backendName], - ['backend ID', expected.backendId] - ]) { - if (!resource) throw new Error(`${cellId} has no ${name}`) - } - if (!/^[a-z0-9.-]+\/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$/.test(expected.image)) { - throw new Error(`${cellId} image is not digest-pinned`) - } - if (typeof expected.initiallyEnabled !== 'boolean') { - throw new Error(`${cellId} has no initial admission state`) - } - if ( - (expected.connectionHardCap === undefined) !== - (expected.connectionUnobservedBound === undefined) || - (expected.connectionHardCap !== undefined && - (!SUPPORTED_CONNECTION_HARD_CAPS.has(expected.connectionHardCap) || - expected.connectionUnobservedBound >= - expected.connectionHardCap - CONNECTION_CONTROL_REBIND_RESERVE)) - ) { - throw new Error(`${cellId} has invalid connection capacity`) - } - return expected -} - -export function assertDeploymentConnectionCapacity(expected, runtime, director) { - if (expected.connectionHardCap === undefined) { - if (runtime !== null || director !== null) { - throw new Error(`${expected.cellId} connection capacity differs from Terraform`) - } - return - } - const hardCap = expected.connectionHardCap - const unobservedBound = expected.connectionUnobservedBound - const ordinaryConnectionLimit = hardCap - CONNECTION_CONTROL_REBIND_RESERVE - const normalAdmissionPause = ordinaryConnectionLimit - unobservedBound - const matches = (capacity) => - capacity?.hardCap === hardCap && - capacity.controlRebindReserve === CONNECTION_CONTROL_REBIND_RESERVE && - capacity.ordinaryConnectionLimit === ordinaryConnectionLimit && - capacity.unobservedBound === unobservedBound && - capacity.normalAdmissionPause === normalAdmissionPause - if (!matches(runtime) || !matches(director) || director.heartbeatFresh !== true) { - throw new Error(`${expected.cellId} connection capacity differs from Terraform`) - } -} - -export function selectDeployments(topology, sourceCellId, targetCellId) { - if (sourceCellId === targetCellId) throw new Error('source and target cell IDs must differ') - const source = deployment(topology[sourceCellId], sourceCellId) - const target = deployment(topology[targetCellId], targetCellId) - for (const key of ['origin', 'migName', 'instanceGroup', 'backendName', 'backendId']) { - if (source[key] === target[key]) throw new Error(`source and target ${key} overlap`) - } - if (target.initiallyEnabled) throw new Error('candidate must be declared initially disabled') - return { source, target } -} - -export function validateMig(mig, instances, expected) { - if (Number(mig.targetSize) !== 1) throw new Error(`${expected.cellId} MIG is not fixed-one`) - const policy = mig.updatePolicy ?? {} - if ( - policy.replacementMethod !== 'RECREATE' || - Number(policy.maxSurge?.fixed ?? policy.maxSurge) !== 0 || - Number(policy.maxUnavailable?.fixed ?? policy.maxUnavailable) !== 1 - ) { - throw new Error(`${expected.cellId} MIG replacement policy is unsafe`) - } - const serving = instances.filter( - (entry) => entry.instanceStatus === 'RUNNING' && entry.currentAction === 'NONE' - ) - if (instances.length !== 1 || serving.length !== 1) { - throw new Error(`${expected.cellId} MIG must have one running endpoint`) - } - return serving[0].instance.split('/').at(-1) -} - -export function validateInstance(instance, expected, runtimeServiceAccount) { - const publicConfigs = (instance.networkInterfaces ?? []).flatMap( - (network) => network.accessConfigs ?? [] - ) - if (publicConfigs.length !== 0) throw new Error(`${expected.cellId} instance has a public IP`) - const serviceAccounts = (instance.serviceAccounts ?? []).map((entry) => entry.email) - if (serviceAccounts.length !== 1 || serviceAccounts[0] !== runtimeServiceAccount) { - throw new Error(`${expected.cellId} runtime service account mismatch`) - } -} - -export function validateBackend(backend, expected) { - if ( - backend.protocol !== 'HTTP' || - Number(backend.timeoutSec) !== 86_400 || - (backend.backends ?? []).length !== 1 || - backend.backends[0].group !== expected.instanceGroup - ) { - throw new Error(`${expected.cellId} backend topology mismatch`) - } -} - -export function defaultCommandJson(args) { - const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) - if (result.status !== 0) { - throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`) - } - return JSON.parse(result.stdout) -} - -export function suppliedAdminIdentityToken(environment = process.env) { - const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN - if (token === undefined) return null - return validatedIdentityToken(token, 'admin') -} - -export function suppliedFenceMutationIdentityToken(environment = process.env) { - const token = environment.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN - if (token === undefined) return null - return validatedIdentityToken(token, 'fence mutation') -} - -function validatedIdentityToken(token, label) { - // WIF supplies a masked Google ID token because external-account gcloud cannot mint one directly. - if (token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) { - throw new Error(`invalid supplied ${label} identity token`) - } - return token -} - -export function defaultIdentityToken(audience) { - const supplied = suppliedAdminIdentityToken() - if (supplied !== null) return supplied - const result = spawnSync( - 'gcloud', - ['auth', 'print-identity-token', `--audiences=${audience}`], - { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] } - ) - if (result.status !== 0) throw new Error('gcloud identity-token command failed') - return result.stdout.trim() -} - -async function responseJson(response, label, cellId) { - const body = await response.json().catch(() => ({ error: `http_${response.status}` })) - if (isReserveCellRefusal(response.status, body)) throw reserveCellRollRefusal(cellId) - if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`) - return body -} - -export function createAdminPost(config, deps, token) { - return async (origin, path, body) => { - const requestToken = typeof token === 'function' ? token(path) : token - for (let attempt = 1; attempt <= ADMIN_RETRY_ATTEMPTS; attempt++) { - let response - try { - response = await deps.fetch(`${origin}${path}`, { - method: 'POST', - headers: { - authorization: `Bearer ${requestToken}`, - 'content-type': 'application/json' - }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(30_000) - }) - } catch (error) { - if (!RETRYABLE_ADMIN_PATHS.has(path) || attempt === ADMIN_RETRY_ATTEMPTS) throw error - deps.emit({ event: 'candidate_admin_retry', path, attempt, reason: 'transport' }) - await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1)) - continue - } - if ( - RETRYABLE_ADMIN_PATHS.has(path) && - ([502, 503, 504].includes(response.status) || - (path === '/v1/admin/evacuation-status' && response.status === 500)) && - attempt < ADMIN_RETRY_ATTEMPTS - ) { - // These endpoints are read-only or transactionally idempotent, so a - // lost response may be retried without widening deployment authority. - deps.emit({ - event: 'candidate_admin_retry', - path, - attempt, - reason: `http_${response.status}` - }) - await response.arrayBuffer().catch(() => undefined) - await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1)) - continue - } - // The director names no cell, so a two-cell request names both. - const cells = - body?.cellId ?? [body?.sourceCellId, body?.targetCellId].filter(Boolean).join(' or ') - return await responseJson(response, path, cells || null) - } - throw new Error(`${path} retry attempts exhausted`) - } -} - -async function checkHttp(deps, origin, path) { - const response = await deps.fetch(`${origin}${path}`, { signal: AbortSignal.timeout(15_000) }) - const body = await response.json().catch(() => ({})) - if (!response.ok || body.ok !== true) throw new Error(`${origin}${path} is unavailable`) -} - -export async function inspectCell(config, deps, adminPost, expected) { - const common = ['--project', config.project, '--zone', expected.zone, '--format=json'] - const mig = deps.commandJson([ - 'compute', - 'instance-groups', - 'managed', - 'describe', - expected.migName, - ...common - ]) - const instances = deps.commandJson([ - 'compute', - 'instance-groups', - 'managed', - 'list-instances', - expected.migName, - ...common - ]) - const instanceName = validateMig(mig, instances, expected) - const instance = deps.commandJson([ - 'compute', - 'instances', - 'describe', - instanceName, - ...common - ]) - validateInstance(instance, expected, config.runtimeServiceAccount) - const backend = deps.commandJson([ - 'compute', - 'backend-services', - 'describe', - expected.backendName, - '--global', - '--project', - config.project, - '--format=json' - ]) - validateBackend(backend, expected) - await checkHttp(deps, expected.origin, '/health') - await checkHttp(deps, expected.origin, '/ready') - const runtime = await adminPost(expected.origin, '/v1/admin/runtime-status', { v: 1 }) - if ( - runtime.role !== 'cell' || - runtime.cellId !== expected.cellId || - runtime.cellUrl !== expected.origin || - (runtime.region ?? 'us-central1') !== expected.region || - runtime.imageDigest !== expected.imageDigest - ) { - throw new Error(`${expected.cellId} served runtime does not match Terraform topology`) - } - const status = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { - v: 1, - cellId: expected.cellId - }) - if ( - status.status?.cellUrl !== expected.origin || - (status.status?.region ?? 'us-central1') !== expected.region || - status.status?.runtime?.cellUrl !== expected.origin || - status.status?.runtime?.ready !== true || - status.status?.runtime?.heartbeatFresh !== true - ) { - throw new Error(`${expected.cellId} has no fresh ready authenticated heartbeat`) - } - assertDeploymentConnectionCapacity( - expected, - runtime.connectionCapacity ?? null, - status.status.connectionCapacity ?? null - ) - return { - ...status.status, - draining: runtime.draining === true, - process: runtime.runtime ?? null, - runtimeConnectionCapacity: runtime.connectionCapacity ?? null - } -} - -async function waitForMigration(config, deps, adminPost, completeReady) { - const deadline = deps.now() + config.timeoutMs - while (deps.now() < deadline) { - const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', { - v: 1, - sourceCellId: config.sourceCellId, - targetCellId: config.targetCellId, - completeReady - }) - deps.emit({ event: completeReady ? 'migration_completion' : 'migration_registration', ...status }) - if (completeReady ? status.inProgress === 0 : status.inProgress === status.targetRegistered) { - return status - } - if ( - completeReady && - status.targetRegistered === status.inProgress && - status.registeredSourceActive === 0 && - status.registeredCompletable === 0 && - status.registeredTargetInactive === status.inProgress - ) { - // CI waiting cannot revive an offline desktop; keep its proven migration - // pending until that target control reconnects. - return status - } - await deps.wait(config.pollIntervalMs) - } - throw new Error('timed out waiting for candidate migration') -} - -export async function setCellState(config, adminPost, cellId, enabled) { - const post = async (path, body) => await adminPost(config.directorOrigin, path, body) - const inspected = await inspectAdmissionSelector(post) - if (inspected.selector.generation > 0) { - await transitionAdmissionSelector(post, { - [cellId]: enabled ? 'general' : 'existing-only' - }) - return - } - await post('/v1/admin/cell-state', { v: 1, cellId, enabled }) -} - -function assertNoDurableActivity(status, operation) { - const activity = [ - status.assignments, - status.activityLeases, - status.reservedRequests, - status.outgoingMigrations, - status.incomingMigrations - ] - if (activity.some((value) => Number(value) !== 0)) { - throw new Error(`${operation} requires zero durable activity`) - } -} - -async function recoverCandidateFailure( - config, - deps, - adminPost, - source, - target, - allowEmptyAdmissionRollback, - selectorActive -) { - const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', { - v: 1, - sourceCellId: source.cellId, - targetCellId: target.cellId, - completeReady: false - }).catch(() => null) - if (!selectorActive && allowEmptyAdmissionRollback && status?.inProgress === 0) { - await setCellState(config, adminPost, source.cellId, true).catch(() => undefined) - await setCellState(config, adminPost, target.cellId, false).catch(() => undefined) - return - } - if (!selectorActive && status && status.inProgress > 0 && status.targetRegistered === 0) { - await setCellState(config, adminPost, source.cellId, true).catch(() => undefined) - deps.emit({ - event: 'candidate_rollback_waiting_for_lease_expiry', - sourceCellId: source.cellId, - targetCellId: target.cellId, - inProgress: status.inProgress - }) - return - } - deps.emit({ - event: 'candidate_forward_recovery_required', - sourceCellId: source.cellId, - targetCellId: target.cellId, - targetRegistered: status?.targetRegistered ?? null - }) -} - -export async function drainSource( - config, - deps, - token, - source, - graceMs = config.drainGraceMs, - traceValue -) { - const response = await deps.fetch(`${source.origin}/v1/admin/drain`, { - method: 'POST', - headers: { - authorization: `Bearer ${token}`, - 'content-type': 'application/json', - ...(traceValue ? { 'x-orca-drain-trace': traceValue } : {}) - }, - body: JSON.stringify({ v: 1, graceMs }), - signal: AbortSignal.timeout(30_000) - }) - await responseJson(response, 'source drain') - return { - backendStatus: response.status, - backendInstance: response.headers.get('x-orca-backend-instance') ?? undefined - } -} - -async function verifyCandidateCompletion( - config, - adminPost, - source, - target, - event, - eventName = 'candidate_complete' -) { - const finalSource = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { - v: 1, - cellId: source.cellId - }) - const finalTarget = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { - v: 1, - cellId: target.cellId - }) - if ( - finalSource.status.activityLeases !== 0 || - finalSource.status.reservedRequests !== 0 || - finalSource.status.outgoingMigrations !== 0 || - finalSource.status.runtime?.observedRequests !== 0 || - finalTarget.status.incomingMigrations !== 0 || - finalTarget.status.reservedRequests !== finalTarget.status.activityRequestUnits - ) { - throw new Error('aggregate post-migration counts are not reconciled') - } - event({ - event: eventName, - sourceCellId: source.cellId, - targetCellId: target.cellId, - dormantSourceAssignments: finalSource.status.assignments, - targetAssignments: finalTarget.status.assignments, - targetActivityLeases: finalTarget.status.activityLeases, - targetReservedRequests: finalTarget.status.reservedRequests - }) -} - -export async function runCandidateDeployment(config, overrides = {}) { - const deps = { - commandJson: overrides.commandJson ?? defaultCommandJson, - identityToken: overrides.identityToken ?? defaultIdentityToken, - fetch: overrides.fetch ?? fetch, - emit: - overrides.emit ?? - ((event) => process.stdout.write(`${JSON.stringify(event)}\n`)), - now: overrides.now ?? Date.now, - wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))), - random: overrides.random ?? Math.random - } - const topology = JSON.parse(readFileSync(config.topologyFile, 'utf8')) - const { source, target } = selectDeployments( - topology, - config.sourceCellId, - config.targetCellId - ) - const token = deps.identityToken(config.adminAudience) - const adminPost = createAdminPost(config, deps, token) - const selectorPost = async (path, body) => - await adminPost(config.directorOrigin, path, body) - const selectorInspection = await inspectAdmissionSelector(selectorPost) - const selectorActive = selectorInspection.selector.generation > 0 - const sourceStatus = await inspectCell(config, deps, adminPost, source) - const targetStatus = await inspectCell(config, deps, adminPost, target) - const sourceAdmission = selectorActive - ? selectorCellState(selectorInspection.selector, source.cellId) - : sourceStatus.enabled - ? 'general' - : 'existing-only' - const targetAdmission = selectorActive - ? selectorCellState(selectorInspection.selector, target.cellId) - : targetStatus.enabled - ? 'general' - : 'existing-only' - if (config.mode === 'audit') { - const migration = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', { - v: 1, - sourceCellId: source.cellId, - targetCellId: target.cellId, - completeReady: false - }) - // Forward recovery needs durable aggregate evidence without exposing assignment identities. - deps.emit({ - event: 'candidate_audit', - source: aggregateCellStatus(sourceStatus), - target: aggregateCellStatus(targetStatus), - migration - }) - return - } - if (config.mode === 'recover-forward') { - if ( - selectorActive - ? sourceAdmission !== 'existing-only' || targetAdmission !== 'migration-only' - : sourceStatus.enabled || !targetStatus.enabled - ) { - throw new Error( - selectorActive - ? 'forward recovery requires existing-only source and migration-only target' - : 'forward recovery requires disabled source and enabled target' - ) - } - await drainSource(config, deps, token, source) - await waitForMigration(config, deps, adminPost, false) - const completion = await waitForMigration(config, deps, adminPost, true) - if (completion.inProgress > 0) { - deps.emit({ - event: 'candidate_forward_pending', - sourceCellId: source.cellId, - targetCellId: target.cellId, - inProgress: completion.inProgress, - registeredSourceActive: completion.registeredSourceActive, - registeredCompletable: completion.registeredCompletable, - registeredTargetInactive: completion.registeredTargetInactive - }) - throw new Error('forward recovery remains pending for inactive target controls') - } - await verifyCandidateCompletion( - config, - adminPost, - source, - target, - deps.emit, - 'candidate_forward_recovered' - ) - return - } - if (config.mode === 'disable-cell') { - if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false) - // Disabling new admission preserves origin-owned sessions and durable recovery work. - deps.emit({ - event: 'cell_admission_disabled', - targetCellId: target.cellId, - changed: targetStatus.enabled, - assignments: targetStatus.assignments, - activityLeases: targetStatus.activityLeases, - reservedRequests: targetStatus.reservedRequests, - outgoingMigrations: targetStatus.outgoingMigrations, - incomingMigrations: targetStatus.incomingMigrations - }) - return - } - // Repair is safe only before a candidate owns assignments or origin-scoped work. - if (config.mode === 'reset-empty-candidate') { - assertNoDurableActivity(targetStatus, 'candidate admission reset') - if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false) - deps.emit({ - event: 'candidate_admission_reset', - targetCellId: target.cellId, - changed: targetStatus.enabled - }) - return - } - if (config.mode === 'enable-empty-cell') { - assertNoDurableActivity(targetStatus, 'cell admission enable') - if (selectorActive ? targetAdmission === 'general' : targetStatus.enabled) { - deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: false }) - return - } - } - const continuingEvacuation = config.mode === 'continue-evacuation' - if (continuingEvacuation) { - if ( - selectorActive - ? targetAdmission !== 'migration-only' - : !targetStatus.enabled - ) { - throw new Error( - selectorActive - ? 'continued evacuation requires migration-only target' - : 'continued evacuation requires enabled target' - ) - } - } else if (!selectorActive && targetStatus.enabled) { - throw new Error('candidate cell is already enabled') - } else if ( - selectorActive && - !['migration-only', 'existing-only'].includes(targetAdmission) - ) { - throw new Error('candidate cell must not be generally admitted') - } - const capacity = await adminPost(config.directorOrigin, '/v1/admin/evacuation-capacity', { - v: 1, - sourceCellId: source.cellId, - targetCellId: target.cellId - }) - if (capacity.requiredTargetUnits > capacity.availableTargetUnits) { - throw new Error('candidate lacks survivor request-unit headroom') - } - deps.emit({ - event: 'candidate_preflight', - mode: config.mode, - sourceCellId: source.cellId, - targetCellId: target.cellId, - sourceOrigin: source.origin, - targetOrigin: target.origin, - sourceMig: source.migName, - targetMig: target.migName, - sourceBackend: source.backendName, - targetBackend: target.backendName, - sourceDigest: source.imageDigest, - targetDigest: target.imageDigest, - sourceAssignments: capacity.sourceAssignments, - requiredTargetUnits: capacity.requiredTargetUnits, - availableTargetUnits: capacity.availableTargetUnits - }) - if (config.mode === 'preflight') return - if (config.mode === 'enable-empty-cell') { - await setCellState(config, adminPost, target.cellId, true) - deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: true }) - return - } - // Fresh execution starts from source-only admission; continuation preserves its target. - if ( - !continuingEvacuation && - (selectorActive ? sourceAdmission !== 'existing-only' : !sourceStatus.enabled) - ) { - throw new Error( - selectorActive ? 'source cell is not existing-only' : 'source cell is not enabled' - ) - } - if (selectorActive && targetAdmission !== 'migration-only') { - throw new Error('target cell is not migration-only') - } - - let migrationsStarted = 0 - try { - if (!selectorActive) { - if (sourceStatus.enabled) await setCellState(config, adminPost, source.cellId, false) - if (!targetStatus.enabled) await setCellState(config, adminPost, target.cellId, true) - } - for (;;) { - const result = await adminPost(config.directorOrigin, '/v1/admin/evacuate-cell', { - v: 1, - sourceCellId: source.cellId, - targetCellId: target.cellId, - limit: config.batchSize - }) - migrationsStarted += result.started - deps.emit({ event: 'migration_batch', started: result.started, totalStarted: migrationsStarted }) - if (result.started === 0) break - } - } catch (error) { - await recoverCandidateFailure( - config, - deps, - adminPost, - source, - target, - !continuingEvacuation, - selectorActive - ) - throw error - } - - try { - if (selectorActive) { - const currentSelector = await inspectAdmissionSelector(selectorPost) - if ( - currentSelector.selector.generation !== selectorInspection.selector.generation || - JSON.stringify(currentSelector.selector.membership) !== - JSON.stringify(selectorInspection.selector.membership) - ) { - throw new Error('admission selector changed before drain') - } - } - await drainSource(config, deps, token, source) - await waitForMigration(config, deps, adminPost, false) - const completion = await waitForMigration(config, deps, adminPost, true) - if (completion.inProgress > 0) { - throw new Error('candidate migration remains pending for inactive target controls') - } - } catch (error) { - // A completion response can be lost after its transaction commits. Never - // reverse admission here merely because no in-progress row remains. - await recoverCandidateFailure( - config, - deps, - adminPost, - source, - target, - false, - selectorActive - ) - throw error - } - - await verifyCandidateCompletion(config, adminPost, source, target, deps.emit) -} - -export function aggregateCellStatus(status) { - return { - cellId: status.cellId, - enabled: status.enabled, - assignments: status.assignments, - activityLeases: status.activityLeases, - activityRequestUnits: status.activityRequestUnits, - reservedRequests: status.reservedRequests, - outgoingMigrations: status.outgoingMigrations, - incomingMigrations: status.incomingMigrations, - runtimeReady: status.runtime?.ready ?? false, - heartbeatFresh: status.runtime?.heartbeatFresh ?? false, - observedRequests: status.runtime?.observedRequests ?? null - } -} - -export async function main(argv = process.argv.slice(2)) { - await runCandidateDeployment(parseArguments(argv)) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs b/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs deleted file mode 100644 index 3dc33d65953..00000000000 --- a/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs +++ /dev/null @@ -1,922 +0,0 @@ -import assert from 'node:assert/strict' -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { test } from 'node:test' -import { - assertDeploymentConnectionCapacity, - createAdminPost, - deployment, - parseArguments, - runCandidateDeployment, - selectDeployments, - suppliedAdminIdentityToken, - validateBackend, - validateInstance, - validateMig -} from './deploy-relay-gce-candidate.mjs' - -const runtimeServiceAccount = 'orca-relay@example.iam.gserviceaccount.com' -const digestA = `sha256:${'a'.repeat(64)}` -const digestB = `sha256:${'b'.repeat(64)}` - -test('retries evacuation-status HTTP 500 without widening other admin retries', async () => { - const events = [] - let calls = 0 - const adminPost = createAdminPost({}, { - fetch: async () => { - calls++ - return calls === 1 - ? new Response(JSON.stringify({ error: 'transient' }), { status: 500 }) - : new Response(JSON.stringify({ ok: true }), { status: 200 }) - }, - emit: (event) => events.push(event), - wait: async () => {}, - random: () => 0 - }, 'token') - assert.deepEqual( - await adminPost('https://relay.example', '/v1/admin/evacuation-status', {}), - { ok: true } - ) - assert.equal(calls, 2) - assert.deepEqual(events, [{ - event: 'candidate_admin_retry', - path: '/v1/admin/evacuation-status', - attempt: 1, - reason: 'http_500' - }]) - - calls = 0 - await assert.rejects( - createAdminPost({}, { - fetch: async () => { - calls++ - return new Response(JSON.stringify({ error: 'persistent' }), { status: 500 }) - }, - emit: () => {}, - wait: async () => {}, - random: () => 0 - }, 'token')('https://relay.example', '/v1/admin/runtime-status', {}), - /runtime-status failed: persistent/ - ) - assert.equal(calls, 1) -}) - -test('a reserve-cell refusal names the flip back to db; other 404s keep their error', async () => { - const post = (body) => createAdminPost({}, { - fetch: async () => Response.json(body, { status: 404 }), - emit: () => {}, - wait: async () => {}, - random: () => 0 - }, 'token') - await assert.rejects( - post({ error: 'cell_in_reserve_mode' })( - 'https://relay.example', - '/v1/admin/cell-status', - { v: 1, cellId: 'production-gce-c7' } - ), - /production-gce-c7 is in reserve mode .*flip it to admitMode=db/ - ) - await assert.rejects( - post({ error: 'cell_not_found' })( - 'https://relay.example', - '/v1/admin/cell-status', - { v: 1, cellId: 'production-gce-c7' } - ), - /cell-status failed: cell_not_found/ - ) - await assert.rejects( - post({ error: 'cell_in_reserve_mode' })( - 'https://relay.example', - '/v1/admin/evacuation-status', - { v: 1, sourceCellId: 'production-gce-c7', targetCellId: 'production-gce-c8' } - ), - /production-gce-c7 or production-gce-c8 is in reserve mode/ - ) -}) - -test('verifies a 1,000-cap cell against both runtime and director telemetry', () => { - const expected = deployment( - { - ...topology().target, - connection_hard_cap: 1_000, - connection_unobserved_bound: 60 - }, - 'target' - ) - const capacity = { - hardCap: 1_000, - controlRebindReserve: 100, - ordinaryConnectionLimit: 900, - unobservedBound: 60, - normalAdmissionPause: 840 - } - assert.doesNotThrow(() => - assertDeploymentConnectionCapacity(expected, capacity, { - ...capacity, - heartbeatFresh: true - }) - ) - assert.throws( - () => - assertDeploymentConnectionCapacity(expected, capacity, { - ...capacity, - hardCap: 600, - heartbeatFresh: true - }), - /differs from Terraform/ - ) -}) - -test('verifies a 3,000-cap regional cell with a 2,840 placement boundary', () => { - const expected = deployment( - { - ...topology().target, - connection_hard_cap: 3_000, - connection_unobserved_bound: 60 - }, - 'target' - ) - const capacity = { - hardCap: 3_000, - controlRebindReserve: 100, - ordinaryConnectionLimit: 2_900, - unobservedBound: 60, - normalAdmissionPause: 2_840 - } - - assert.doesNotThrow(() => - assertDeploymentConnectionCapacity(expected, capacity, { - ...capacity, - heartbeatFresh: true - }) - ) -}) - -function topology() { - return { - source: { - origin: 'https://c1.relay.example.com', - zone: 'us-central1-b', - mig_name: 'relay-c1', - instance_group: 'https://compute.example/instanceGroups/relay-c1', - backend_name: 'relay-c1', - backend_id: 'https://compute.example/backendServices/relay-c1', - image: `us-central1-docker.pkg.dev/project/repo/relay@${digestA}`, - capacity_requests: 4_000, - initially_enabled: true - }, - target: { - origin: 'https://c2.relay.example.com', - zone: 'us-central1-c', - mig_name: 'relay-c2', - instance_group: 'https://compute.example/instanceGroups/relay-c2', - backend_name: 'relay-c2', - backend_id: 'https://compute.example/backendServices/relay-c2', - image: `us-central1-docker.pkg.dev/project/repo/relay@${digestB}`, - capacity_requests: 4_000, - initially_enabled: false - } - } -} - -function withTopology(operation) { - const directory = mkdtempSync(join(tmpdir(), 'relay-gce-candidate-')) - const file = join(directory, 'topology.json') - writeFileSync(file, JSON.stringify(topology())) - return Promise.resolve(operation(file)).finally(() => rmSync(directory, { recursive: true })) -} - -function config(topologyFile, mode = 'preflight') { - return { - project: 'test-project', - directorOrigin: 'https://relay.example.com', - adminAudience: 'https://relay.example.com/v1/admin/drain', - topologyFile, - sourceCellId: 'source', - targetCellId: 'target', - runtimeServiceAccount, - mode, - batchSize: 100, - drainGraceMs: 120_000, - pollIntervalMs: 1, - timeoutMs: 1_000 - } -} - -function response(body, status = 200) { - return new Response(JSON.stringify(body), { - status, - headers: { 'content-type': 'application/json' } - }) -} - -function migrationResponse(body) { - return response({ - registeredSourceActive: 0, - registeredCompletable: 0, - registeredTargetInactive: 0, - expiredUnregistered: 0, - repairableExpiredUnregistered: 0, - abortableExpiredUnregistered: 0, - blockedExpiredUnregistered: 0, - blockedExpiredOnNewerTargetAssignment: 0, - ...body - }) -} - -function fakeCommand(args) { - const name = args[args.indexOf('describe') + 1] ?? args[args.indexOf('list-instances') + 1] - if (args.includes('list-instances')) { - return [ - { - instance: `https://compute.example/instances/${name}-vm`, - instanceStatus: 'RUNNING', - currentAction: 'NONE' - } - ] - } - if (args.includes('instance-groups')) { - return { - targetSize: 1, - updatePolicy: { - replacementMethod: 'RECREATE', - maxSurge: { fixed: 0 }, - maxUnavailable: { fixed: 1 } - } - } - } - if (args.includes('instances')) { - return { - networkInterfaces: [{ networkIP: '10.42.0.2' }], - serviceAccounts: [{ email: runtimeServiceAccount }] - } - } - const cell = name === 'relay-c1' ? topology().source : topology().target - return { protocol: 'HTTP', timeoutSec: 86_400, backends: [{ group: cell.instance_group }] } -} - -function harness({ - failTargetEnable = false, - failDrain = false, - failAfterRegistration = false, - failCompletionResponse = false, - sourceEnabled = true, - targetEnabled = false, - targetAssignments = 0, - migrationInProgress = 0, - migrationTargetRegistered = 0, - migrationTargetInactive = 0, - transientCompletionFailures = 0, - dormantSourceAssignments = 0, - selectorGeneration = 0 -} = {}) { - const state = { - source: { - enabled: selectorGeneration > 0 ? false : sourceEnabled, - assignments: 2, - activityLeases: 2 - }, - target: { - enabled: selectorGeneration > 0 ? true : targetEnabled, - assignments: targetAssignments, - activityLeases: targetAssignments - } - } - const events = [] - const stateChanges = [] - let batch = 0 - let migrationCompleted = false - let remainingTransientCompletionFailures = transientCompletionFailures - const fetch = async (url, options = {}) => { - const parsed = new URL(url) - if (parsed.pathname === '/health' || parsed.pathname === '/ready') return response({ ok: true }) - const body = JSON.parse(options.body ?? '{}') - if (parsed.pathname === '/v1/admin/runtime-status') { - const target = parsed.origin.includes('c2.') - return response({ - v: 1, - role: 'cell', - cellId: target ? 'target' : 'source', - cellUrl: parsed.origin, - imageDigest: target ? digestB : digestA - }) - } - if (parsed.pathname === '/v1/admin/admission-selector/status') { - return response({ - v: 1, - selector: { - generation: selectorGeneration, - attemptId: null, - membership: { - existingOnly: - selectorGeneration > 0 - ? ['source'] - : Object.keys(state).filter((cellId) => !state[cellId].enabled), - migrationOnly: selectorGeneration > 0 ? ['target'] : [], - general: - selectorGeneration > 0 - ? [] - : Object.keys(state).filter((cellId) => state[cellId].enabled) - } - }, - intent: null - }) - } - if (parsed.pathname === '/v1/admin/cell-status') { - const cell = state[body.cellId] - return response({ - v: 1, - status: { - cellId: body.cellId, - cellUrl: topology()[body.cellId].origin, - enabled: cell.enabled, - admissionState: - selectorGeneration > 0 - ? body.cellId === 'source' - ? 'existing-only' - : 'migration-only' - : cell.enabled - ? 'general' - : 'existing-only', - assignments: cell.assignments, - activityLeases: cell.activityLeases, - activityRequestUnits: cell.activityLeases, - reservedRequests: cell.activityLeases, - outgoingMigrations: 0, - incomingMigrations: 0, - runtime: { - cellUrl: topology()[body.cellId].origin, - ready: true, - heartbeatFresh: true, - observedRequests: cell.activityLeases - } - } - }) - } - if (parsed.pathname === '/v1/admin/evacuation-capacity') { - return response({ sourceAssignments: 2, requiredTargetUnits: 4, availableTargetUnits: 4_000 }) - } - if (parsed.pathname === '/v1/admin/cell-state') { - stateChanges.push([body.cellId, body.enabled]) - if (failTargetEnable && body.cellId === 'target' && body.enabled) { - return response({ error: 'injected_enable_failure' }, 409) - } - state[body.cellId].enabled = body.enabled - return response({ ok: true }) - } - if (parsed.pathname === '/v1/admin/evacuate-cell') { - if (failAfterRegistration && batch > 0) { - return response({ error: 'injected_batch_failure' }, 503) - } - const started = batch++ === 0 ? 2 : 0 - return response({ v: 1, started }) - } - if (parsed.pathname === '/v1/admin/evacuation-status') { - if (failAfterRegistration) { - return migrationResponse({ - v: 1, - inProgress: 2, - targetRegistered: 1, - registeredSourceActive: 1, - completed: 0, - blocked: 1 - }) - } - if (failDrain) { - return migrationResponse({ - v: 1, - inProgress: 2, - targetRegistered: 0, - completed: 0, - blocked: 0 - }) - } - if (body.completeReady) { - state.source.assignments = dormantSourceAssignments - state.source.activityLeases = 0 - state.target.assignments = 2 - state.target.activityLeases = 2 - if (remainingTransientCompletionFailures > 0) { - remainingTransientCompletionFailures-- - throw new TypeError('injected transient fetch failure') - } - if (migrationTargetInactive > 0) { - return migrationResponse({ - v: 1, - inProgress: migrationTargetInactive, - targetRegistered: migrationTargetInactive, - registeredTargetInactive: migrationTargetInactive, - completed: 0, - blocked: migrationTargetInactive - }) - } - migrationCompleted = true - if (failCompletionResponse) { - return response({ error: 'injected_completion_response_failure' }, 503) - } - return migrationResponse({ - v: 1, - inProgress: 0, - targetRegistered: 0, - completed: 2, - blocked: 0 - }) - } - if (migrationCompleted) { - return migrationResponse({ - v: 1, - inProgress: 0, - targetRegistered: 0, - completed: 0, - blocked: 0 - }) - } - if (batch === 0) { - return migrationResponse({ - v: 1, - inProgress: migrationInProgress, - targetRegistered: migrationTargetRegistered, - completed: 0, - blocked: 0 - }) - } - return migrationResponse({ - v: 1, - inProgress: 2, - targetRegistered: 2, - registeredCompletable: 2, - completed: 0, - blocked: 0 - }) - } - if (parsed.pathname === '/v1/admin/drain') { - return failDrain ? response({ error: 'injected_drain_failure' }, 503) : response({ ok: true }) - } - return response({ error: 'unexpected_request' }, 500) - } - return { - overrides: { - commandJson: fakeCommand, - identityToken: () => 'secret-token-never-emitted', - fetch, - emit: (event) => events.push(event), - wait: async () => undefined, - random: () => 0 - }, - events, - stateChanges, - state - } -} - -test('requires explicit dry-run/execute inputs and a distinct disabled candidate', () => { - assert.throws(() => parseArguments([]), /missing --project/) - assert.throws( - () => - parseArguments([ - '--project', - 'project', - '--director-origin', - 'https://relay.example.com', - '--admin-audience', - 'https://relay.example.com/not-drain', - '--topology-file', - 'topology.json', - '--source-cell-id', - 'source', - '--target-cell-id', - 'target', - '--runtime-service-account', - runtimeServiceAccount, - '--mode', - 'preflight' - ]), - /director drain URL/ - ) - assert.throws(() => selectDeployments(topology(), 'source', 'source'), /must differ/) - const overlapping = topology() - overlapping.target.backend_id = overlapping.source.backend_id - assert.throws(() => selectDeployments(overlapping, 'source', 'target'), /backendId overlap/) - const enabled = topology() - enabled.target.initially_enabled = true - assert.throws(() => selectDeployments(enabled, 'source', 'target'), /initially disabled/) -}) - -test('accepts only a bounded JWT-shaped supplied admin identity token', () => { - assert.equal(suppliedAdminIdentityToken({}), null) - assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc') - assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' })) - assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' })) - assert.throws(() => - suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` }) - ) -}) - -test('rejects unsafe fixed-one topology, public IPs, and backend overlap', () => { - const expected = selectDeployments(topology(), 'source', 'target').target - assert.throws(() => validateMig({ targetSize: 2 }, [], expected), /fixed-one/) - assert.throws( - () => - validateInstance( - { - networkInterfaces: [{ accessConfigs: [{ natIP: '203.0.113.1' }] }], - serviceAccounts: [{ email: runtimeServiceAccount }] - }, - expected, - runtimeServiceAccount - ), - /public IP/ - ) - assert.throws( - () => validateBackend({ protocol: 'HTTP', timeoutSec: 86_400, backends: [] }, expected), - /topology mismatch/ - ) -}) - -test('preflights exact served digests and survivor headroom without mutating admission', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness() - await runCandidateDeployment(config(file), overrides) - assert.deepEqual(stateChanges, []) - assert.equal(events[0].event, 'candidate_preflight') - assert.equal(events[0].targetDigest, digestB) - assert.equal(JSON.stringify(events).includes('secret-token'), false) - }) -}) - -test('audits a partially committed migration without changing admission or completing rows', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ - sourceEnabled: false, - targetEnabled: true, - targetAssignments: 1, - migrationInProgress: 2, - migrationTargetRegistered: 2 - }) - await runCandidateDeployment(config(file, 'audit'), overrides) - assert.deepEqual(stateChanges, []) - assert.deepEqual(events, [ - { - event: 'candidate_audit', - source: { - cellId: 'source', - enabled: false, - assignments: 2, - activityLeases: 2, - activityRequestUnits: 2, - reservedRequests: 2, - outgoingMigrations: 0, - incomingMigrations: 0, - runtimeReady: true, - heartbeatFresh: true, - observedRequests: 2 - }, - target: { - cellId: 'target', - enabled: true, - assignments: 1, - activityLeases: 1, - activityRequestUnits: 1, - reservedRequests: 1, - outgoingMigrations: 0, - incomingMigrations: 0, - runtimeReady: true, - heartbeatFresh: true, - observedRequests: 1 - }, - migration: { - v: 1, - inProgress: 2, - targetRegistered: 2, - registeredSourceActive: 0, - registeredCompletable: 0, - registeredTargetInactive: 0, - completed: 0, - blocked: 0, - expiredUnregistered: 0, - repairableExpiredUnregistered: 0, - abortableExpiredUnregistered: 0, - blockedExpiredUnregistered: 0, - blockedExpiredOnNewerTargetAssignment: 0 - } - } - ]) - }) -}) - -test('preflights with source admission disabled but still refuses execution', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ sourceEnabled: false }) - await runCandidateDeployment(config(file), overrides) - assert.deepEqual(stateChanges, []) - assert.equal(events.at(-1).event, 'candidate_preflight') - await assert.rejects( - runCandidateDeployment(config(file, 'execute'), overrides), - /source cell is not enabled/ - ) - assert.deepEqual(stateChanges, []) - }) -}) - -test('explicitly resets only an empty declared candidate to disabled admission', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ - sourceEnabled: false, - targetEnabled: true - }) - await runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides) - assert.deepEqual(stateChanges, [['target', false]]) - assert.deepEqual(events.at(-1), { - event: 'candidate_admission_reset', - targetCellId: 'target', - changed: true - }) - }) -}) - -test('refuses to reset candidate admission while it owns durable activity', async () => { - await withTopology(async (file) => { - const { overrides, stateChanges } = harness({ targetEnabled: true, targetAssignments: 1 }) - await assert.rejects( - runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides), - /requires zero durable activity/ - ) - assert.deepEqual(stateChanges, []) - }) -}) - -test('disables only new admission while preserving durable candidate activity', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ - targetEnabled: true, - targetAssignments: 1 - }) - await runCandidateDeployment(config(file, 'disable-cell'), overrides) - assert.deepEqual(stateChanges, [['target', false]]) - assert.deepEqual(events.at(-1), { - event: 'cell_admission_disabled', - targetCellId: 'target', - changed: true, - assignments: 1, - activityLeases: 1, - reservedRequests: 1, - outgoingMigrations: 0, - incomingMigrations: 0 - }) - }) -}) - -test('explicitly enables only an empty preflighted cell for admission', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ sourceEnabled: false }) - await runCandidateDeployment(config(file, 'enable-empty-cell'), overrides) - assert.deepEqual(stateChanges, [['target', true]]) - assert.equal(events.at(-2).event, 'candidate_preflight') - assert.deepEqual(events.at(-1), { - event: 'cell_admission_enabled', - targetCellId: 'target', - changed: true - }) - }) -}) - -test('refuses to enable cell admission while it owns durable activity', async () => { - await withTopology(async (file) => { - const { overrides, stateChanges } = harness({ targetAssignments: 1 }) - await assert.rejects( - runCandidateDeployment(config(file, 'enable-empty-cell'), overrides), - /requires zero durable activity/ - ) - assert.deepEqual(stateChanges, []) - }) -}) - -test('executes target-first evacuation and verifies aggregate drained counts', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness() - await runCandidateDeployment(config(file, 'execute'), overrides) - assert.deepEqual(stateChanges.slice(0, 2), [ - ['source', false], - ['target', true] - ]) - assert.equal(events.at(-1).event, 'candidate_complete') - assert.equal(events.at(-1).targetAssignments, 2) - }) -}) - -test('executes within selector membership without legacy admission writes', async () => { - await withTopology(async (file) => { - const testHarness = harness({ selectorGeneration: 1 }) - await runCandidateDeployment(config(file, 'execute'), testHarness.overrides) - assert.deepEqual(testHarness.stateChanges, []) - assert.equal(testHarness.state.source.enabled, false) - assert.equal(testHarness.state.target.enabled, true) - }) -}) - -test('never restores legacy general admission after selector-era failure', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - selectorGeneration: 1, - failAfterRegistration: true - }) - await assert.rejects( - runCandidateDeployment(config(file, 'execute'), testHarness.overrides), - /injected_batch_failure/ - ) - assert.deepEqual(testHarness.stateChanges, []) - assert.equal(testHarness.state.source.enabled, false) - }) -}) - -test('continues a partial evacuation without resetting target admission', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ - sourceEnabled: true, - targetEnabled: true, - targetAssignments: 1, - migrationInProgress: 1, - migrationTargetRegistered: 1 - }) - await runCandidateDeployment(config(file, 'continue-evacuation'), overrides) - assert.deepEqual(stateChanges, [['source', false]]) - assert.equal(events.at(-1).event, 'candidate_complete') - }) -}) - -test('refuses continued evacuation unless target admission is already enabled', async () => { - await withTopology(async (file) => { - const { overrides, stateChanges } = harness() - await assert.rejects( - runCandidateDeployment(config(file, 'continue-evacuation'), overrides), - /requires enabled target/ - ) - assert.deepEqual(stateChanges, []) - }) -}) - -test('preserves partial target admission when continued batching fails', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ - failAfterRegistration: true, - sourceEnabled: true, - targetEnabled: true, - targetAssignments: 1, - migrationInProgress: 1, - migrationTargetRegistered: 1 - }) - await assert.rejects( - runCandidateDeployment(config(file, 'continue-evacuation'), overrides), - /injected_batch_failure/ - ) - assert.deepEqual(stateChanges, [['source', false]]) - assert.equal(events.at(-1).event, 'candidate_forward_recovery_required') - }) -}) - -test('resumes only a committed forward migration and preserves dormant source assignments', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ - sourceEnabled: false, - targetEnabled: true, - migrationInProgress: 2, - migrationTargetRegistered: 2, - dormantSourceAssignments: 7 - }) - await runCandidateDeployment(config(file, 'recover-forward'), overrides) - assert.deepEqual(stateChanges, []) - assert.deepEqual(events.at(-1), { - event: 'candidate_forward_recovered', - sourceCellId: 'source', - targetCellId: 'target', - dormantSourceAssignments: 7, - targetAssignments: 2, - targetActivityLeases: 2, - targetReservedRequests: 2 - }) - }) -}) - -test('retries a transient idempotent completion request without reversing admission', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ - sourceEnabled: false, - targetEnabled: true, - migrationInProgress: 2, - migrationTargetRegistered: 2, - transientCompletionFailures: 1 - }) - await runCandidateDeployment(config(file, 'recover-forward'), overrides) - assert.deepEqual(stateChanges, []) - assert.deepEqual( - events.filter(({ event }) => event === 'candidate_admin_retry'), - [ - { - event: 'candidate_admin_retry', - path: '/v1/admin/evacuation-status', - attempt: 1, - reason: 'transport' - } - ] - ) - assert.equal(events.at(-1).event, 'candidate_forward_recovered') - }) -}) - -test('stops forward recovery promptly when only registered offline targets remain', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ - sourceEnabled: false, - targetEnabled: true, - migrationInProgress: 2, - migrationTargetRegistered: 2, - migrationTargetInactive: 2 - }) - await assert.rejects( - runCandidateDeployment(config(file, 'recover-forward'), overrides), - /pending for inactive target controls/ - ) - assert.deepEqual(stateChanges, []) - assert.deepEqual(events.at(-1), { - event: 'candidate_forward_pending', - sourceCellId: 'source', - targetCellId: 'target', - inProgress: 2, - registeredSourceActive: 0, - registeredCompletable: 0, - registeredTargetInactive: 2 - }) - }) -}) - -test('refuses forward recovery unless source and target admission match committed direction', async () => { - await withTopology(async (file) => { - const { overrides, stateChanges } = harness() - await assert.rejects( - runCandidateDeployment(config(file, 'recover-forward'), overrides), - /requires disabled source and enabled target/ - ) - assert.deepEqual(stateChanges, []) - }) -}) - -test('re-enables an intact source when candidate admission fails before migration', async () => { - await withTopology(async (file) => { - const { overrides, stateChanges } = harness({ failTargetEnable: true }) - await assert.rejects( - runCandidateDeployment(config(file, 'execute'), overrides), - /injected_enable_failure/ - ) - assert.deepEqual(stateChanges, [ - ['source', false], - ['target', true], - ['source', true], - ['target', false] - ]) - }) -}) - -test('re-enables the source and waits for lease rollback when no target registered', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ failDrain: true }) - await assert.rejects( - runCandidateDeployment(config(file, 'execute'), overrides), - /injected_drain_failure/ - ) - assert.deepEqual(stateChanges.slice(-1), [['source', true]]) - assert.equal(events.at(-1).event, 'candidate_rollback_waiting_for_lease_expiry') - }) -}) - -test('preserves both routes for forward recovery after a target registration', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ failAfterRegistration: true }) - await assert.rejects( - runCandidateDeployment(config(file, 'execute'), overrides), - /injected_batch_failure/ - ) - assert.deepEqual(stateChanges, [ - ['source', false], - ['target', true] - ]) - assert.equal(events.at(-1).event, 'candidate_forward_recovery_required') - assert.equal(events.at(-1).targetRegistered, 1) - }) -}) - -test('does not reverse admission after completion commits but its response is lost', async () => { - await withTopology(async (file) => { - const { overrides, events, stateChanges } = harness({ failCompletionResponse: true }) - await assert.rejects( - runCandidateDeployment(config(file, 'execute'), overrides), - /injected_completion_response_failure/ - ) - assert.deepEqual(stateChanges, [ - ['source', false], - ['target', true] - ]) - assert.equal(events.at(-1).event, 'candidate_forward_recovery_required') - assert.equal(events.at(-1).targetRegistered, 0) - }) -}) diff --git a/cloud/dev/scripts/deploy-relay-gce-multi-target.mjs b/cloud/dev/scripts/deploy-relay-gce-multi-target.mjs deleted file mode 100644 index e36570947ad..00000000000 --- a/cloud/dev/scripts/deploy-relay-gce-multi-target.mjs +++ /dev/null @@ -1,3135 +0,0 @@ -import { readFileSync } from 'node:fs' -import { spawnSync } from 'node:child_process' -import { randomUUID } from 'node:crypto' -import { resolve4 } from 'node:dns/promises' -import { pathToFileURL } from 'node:url' -import { - aggregateCellStatus, - assertDeploymentConnectionCapacity, - createAdminPost, - defaultCommandJson, - defaultIdentityToken, - deployment, - drainSource, - inspectCell, - setCellState, - suppliedFenceMutationIdentityToken, - validateBackend, - validateInstance, - validateMig -} from './deploy-relay-gce-candidate.mjs' -import { - abortSupersededTerraformFenceBeforeUpload, - abortTerraformFenceBeforeApply, - adoptLegacyTerraformFence, - assertTerraformFenceSet, - assertTerraformFenceZeroDiff, - deleteTerraformFencePlan, - downloadTerraformFencePlan, - inspectCompletedTerraformFenceProgress, - inspectTerraformFenceProgress, - assertTerraformFenceStateFenced, - readTerraformStateObjectBinding, - recoverSupersededCompletedTerraformFence, - resolveTerraformFencePlanGeneration, - resumeTerraformFence, - runTerraformFenceApply, - uploadTerraformFencePlan -} from './relay-gce-terraform-fence.mjs' -import { - addExactMigrationCells, - applyExactAdmissionSelector, - inspectAdmissionSelector, - membershipWithStates, - selectorCellState -} from './relay-admission-selector.mjs' - -const DEFAULT_BATCH_SIZE = 100 -const DEFAULT_CONNECTION_CEILING = 600 -const DEFAULT_MINIMUM_LEASE_MS = 10 * 60 * 1_000 -const DEFAULT_POLL_MS = 5_000 -const DEFAULT_TIMEOUT_MS = 14 * 60 * 1_000 -const CUTOVER_CONNECTION_HARD_CAP = 600 -const CUTOVER_CONTROL_REBIND_RESERVE = 100 -const MAX_PRE_AUTH_CONNECTIONS = 45 -const SELECTOR_ROLLBACK_TAG = 'selector-rollback' -const SELECTOR_REVISION_MARKER = '3' -const FENCE_BROKER_MUTATION_ROUTES = new Set([ - '/v1/admin/cell-fence-adopt-legacy', - '/v1/admin/cell-fence-commit-legacy-adoption', - '/v1/admin/cell-fence-attest', - '/v1/admin/cell-fence-attempt-prepare', - '/v1/admin/cell-fence-attempt-start', - '/v1/admin/cell-fence-attempt-plan', - '/v1/admin/cell-fence-attempt-operation', - '/v1/admin/cell-fence-attempt-abort', - '/v1/admin/migration-supersede-cell' -]) - -function positiveInteger(value, name, maximum = Number.MAX_SAFE_INTEGER) { - const parsed = Number(value) - if (!Number.isInteger(parsed) || parsed <= 0 || parsed > maximum) { - throw new Error(`${name} must be a positive integer`) - } - return parsed -} - -function nonnegativeInteger(value, name, maximum = Number.MAX_SAFE_INTEGER) { - const parsed = Number(value) - if (!Number.isSafeInteger(parsed) || parsed < 0 || parsed > maximum) { - throw new Error(`${name} must be a nonnegative integer`) - } - return parsed -} - -function canonicalOrigin(value, name) { - const url = new URL(value) - if (url.protocol !== 'https:' || url.origin !== value || url.pathname !== '/') { - throw new Error(`${name} must be a canonical HTTPS origin`) - } - return value -} - -function targetIds(value, minimum) { - const ids = [...new Set(value.split(',').map((item) => item.trim()).filter(Boolean))].sort() - if (ids.length < minimum || ids.some((id) => !/^[a-z][a-z0-9-]{0,127}$/.test(id))) { - throw new Error( - `--target-cell-ids must contain at least ${minimum} distinct cell ID${minimum === 1 ? '' : 's'}` - ) - } - return ids -} - -function optionalCellIds(value, name) { - const ids = [...new Set(String(value ?? '').split(',').map((item) => item.trim()).filter(Boolean))] - .sort() - if (ids.some((id) => !/^[a-z][a-z0-9-]{0,127}$/.test(id))) { - throw new Error(`${name} contains an invalid cell ID`) - } - return ids -} - -export function parseMultiTargetArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`) - values[key.slice(2)] = value - } - for (const key of [ - 'project', - 'director-origin', - 'admin-audience', - 'topology-file', - 'source-cell-id', - 'target-cell-ids', - 'runtime-service-account', - 'mode' - ]) { - if (!values[key]) throw new Error(`missing --${key}`) - } - if ( - ![ - 'audit', - 'preflight', - 'execute', - 'cutover-admission', - 'add-migration-cells', - 'promote-general-cell', - 'retire-migration-cell', - 'recover-forward', - 'fence-source', - 'abort-fence-source', - 'supersede-target' - ].includes(values.mode) - ) { - throw new Error( - '--mode must be audit, preflight, execute, cutover-admission, add-migration-cells, promote-general-cell, retire-migration-cell, recover-forward, fence-source, abort-fence-source, or supersede-target' - ) - } - const singleTargetMode = [ - 'add-migration-cells', - 'promote-general-cell', - 'retire-migration-cell' - ].includes(values.mode) - const parsedTargetIds = targetIds(values['target-cell-ids'], singleTargetMode ? 1 : 2) - const generalCellIds = optionalCellIds(values['general-cell-ids'], '--general-cell-ids') - const failedTargetCellId = values['failed-target-cell-id'] - const replacementTargetCellId = values['replacement-target-cell-id'] - if ( - ['promote-general-cell', 'retire-migration-cell'].includes(values.mode) && - parsedTargetIds.length !== 1 - ) { - throw new Error(`${values.mode} requires exactly one target cell ID`) - } - if (values.mode === 'supersede-target') { - if (!failedTargetCellId || !replacementTargetCellId) { - throw new Error('supersede-target requires failed and replacement target cell IDs') - } - if ( - failedTargetCellId === replacementTargetCellId || - !parsedTargetIds.includes(failedTargetCellId) || - !parsedTargetIds.includes(replacementTargetCellId) || - parsedTargetIds.length !== 2 - ) { - throw new Error('supersede-target target set must exactly match failed and replacement') - } - } - const selectorMode = [ - 'cutover-admission', - 'add-migration-cells', - 'promote-general-cell', - 'retire-migration-cell' - ].includes(values.mode) - if (selectorMode) { - for (const key of ['director-region', 'director-service', 'director-min-instances']) { - if (!values[key]) throw new Error(`${values.mode} requires --${key}`) - } - if (values.mode === 'cutover-admission' && generalCellIds.length === 0) { - throw new Error('cutover-admission requires --general-cell-ids') - } - if ( - values['selector-attempt-id'] && - !/^[A-Za-z0-9_-]{8,128}$/.test(values['selector-attempt-id']) - ) { - throw new Error('--selector-attempt-id is invalid') - } - if ( - ['add-migration-cells', 'promote-general-cell', 'retire-migration-cell'].includes( - values.mode - ) && - !values['selector-attempt-id'] - ) { - throw new Error(`${values.mode} requires --selector-attempt-id`) - } - } - const capacityBoundModes = [ - 'cutover-admission', - 'add-migration-cells', - 'recover-forward', - 'fence-source' - ] - if ( - capacityBoundModes.includes(values.mode) && - !values['unobserved-connection-bound'] - ) { - throw new Error(`${values.mode} requires --unobserved-connection-bound`) - } - const adminAudience = new URL(values['admin-audience']) - if ( - adminAudience.protocol !== 'https:' || - adminAudience.pathname !== '/v1/admin/drain' || - adminAudience.search || - adminAudience.hash - ) { - throw new Error('--admin-audience must be the director drain URL') - } - if (!['staging', 'production'].includes(values.environment ?? 'production')) { - throw new Error('--environment must be staging or production') - } - if ( - ['fence-source', 'abort-fence-source', 'supersede-target'].includes(values.mode) && - !/^[a-f0-9]{40}$/.test(values['fence-commit'] ?? '') - ) { - throw new Error('Terraform fence modes require the exact --fence-commit') - } - const completedFenceFields = { - attemptId: values['completed-fence-attempt-id'], - fenceCommit: values['completed-fence-commit'], - gceOperation: values['completed-fence-operation'], - terraformStateSerial: values['completed-fence-state-serial'], - planObjectGeneration: values['completed-fence-plan-generation'], - terraformStateObjectGeneration: values['completed-fence-state-generation'], - terraformStateObjectSha256: values['completed-fence-state-sha256'], - principalEmail: values['fence-broker-service-account'] - } - const completedFenceValues = Object.values(completedFenceFields) - const completedFenceRecovery = - completedFenceValues.every((value) => value === undefined) - ? undefined - : completedFenceFields - if ( - completedFenceRecovery && - (values.mode !== 'supersede-target' || - completedFenceValues.some((value) => value === undefined) || - !/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test( - completedFenceRecovery.attemptId - ) || - !/^[a-f0-9]{40}$/.test(completedFenceRecovery.fenceCommit) || - completedFenceRecovery.fenceCommit === values['fence-commit'] || - !/^[A-Za-z0-9._-]{1,256}$/.test(completedFenceRecovery.gceOperation) || - !/^[1-9][0-9]{0,30}$/.test(completedFenceRecovery.planObjectGeneration) || - !/^[1-9][0-9]{0,30}$/.test( - completedFenceRecovery.terraformStateObjectGeneration - ) || - !/^[a-f0-9]{64}$/.test( - completedFenceRecovery.terraformStateObjectSha256 - ) || - !/^[^@\s]+@[^@\s]+\.gserviceaccount\.com$/.test( - completedFenceRecovery.principalEmail - )) - ) { - throw new Error('completed Terraform fence recovery inputs are invalid') - } - const minimumLeaseRemainingMs = positiveInteger( - values['minimum-lease-remaining-ms'] ?? DEFAULT_MINIMUM_LEASE_MS, - '--minimum-lease-remaining-ms', - 60 * 60 * 1_000 - ) - if (minimumLeaseRemainingMs < DEFAULT_MINIMUM_LEASE_MS) { - throw new Error('--minimum-lease-remaining-ms cannot be below 600000') - } - return { - project: values.project, - directorOrigin: canonicalOrigin(values['director-origin'], '--director-origin'), - adminAudience: adminAudience.toString(), - topologyFile: values['topology-file'], - sourceCellId: values['source-cell-id'], - targetCellIds: parsedTargetIds, - generalCellIds, - directorRegion: values['director-region'], - directorService: values['director-service'], - directorMinimumInstances: selectorMode - ? positiveInteger(values['director-min-instances'], '--director-min-instances', 1_000) - : undefined, - selectorAttemptId: values['selector-attempt-id'], - unobservedConnectionBound: - capacityBoundModes.includes(values.mode) - ? nonnegativeInteger( - values['unobserved-connection-bound'], - '--unobserved-connection-bound', - CUTOVER_CONNECTION_HARD_CAP - CUTOVER_CONTROL_REBIND_RESERVE - 1 - ) - : undefined, - failedTargetCellId, - replacementTargetCellId, - completedFenceRecovery: completedFenceRecovery - ? { - ...completedFenceRecovery, - terraformStateSerial: nonnegativeInteger( - completedFenceRecovery.terraformStateSerial, - '--completed-fence-state-serial' - ) - } - : undefined, - runtimeServiceAccount: values['runtime-service-account'], - environment: values.environment ?? 'production', - fenceCommit: values['fence-commit'], - terraformDir: values['terraform-dir'] ?? 'infra/terraform', - terraformVarFile: - values['terraform-var-file'] ?? - `environments/${values.environment ?? 'production'}.tfvars`, - mode: values.mode, - batchSize: positiveInteger(values['batch-size'] ?? DEFAULT_BATCH_SIZE, '--batch-size', 100), - connectionCeiling: positiveInteger( - values['connection-ceiling'] ?? DEFAULT_CONNECTION_CEILING, - '--connection-ceiling', - 100_000 - ), - minimumLeaseRemainingMs, - drainGraceMs: positiveInteger( - values['drain-grace-ms'] ?? 120_000, - '--drain-grace-ms', - 60 * 60 * 1_000 - ), - pollIntervalMs: positiveInteger( - values['poll-interval-ms'] ?? DEFAULT_POLL_MS, - '--poll-interval-ms', - 60_000 - ), - timeoutMs: positiveInteger( - values['timeout-ms'] ?? DEFAULT_TIMEOUT_MS, - '--timeout-ms', - 60 * 60 * 1_000 - ) - } -} - -export function selectMultiTargetDeployments(topology, sourceCellId, targetCellIds) { - const legacyCapacityTopology = Object.values(topology).every( - (cell) => - cell && - typeof cell === 'object' && - !Object.hasOwn(cell, 'connection_hard_cap') && - !Object.hasOwn(cell, 'connection_unobserved_bound') - ) - const fromTopology = (cellId) => ({ - ...deployment(topology[cellId], cellId), - legacyCapacityTopology - }) - const source = fromTopology(sourceCellId) - const targets = targetCellIds.map(fromTopology) - const resources = new Map() - for (const cell of [source, ...targets]) { - for (const key of ['origin', 'migName', 'instanceGroup', 'backendName', 'backendId']) { - const resourceKey = `${key}:${cell[key]}` - const previous = resources.get(resourceKey) - if (previous) throw new Error(`${cell.cellId} ${key} overlaps ${previous}`) - resources.set(resourceKey, cell.cellId) - } - } - if (targets.some((target) => target.initiallyEnabled)) { - throw new Error('every target must be declared initially disabled') - } - return { source, targets } -} - -function revisionEnvironment(revision) { - return Object.fromEntries( - (revision.spec?.containers?.[0]?.env ?? []) - .filter((entry) => entry.name && 'value' in entry) - .map((entry) => [entry.name, entry.value]) - ) -} - -function revisionMinimum(revision) { - return Number(revision.metadata?.annotations?.['autoscaling.knative.dev/minScale'] ?? 0) -} - -function directorInventory(environment, revisionName) { - let cells - try { - cells = JSON.parse(environment.ORCA_RELAY_CELLS_JSON) - } catch { - throw new Error(`${revisionName} has an invalid director inventory`) - } - const ids = Array.isArray(cells) ? cells.map((cell) => cell?.id) : [] - if ( - ids.length === 0 || - ids.some((id) => typeof id !== 'string' || id.length === 0) || - new Set(ids).size !== ids.length - ) { - throw new Error(`${revisionName} has an invalid director inventory`) - } - return JSON.stringify(cells) -} - -export function verifySelectorCompatibleDirector(config, deps) { - const common = [ - '--project', - config.project, - '--region', - config.directorRegion, - '--format=json' - ] - const service = deps.commandJson([ - 'run', - 'services', - 'describe', - config.directorService, - ...common - ]) - const active = (service.status?.traffic ?? []).filter( - (entry) => Number(entry.percent ?? 0) > 0 - ) - const rollback = (service.status?.traffic ?? []).find( - (entry) => entry.tag === SELECTOR_ROLLBACK_TAG - ) - if ( - active.length !== 1 || - Number(active[0].percent) !== 100 || - !active[0].revisionName || - !rollback?.revisionName || - Number(rollback.percent ?? 0) !== 0 || - rollback.revisionName === active[0].revisionName - ) { - throw new Error('director lacks an isolated compatible rollback revision') - } - const revisions = deps.commandJson([ - 'run', - 'revisions', - 'list', - '--service', - config.directorService, - ...common - ]) - const allowed = new Set([active[0].revisionName, rollback.revisionName]) - const names = revisions.map((revision) => revision.metadata?.name).filter(Boolean) - if ( - revisions.length !== 2 || - names.length !== 2 || - names.some((name) => !allowed.has(name)) - ) { - throw new Error('old or pre-selector director revisions still exist') - } - let compatibleImage - let compatibleInventory - for (const revisionName of allowed) { - const revision = deps.commandJson([ - 'run', - 'revisions', - 'describe', - revisionName, - ...common - ]) - const environment = revisionEnvironment(revision) - if ( - environment.ORCA_RELAY_ROLE !== 'director' || - environment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== SELECTOR_REVISION_MARKER - ) { - throw new Error(`${revisionName} is not selector-compatible`) - } - const inventory = directorInventory(environment, revisionName) - if (compatibleInventory && inventory !== compatibleInventory) { - throw new Error('active and rollback director inventories do not match') - } - compatibleInventory = inventory - const image = revision.spec?.containers?.[0]?.image - if (!image || (compatibleImage && image !== compatibleImage)) { - throw new Error('active and rollback director images do not match') - } - compatibleImage = image - if (revisionName === rollback.revisionName && revisionMinimum(revision) !== 0) { - throw new Error('selector rollback revision is not scale-to-zero') - } - if ( - revisionName === active[0].revisionName && - !(revisionMinimum(revision) >= config.directorMinimumInstances) - ) { - throw new Error('active selector revision is below the required floor') - } - } - return { - activeRevision: active[0].revisionName, - rollbackRevision: rollback.revisionName - } -} - -function verifyActiveSelectorDirector(config, deps, cellId) { - const common = [ - '--project', - config.project, - '--region', - config.directorRegion, - '--format=json' - ] - const service = deps.commandJson([ - 'run', - 'services', - 'describe', - config.directorService, - ...common - ]) - const active = (service.status?.traffic ?? []).filter( - (entry) => Number(entry.percent ?? 0) > 0 - ) - if ( - active.length !== 1 || - Number(active[0].percent) !== 100 || - !active[0].revisionName - ) { - throw new Error('director lacks one active selector revision') - } - const revision = deps.commandJson([ - 'run', - 'revisions', - 'describe', - active[0].revisionName, - ...common - ]) - const environment = revisionEnvironment(revision) - const inventory = JSON.parse(directorInventory(environment, active[0].revisionName)) - if ( - environment.ORCA_RELAY_ROLE !== 'director' || - environment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== SELECTOR_REVISION_MARKER || - !(revisionMinimum(revision) >= config.directorMinimumInstances) || - !inventory.some((cell) => cell.id === cellId) - ) { - throw new Error('active director is not compatible with the promoted cell') - } - return { activeRevision: active[0].revisionName } -} - -export function pruneIncompatibleDirectorRevisions(config, deps) { - const common = [ - '--project', - config.project, - '--region', - config.directorRegion, - '--format=json' - ] - const service = deps.commandJson([ - 'run', - 'services', - 'describe', - config.directorService, - ...common - ]) - const traffic = service.status?.traffic ?? [] - const active = traffic.filter((entry) => Number(entry.percent ?? 0) > 0) - const rollback = traffic.find((entry) => entry.tag === SELECTOR_ROLLBACK_TAG) - const unexpectedTags = traffic.filter( - (entry) => entry.tag && entry.tag !== SELECTOR_ROLLBACK_TAG - ) - if ( - active.length !== 1 || - Number(active[0].percent) !== 100 || - !active[0].revisionName || - !rollback?.revisionName || - Number(rollback.percent ?? 0) !== 0 || - rollback.revisionName === active[0].revisionName || - unexpectedTags.length > 0 - ) { - throw new Error('director traffic is not ready for selector cutover') - } - const activeRevision = deps.commandJson([ - 'run', - 'revisions', - 'describe', - active[0].revisionName, - ...common - ]) - const rollbackRevision = deps.commandJson([ - 'run', - 'revisions', - 'describe', - rollback.revisionName, - ...common - ]) - const activeEnvironment = revisionEnvironment(activeRevision) - const rollbackEnvironment = revisionEnvironment(rollbackRevision) - const activeInventory = directorInventory(activeEnvironment, active[0].revisionName) - const rollbackInventory = directorInventory(rollbackEnvironment, rollback.revisionName) - if ( - activeEnvironment.ORCA_RELAY_ROLE !== 'director' || - rollbackEnvironment.ORCA_RELAY_ROLE !== 'director' || - activeEnvironment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== - SELECTOR_REVISION_MARKER || - rollbackEnvironment.ORCA_RELAY_ADMISSION_SELECTOR_VERSION !== - SELECTOR_REVISION_MARKER || - !activeRevision.spec?.containers?.[0]?.image || - activeRevision.spec?.containers?.[0]?.image !== - rollbackRevision.spec?.containers?.[0]?.image || - activeInventory !== rollbackInventory || - revisionMinimum(rollbackRevision) !== 0 || - !(revisionMinimum(activeRevision) >= config.directorMinimumInstances) - ) { - throw new Error('director compatibility pair failed before revision pruning') - } - const retained = new Set([active[0].revisionName, rollback.revisionName]) - const revisions = deps.commandJson([ - 'run', - 'revisions', - 'list', - '--service', - config.directorService, - ...common - ]) - for (const revision of revisions) { - const revisionName = revision.metadata?.name - if (!revisionName) throw new Error('director revision list contains an unnamed revision') - if (retained.has(revisionName)) continue - deps.command([ - 'run', - 'revisions', - 'delete', - revisionName, - '--project', - config.project, - '--region', - config.directorRegion, - '--quiet' - ]) - } -} - -export function cutoverMembership(topology, config) { - const all = Object.keys(topology).sort() - const migration = new Set(config.targetCellIds) - const general = new Set(config.generalCellIds) - if ([...migration].some((cellId) => general.has(cellId))) { - throw new Error('general and migration-only cell sets overlap') - } - if (migration.has(config.sourceCellId) || general.has(config.sourceCellId)) { - throw new Error('cutover source must remain existing-only') - } - for (const cellId of [...migration, ...general]) { - if (!all.includes(cellId)) throw new Error(`selector cell ${cellId} is absent from topology`) - } - return { - existingOnly: all.filter((cellId) => !migration.has(cellId) && !general.has(cellId)), - migrationOnly: [...migration].sort(), - general: [...general].sort() - } -} - -function assertDistinctCutoverResources(cells) { - const resources = new Map() - for (const cell of cells) { - for (const key of ['origin', 'migName', 'instanceGroup', 'backendName', 'backendId']) { - const resourceKey = `${key}:${cell[key]}` - const previous = resources.get(resourceKey) - if (previous) throw new Error(`${cell.cellId} ${key} overlaps ${previous}`) - resources.set(resourceKey, cell.cellId) - } - } -} - -export function assertCutoverCellReady(cellId, status, unobservedConnectionBound) { - const capacity = status.runtimeConnectionCapacity - const directorCapacity = status.connectionCapacity - const process = status.process - const expectedPause = - CUTOVER_CONNECTION_HARD_CAP - - CUTOVER_CONTROL_REBIND_RESERVE - - unobservedConnectionBound - if ( - !capacity || - capacity.hardCap !== CUTOVER_CONNECTION_HARD_CAP || - capacity.controlRebindReserve !== CUTOVER_CONTROL_REBIND_RESERVE || - capacity.ordinaryConnectionLimit !== - CUTOVER_CONNECTION_HARD_CAP - CUTOVER_CONTROL_REBIND_RESERVE || - capacity.unobservedBound !== unobservedConnectionBound || - capacity.normalAdmissionPause !== expectedPause || - !directorCapacity || - directorCapacity.hardCap !== capacity.hardCap || - directorCapacity.controlRebindReserve !== capacity.controlRebindReserve || - directorCapacity.ordinaryConnectionLimit !== capacity.ordinaryConnectionLimit || - directorCapacity.unobservedBound !== capacity.unobservedBound || - directorCapacity.normalAdmissionPause !== capacity.normalAdmissionPause || - directorCapacity.heartbeatFresh !== true || - expectedPause <= 0 - ) { - throw new Error(`${cellId} does not expose the reviewed connection-capacity policy`) - } - if ( - !process || - !Number.isSafeInteger(process.enforcedConnectionUnits) || - process.enforcedConnectionUnits < 0 || - !Number.isSafeInteger(process.preAuthConnections) || - process.preAuthConnections < 0 || - !Number.isSafeInteger(directorCapacity.pendingControlReservations) || - directorCapacity.pendingControlReservations < 0 - ) { - throw new Error(`${cellId} has incomplete connection-capacity evidence`) - } - if (process.preAuthConnections >= MAX_PRE_AUTH_CONNECTIONS) { - throw new Error(`${cellId} has insufficient pre-auth connection headroom`) - } - const committedUnits = - process.enforcedConnectionUnits + directorCapacity.pendingControlReservations - if (!Number.isSafeInteger(committedUnits) || committedUnits >= expectedPause) { - throw new Error(`${cellId} has insufficient normal-admission connection headroom`) - } - if (status.draining) throw new Error(`${cellId} is draining before selector cutover`) -} - -async function inspectCutoverCells(topology, config, deps, adminPost, membership) { - const selectedIds = [...membership.migrationOnly, ...membership.general] - const selected = selectedIds.map((cellId) => deployment(topology[cellId], cellId)) - assertDistinctCutoverResources([ - deployment(topology[config.sourceCellId], config.sourceCellId), - ...selected - ]) - for (const cell of selected) { - const status = await inspectCell(config, deps, adminPost, cell) - assertCutoverCellReady(cell.cellId, status, config.unobservedConnectionBound) - } -} - -function projection(total, quota, assignments) { - return assignments === 0 ? 0 : Math.ceil((total * quota) / assignments) -} - -function connectionProjection(current, sourceConnections, sourceAssignments, quota) { - const unboundConnections = Math.max(0, sourceConnections - sourceAssignments) - return current + quota + unboundConnections -} - -function targetConnectionCeiling(config, target) { - return Math.min( - config.connectionCeiling, - target.connectionHardCap ?? CUTOVER_CONNECTION_HARD_CAP - ) -} - -function connectionReservationHeadroom(status, cellId) { - const capacity = status.connectionCapacity - const values = [ - capacity?.hardCap, - capacity?.controlRebindReserve, - capacity?.ordinaryConnectionLimit, - capacity?.unobservedBound, - capacity?.normalAdmissionPause, - capacity?.enforcedConnectionUnits, - capacity?.pendingControlReservations - ] - if ( - capacity?.heartbeatFresh !== true || - values.some((value) => !Number.isSafeInteger(value) || value < 0) || - capacity.ordinaryConnectionLimit !== capacity.hardCap - capacity.controlRebindReserve || - capacity.normalAdmissionPause !== - capacity.ordinaryConnectionLimit - capacity.unobservedBound - ) { - throw new Error(`${cellId} has inconsistent connection-reservation capacity`) - } - return Math.max( - 0, - capacity.normalAdmissionPause - - capacity.enforcedConnectionUnits - - capacity.pendingControlReservations - ) -} - -export function allocateTargetQuotas({ - sourceAssignments, - sourceConnections, - requiredTargetUnits, - targets, - connectionCeiling -}) { - const quotas = new Map(targets.map((target) => [target.cellId, 0])) - for (let assigned = 0; assigned < sourceAssignments; assigned++) { - const candidates = targets - .map((target) => { - const quota = quotas.get(target.cellId) + 1 - const projectedConnections = connectionProjection( - target.currentConnections, - sourceConnections, - sourceAssignments, - quota - ) - const projectedUnits = projection(requiredTargetUnits, quota, sourceAssignments) - return { target, quota, projectedConnections, projectedUnits } - }) - .filter( - ({ target, quota, projectedConnections, projectedUnits }) => - projectedConnections < (target.connectionCeiling ?? connectionCeiling) && - quota <= target.availableConnectionReservations && - projectedUnits <= target.availableTargetUnits - ) - .sort( - (left, right) => - left.projectedConnections - right.projectedConnections || - left.target.cellId.localeCompare(right.target.cellId) - ) - const selected = candidates[0] - if (!selected) throw new Error('multi-target connection or request-unit headroom exhausted') - quotas.set(selected.target.cellId, selected.quota) - } - return targets.map((target) => ({ - ...target, - quota: quotas.get(target.cellId), - projectedConnections: connectionProjection( - target.currentConnections, - sourceConnections, - sourceAssignments, - quotas.get(target.cellId) - ), - projectedUnits: projection( - requiredTargetUnits, - quotas.get(target.cellId), - sourceAssignments - ) - })) -} - -function defaultCommand(args) { - const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) - if (result.status !== 0) { - throw new Error(`gcloud ${args.slice(0, 5).join(' ')} failed: ${result.stderr.trim()}`) - } -} - -function defaultCommandResult(args) { - const result = spawnSync('gcloud', args, { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'pipe'] - }) - return { status: result.status, stdout: result.stdout, stderr: result.stderr } -} - -function validatedProcessCounts(counts, cellId) { - for (const field of [ - 'totalConnections', - 'preAuthConnections', - 'controls', - 'splices', - 'pendingSplices', - 'queuedBytes' - ]) { - if (!Number.isSafeInteger(counts?.[field]) || counts[field] < 0) { - throw new Error(`${cellId} runtime metrics have invalid ${field}`) - } - } - return counts -} - -function processCounts(config, deps, status, cellId) { - if (status.process) return validatedProcessCounts(status.process, cellId) - const filter = [ - 'resource.type="gce_instance"', - 'jsonPayload.event="orca_relay_runtime_metrics"', - `jsonPayload.cellId="${cellId}"` - ].join(' AND ') - const entries = deps.commandJson([ - 'logging', - 'read', - filter, - '--project', - config.project, - '--freshness=5m', - '--limit=1', - '--order=desc', - '--format=json' - ]) - const entry = entries[0] - if (!entry?.jsonPayload) throw new Error(`${cellId} has no fresh runtime metrics`) - const timestamp = Date.parse(entry.timestamp) - if (!Number.isFinite(timestamp) || timestamp < deps.now() - 90_000) { - throw new Error(`${cellId} runtime metrics are stale`) - } - return validatedProcessCounts(entry.jsonPayload, cellId) -} - -function runtimeConnections(counts, cellId) { - const count = counts?.totalConnections - if (!Number.isSafeInteger(count) || count < 0) { - throw new Error(`${cellId} runtime does not expose totalConnections`) - } - return count -} - -function runtimeIncarnation(status, cellId) { - const incarnation = status.runtime?.cellIncarnation - if (typeof incarnation !== 'string' || incarnation.length === 0) { - throw new Error(`${cellId} has no exact runtime incarnation`) - } - return incarnation -} - -async function pairStatus(config, adminPost, targetCellId, completeReady) { - return await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', { - v: 1, - sourceCellId: config.sourceCellId, - targetCellId, - completeReady - }) -} - -async function allPairStatuses(config, adminPost, completeReady) { - const statuses = [] - for (const targetCellId of config.targetCellIds) { - statuses.push({ - targetCellId, - status: await pairStatus(config, adminPost, targetCellId, completeReady) - }) - } - return statuses -} - -function statusTotals(statuses) { - return statuses.reduce( - (totals, { status }) => ({ - inProgress: totals.inProgress + status.inProgress, - targetRegistered: totals.targetRegistered + status.targetRegistered, - registeredSourceActive: totals.registeredSourceActive + status.registeredSourceActive, - registeredCompletable: totals.registeredCompletable + status.registeredCompletable, - registeredTargetInactive: - totals.registeredTargetInactive + status.registeredTargetInactive, - completed: totals.completed + status.completed, - blocked: totals.blocked + status.blocked, - expiredUnregistered: totals.expiredUnregistered + status.expiredUnregistered, - repairableExpiredUnregistered: - totals.repairableExpiredUnregistered + status.repairableExpiredUnregistered, - abortableExpiredUnregistered: - totals.abortableExpiredUnregistered + status.abortableExpiredUnregistered, - blockedExpiredUnregistered: - totals.blockedExpiredUnregistered + status.blockedExpiredUnregistered, - blockedExpiredOnNewerTargetAssignment: - totals.blockedExpiredOnNewerTargetAssignment + - status.blockedExpiredOnNewerTargetAssignment - }), - { - inProgress: 0, - targetRegistered: 0, - registeredSourceActive: 0, - registeredCompletable: 0, - registeredTargetInactive: 0, - completed: 0, - blocked: 0, - expiredUnregistered: 0, - repairableExpiredUnregistered: 0, - abortableExpiredUnregistered: 0, - blockedExpiredUnregistered: 0, - blockedExpiredOnNewerTargetAssignment: 0 - } - ) -} - -function hasDurableTargetOwnership(totals) { - return ( - totals.inProgress === totals.targetRegistered && - totals.targetRegistered === - totals.registeredSourceActive + - totals.registeredCompletable + - totals.registeredTargetInactive && - totals.registeredSourceActive === 0 && - totals.expiredUnregistered === 0 && - totals.repairableExpiredUnregistered === 0 && - totals.abortableExpiredUnregistered === 0 && - totals.blockedExpiredUnregistered === 0 && - totals.blockedExpiredOnNewerTargetAssignment === 0 - ) -} - -function boundedUnregisteredMigrations(totals, unobservedConnectionBound) { - const unregistered = totals.inProgress - totals.targetRegistered - return ( - Number.isSafeInteger(unobservedConnectionBound) && - unregistered > 0 && - unregistered <= unobservedConnectionBound && - totals.targetRegistered === - totals.registeredSourceActive + - totals.registeredCompletable + - totals.registeredTargetInactive && - totals.registeredSourceActive === 0 && - totals.blocked === 0 && - totals.expiredUnregistered === 0 && - totals.repairableExpiredUnregistered === 0 && - totals.abortableExpiredUnregistered === 0 && - totals.blockedExpiredUnregistered === 0 && - totals.blockedExpiredOnNewerTargetAssignment === 0 - ) -} - -function isSettledOrOffline(totals) { - return ( - hasDurableTargetOwnership(totals) && - totals.registeredCompletable === 0 && - totals.blocked === totals.registeredTargetInactive - ) -} - -function assertLeaseGate(statuses, minimumLeaseRemainingMs) { - const remaining = statuses - .map(({ status }) => status.oldestRemainingMs) - .filter((value) => value !== null) - if (remaining.length === 0 || Math.min(...remaining) < minimumLeaseRemainingMs) { - throw new Error('oldest migration lease has insufficient time remaining') - } -} - -async function targetRuntime(config, adminPost, target) { - if (config.mode !== 'fence-source') { - const runtime = await adminPost(target.origin, '/v1/admin/runtime-status', { v: 1 }) - const count = runtime.runtime?.totalConnections - if (!Number.isSafeInteger(count) || count < 0) { - throw new Error(`${target.cellId} runtime does not expose totalConnections`) - } - if (count >= targetConnectionCeiling(config, target)) { - throw new Error(`${target.cellId} reached the connection ceiling`) - } - return count - } - const result = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { - v: 1, - cellId: target.cellId - }) - const status = result.status - if ( - status?.cellId !== target.cellId || - status.cellUrl !== target.origin || - status.runtime?.cellUrl !== target.origin || - status.runtime?.ready !== true || - status.runtime?.heartbeatFresh !== true - ) { - throw new Error(`${target.cellId} has no fresh matching director runtime snapshot`) - } - const values = [ - status.runtime.observedRequests, - status.connectionCapacity?.observedConnections, - status.connectionCapacity?.enforcedConnectionUnits - ] - if (values.some((value) => !Number.isSafeInteger(value) || value < 0)) { - throw new Error(`${target.cellId} has incomplete director runtime counts`) - } - const count = Math.max(...values) - connectionReservationHeadroom(status, target.cellId) - if (count >= Math.min(config.connectionCeiling, status.connectionCapacity.hardCap)) { - throw new Error(`${target.cellId} reached the connection ceiling`) - } - return count -} - -async function checkPublicCellEndpoint(deps, cell, path) { - const response = await deps.fetch(`${cell.origin}${path}`, { - signal: AbortSignal.timeout(15_000) - }) - const body = await response.json().catch(() => ({})) - if (!response.ok || body.ok !== true) { - throw new Error(`${cell.cellId} ${path} is unavailable`) - } -} - -async function inspectGeneralPromotionTarget(config, deps, adminPost, target) { - await checkPublicCellEndpoint(deps, target, '/health') - await checkPublicCellEndpoint(deps, target, '/ready') - const result = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { - v: 1, - cellId: target.cellId - }) - const status = result.status - if ( - status?.cellId !== target.cellId || - status.cellUrl !== target.origin || - status.runtime?.cellUrl !== target.origin || - status.runtime?.ready !== true || - status.runtime?.heartbeatFresh !== true - ) { - throw new Error(`${target.cellId} has no fresh matching director runtime snapshot`) - } - connectionReservationHeadroom(status, target.cellId) - const currentConnections = Math.max( - status.runtime.observedRequests, - status.connectionCapacity.observedConnections, - status.connectionCapacity.enforcedConnectionUnits - ) - if ( - !Number.isSafeInteger(currentConnections) || - currentConnections >= targetConnectionCeiling(config, target) - ) { - throw new Error(`${target.cellId} reached the connection ceiling`) - } -} - -function validateReviewedInstanceTemplate(template, expected, capacityPredecessor) { - const startupScript = (template.properties?.metadata?.items ?? []) - .find((item) => item.key === 'startup-script')?.value - const configuredDigest = startupScript?.match( - /ORCA_RELAY_IMAGE_DIGEST=%s\\n' '(sha256:[a-f0-9]{64})'/ - )?.[1] - const configuredImages = [ - ...String(startupScript ?? '').matchAll( - /'(?:[a-z0-9.-]+\/)+[a-z0-9._/-]+@(sha256:[a-f0-9]{64})'/g - ) - ].map((match) => match[1]) - if ( - template.selfLink !== expected.generationIdentity || - configuredDigest !== expected.imageDigest || - !configuredImages.includes(expected.imageDigest) - ) { - throw new Error(`${expected.cellId} instance template does not pin the reviewed image`) - } - const hardCaps = [ - ...String(startupScript ?? '').matchAll( - /^ printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '([0-9]+)'$/gm - ) - ].map((match) => Number(match[1])) - const unobservedBounds = [ - ...String(startupScript ?? '').matchAll( - /^ printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '([0-9]+)'$/gm - ) - ].map((match) => Number(match[1])) - const hardCap = hardCaps[0] - const unobservedBound = unobservedBounds[0] - const exactCapacityPredecessor = - capacityPredecessor !== undefined && - hardCaps.length === 1 && - unobservedBounds.length === 1 && - hardCap === capacityPredecessor.hardCap && - unobservedBound === capacityPredecessor.unobservedBound - if (expected.connectionHardCap === undefined) { - if (!exactCapacityPredecessor && (hardCaps.length !== 0 || unobservedBounds.length !== 0)) { - throw new Error(`${expected.cellId} instance template capacity differs from Terraform`) - } - return exactCapacityPredecessor - ? { - ...expected, - connectionHardCap: hardCap, - connectionUnobservedBound: unobservedBound - } - : expected - } - const isReviewedPredecessor = - exactCapacityPredecessor && expected.connectionHardCap === 1_000 - if ( - hardCaps.length !== 1 || - unobservedBounds.length !== 1 || - !Number.isSafeInteger(hardCap) || - !Number.isSafeInteger(unobservedBound) || - (hardCap !== expected.connectionHardCap && !isReviewedPredecessor) || - unobservedBound !== expected.connectionUnobservedBound - ) { - throw new Error(`${expected.cellId} instance template capacity is outside reviewed rollout`) - } - return { - ...expected, - connectionHardCap: hardCap, - connectionUnobservedBound: unobservedBound - } -} - -async function inspectDirectorObservedCell(config, deps, adminPost, expected) { - const common = ['--project', config.project, '--zone', expected.zone, '--format=json'] - const mig = deps.commandJson([ - 'compute', - 'instance-groups', - 'managed', - 'describe', - expected.migName, - ...common - ]) - const instances = deps.commandJson([ - 'compute', - 'instance-groups', - 'managed', - 'list-instances', - expected.migName, - ...common - ]) - const instanceName = validateMig(mig, instances, expected) - if ( - mig.instanceTemplate !== expected.generationIdentity || - instances[0]?.version?.instanceTemplate !== expected.generationIdentity - ) { - throw new Error(`${expected.cellId} MIG does not serve the reviewed generation`) - } - const instance = deps.commandJson([ - 'compute', - 'instances', - 'describe', - instanceName, - ...common - ]) - validateInstance(instance, expected, config.runtimeServiceAccount) - const templateName = new URL(expected.generationIdentity).pathname.split('/').at(-1) - const template = deps.commandJson([ - 'compute', - 'instance-templates', - 'describe', - templateName, - '--project', - config.project, - '--format=json' - ]) - const deployed = validateReviewedInstanceTemplate( - template, - expected, - config.mode === 'fence-source' && - (expected.connectionHardCap !== undefined || expected.legacyCapacityTopology) - ? { - hardCap: config.connectionCeiling, - unobservedBound: config.unobservedConnectionBound - } - : undefined - ) - const backend = deps.commandJson([ - 'compute', - 'backend-services', - 'describe', - expected.backendName, - '--global', - '--project', - config.project, - '--format=json' - ]) - validateBackend(backend, expected) - await assertCellRoute(config, deps, expected) - await checkPublicCellEndpoint(deps, expected, '/health') - await checkPublicCellEndpoint(deps, expected, '/ready') - const result = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { - v: 1, - cellId: expected.cellId - }) - const status = result.status - if ( - status?.cellId !== expected.cellId || - status.cellUrl !== expected.origin || - status.runtime?.cellUrl !== expected.origin || - status.runtime?.ready !== true || - status.runtime?.heartbeatFresh !== true - ) { - throw new Error(`${expected.cellId} has no fresh matching director runtime snapshot`) - } - connectionReservationHeadroom(status, expected.cellId) - assertDeploymentConnectionCapacity( - deployed, - status.connectionCapacity ?? null, - status.connectionCapacity ?? null - ) - const connectionValues = [ - status.runtime.observedRequests, - status.connectionCapacity.observedConnections, - status.connectionCapacity.enforcedConnectionUnits - ] - if (connectionValues.some((value) => !Number.isSafeInteger(value) || value < 0)) { - throw new Error(`${expected.cellId} has incomplete director runtime counts`) - } - const currentConnections = Math.max(...connectionValues) - if (currentConnections >= targetConnectionCeiling(config, deployed)) { - throw new Error(`${expected.cellId} reached the connection ceiling`) - } - return { - ...status, - process: { totalConnections: currentConnections } - } -} - -async function waitForMultiStatus( - config, - deps, - adminPost, - targets, - completeReady, - allowBoundedUnregistered = false, - requireZero = false -) { - const deadline = deps.now() + config.timeoutMs - while (deps.now() < deadline) { - const statuses = await allPairStatuses(config, adminPost, completeReady) - for (const target of targets) await targetRuntime(config, adminPost, target) - const totals = statusTotals(statuses) - deps.emit({ - event: completeReady ? 'multi_migration_completion' : 'multi_migration_registration', - ...totals - }) - const boundedUnregistered = - allowBoundedUnregistered && - boundedUnregisteredMigrations(totals, config.unobservedConnectionBound) - const boundedSettlement = boundedUnregistered && totals.registeredCompletable === 0 - if ( - completeReady - ? (requireZero - ? totals.inProgress === 0 - : isSettledOrOffline(totals) || boundedSettlement) - : totals.inProgress === totals.targetRegistered || boundedUnregistered - ) { - if (boundedUnregistered) { - deps.emit({ - event: completeReady - ? 'multi_migration_bounded_offline_complete' - : 'multi_migration_bounded_offline_registered', - unobservedConnectionBound: config.unobservedConnectionBound, - unregistered: totals.inProgress - totals.targetRegistered, - ...totals - }) - } - return statuses - } - await deps.wait(config.pollIntervalMs) - } - throw new Error( - completeReady - ? 'timed out waiting for multi-target completion' - : 'timed out waiting for multi-target registration' - ) -} - -async function waitForRecoveredSourceZero( - config, - deps, - adminPost, - source, - expectedIncarnation -) { - const deadline = deps.now() + config.timeoutMs - while (deps.now() < deadline) { - const status = await inspectCell(config, deps, adminPost, source) - const counts = processCounts(config, deps, status, source.cellId) - deps.emit({ - event: 'source_recovery_runtime_settlement', - draining: status.draining, - activityLeases: status.activityLeases, - reservedRequests: status.reservedRequests, - controls: counts.controls, - splices: counts.splices, - pendingSplices: counts.pendingSplices - }) - if ( - runtimeIncarnation(status, source.cellId) !== expectedIncarnation - ) { - throw new Error('source incarnation changed during recovery settlement') - } - if ( - status.draining && - status.activityLeases === 0 && - status.reservedRequests === 0 && - counts.controls === 0 && - counts.splices === 0 && - counts.pendingSplices === 0 - ) { - return - } - await deps.wait(config.pollIntervalMs) - } - throw new Error('timed out waiting for recovered source runtime to reach zero') -} - -async function rollbackBeforeDrain(config, deps, adminPost, targets, selectorActive) { - let statuses - try { - statuses = await allPairStatuses(config, adminPost, false) - } catch (error) { - deps.emit({ - event: 'multi_forward_recovery_required', - targetRegistered: null, - reason: 'registration_status_unavailable' - }) - throw new Error('cannot prove zero target registrations; preserving forward recovery', { - cause: error - }) - } - const totals = statusTotals(statuses) - if (totals.targetRegistered > 0) { - deps.emit({ event: 'multi_forward_recovery_required', ...totals }) - return - } - if (selectorActive) { - deps.emit({ event: 'multi_rollback_preserved_selector', ...totals }) - return - } - await setCellState(config, adminPost, config.sourceCellId, true).catch(() => undefined) - for (const target of targets) { - await setCellState(config, adminPost, target.cellId, false).catch(() => undefined) - } - deps.emit({ event: 'multi_rollback_waiting_for_lease_expiry', ...totals }) -} - -async function publishMigrations(config, deps, adminPost, plannedTargets) { - for (const target of plannedTargets) { - let remaining = target.quota - while (remaining > 0) { - const limit = Math.min(config.batchSize, remaining) - let result - try { - result = await adminPost(config.directorOrigin, '/v1/admin/evacuate-cell', { - v: 1, - sourceCellId: config.sourceCellId, - targetCellId: target.cellId, - limit - }) - } catch (error) { - if ( - config.mode !== 'recover-forward' || - !(error instanceof Error) || - error.message !== - '/v1/admin/evacuate-cell failed: relay_connection_headroom_exhausted' - ) { - throw error - } - deps.emit({ - event: 'multi_recovery_target_headroom_paused', - targetCellId: target.cellId, - remaining - }) - break - } - if ( - !Number.isSafeInteger(result.started) || - result.started < 0 || - result.started > limit - ) { - throw new Error(`${target.cellId} migration quota could not be filled deterministically`) - } - if (result.started === 0 && config.mode === 'recover-forward') { - deps.emit({ - event: 'multi_recovery_quota_depleted', - targetCellId: target.cellId, - remaining - }) - break - } - if (result.started === 0) { - throw new Error(`${target.cellId} migration quota could not be filled deterministically`) - } - remaining -= result.started - deps.emit({ - event: 'multi_migration_batch', - targetCellId: target.cellId, - started: result.started, - remaining - }) - } - } -} - -function sourceMig(config, deps, source) { - const common = ['--project', config.project, '--zone', source.zone, '--format=json'] - return { - mig: deps.commandJson([ - 'compute', - 'instance-groups', - 'managed', - 'describe', - source.migName, - ...common - ]), - instances: deps.commandJson([ - 'compute', - 'instance-groups', - 'managed', - 'list-instances', - source.migName, - ...common - ]) - } -} - -function validateFencedMig(mig, source) { - const policy = mig.updatePolicy ?? {} - if ( - Number(mig.targetSize) !== 0 || - mig.instanceTemplate !== source.generationIdentity || - policy.replacementMethod !== 'RECREATE' || - Number(policy.maxSurge?.fixed ?? policy.maxSurge) !== 0 || - Number(policy.maxUnavailable?.fixed ?? policy.maxUnavailable) !== 1 - ) { - throw new Error(`${source.cellId} MIG fence topology is unsafe`) - } -} - -function canonicalBackendServiceId(value) { - if (typeof value !== 'string') return null - const prefix = 'https://www.googleapis.com/compute/v1/' - const resource = value.startsWith(prefix) ? value.slice(prefix.length) : value - return /^projects\/[a-z][a-z0-9-]{4,29}\/global\/backendServices\/[A-Za-z0-9_-]{1,63}$/.test( - resource - ) - ? resource - : null -} - -export function sameBackendServiceResource(left, right) { - if (left === right) return true - const leftId = canonicalBackendServiceId(left) - return leftId !== null && leftId === canonicalBackendServiceId(right) -} - -function validateRetainedRoute(urlMap, source) { - const hostname = new URL(source.origin).hostname - const hostRule = (urlMap.hostRules ?? []).find((rule) => - (rule.hosts ?? []).includes(hostname) - ) - const matcher = (urlMap.pathMatchers ?? []).find( - (candidate) => candidate.name === hostRule?.pathMatcher - ) - if ( - !source.urlMapName || - urlMap.name !== source.urlMapName || - !sameBackendServiceResource(matcher?.defaultService, source.backendId) || - (matcher.pathRules?.length ?? 0) !== 0 || - (matcher.routeRules?.length ?? 0) !== 0 || - matcher.defaultRouteAction !== undefined || - matcher.defaultUrlRedirect !== undefined || - matcher.headerAction !== undefined - ) { - throw new Error(`${source.cellId} retained route topology mismatch`) - } -} - -async function assertCellRoute(config, deps, cell) { - const urlMap = deps.commandJson([ - 'compute', - 'url-maps', - 'describe', - cell.urlMapName, - '--global', - '--project', - config.project, - '--format=json' - ]) - validateRetainedRoute(urlMap, cell) - const proxy = deps.commandJson([ - 'compute', - 'target-https-proxies', - 'describe', - cell.urlMapName, - '--global', - '--project', - config.project, - '--format=json' - ]) - const forwardingRule = deps.commandJson([ - 'compute', - 'forwarding-rules', - 'describe', - cell.urlMapName, - '--global', - '--project', - config.project, - '--format=json' - ]) - const address = deps.commandJson([ - 'compute', - 'addresses', - 'describe', - cell.urlMapName, - '--global', - '--project', - config.project, - '--format=json' - ]) - const resolved = await deps.resolve4(new URL(cell.origin).hostname) - if ( - proxy.name !== cell.urlMapName || - proxy.urlMap !== urlMap.selfLink || - forwardingRule.name !== cell.urlMapName || - forwardingRule.target !== proxy.selfLink || - forwardingRule.IPAddress !== address.address || - forwardingRule.portRange !== '443-443' || - forwardingRule.loadBalancingScheme !== 'EXTERNAL_MANAGED' || - !Array.isArray(resolved) || - resolved.length === 0 || - resolved.some((value) => value !== address.address) - ) { - throw new Error(`${cell.cellId} live frontend topology mismatch`) - } -} - -async function inspectFencedSource(config, deps, adminPost, source, mig) { - validateFencedMig(mig, source) - const backend = deps.commandJson([ - 'compute', - 'backend-services', - 'describe', - source.backendName, - '--global', - '--project', - config.project, - '--format=json' - ]) - validateBackend(backend, source) - await assertCellRoute(config, deps, source) - const result = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { - v: 1, - cellId: source.cellId - }) - const status = result.status - if ( - status?.cellUrl !== source.origin || - (status.runtime !== null && status.runtime?.cellUrl !== source.origin) - ) { - throw new Error(`${source.cellId} fenced runtime does not match Terraform topology`) - } - return { ...status, draining: true, process: null } -} - -async function inspectFenceCandidate(config, deps, adminPost, cell, mig) { - const targetSize = Number(mig.targetSize) - const policy = mig.updatePolicy ?? {} - if ( - ![0, 1].includes(targetSize) || - policy.replacementMethod !== 'RECREATE' || - Number(policy.maxSurge?.fixed ?? policy.maxSurge) !== 0 || - Number(policy.maxUnavailable?.fixed ?? policy.maxUnavailable) !== 1 - ) { - throw new Error(`${cell.cellId} MIG fence topology is unsafe`) - } - const backend = deps.commandJson([ - 'compute', - 'backend-services', - 'describe', - cell.backendName, - '--global', - '--project', - config.project, - '--format=json' - ]) - validateBackend(backend, cell) - const result = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { - v: 1, - cellId: cell.cellId - }) - if ( - result.status?.cellUrl !== cell.origin || - result.status.runtime?.cellUrl !== cell.origin - ) { - throw new Error(`${cell.cellId} retained topology does not match Terraform`) - } - return result.status -} - -const CAPACITY_SNAPSHOT_ATTEMPTS = 3 -const CAPACITY_SNAPSHOT_RETRY_MS = 250 -const RECOVERY_CATCH_UP_PASSES = 5 -const RECOVERY_TARGET_OWNERSHIP_TIMEOUT_MS = 2 * 60 * 1_000 - -function conservativeRecoveryCapacity(rounds, targets, requireZeroProof) { - const capacities = rounds.flat() - for (const capacity of capacities) { - if ( - !Number.isSafeInteger(capacity.sourceAssignments) || - capacity.sourceAssignments < 0 || - !Number.isSafeInteger(capacity.requiredTargetUnits) || - capacity.requiredTargetUnits < capacity.sourceAssignments || - !Number.isSafeInteger(capacity.availableTargetUnits) || - capacity.availableTargetUnits < 0 - ) { - throw new Error('target capacity snapshot is internally inconsistent') - } - } - const observedSourceAssignments = capacities.map( - (capacity) => capacity.sourceAssignments - ) - const sourceAssignments = requireZeroProof - ? Math.max(...observedSourceAssignments) - : Math.min(...observedSourceAssignments) - const requiredTargetUnits = - sourceAssignments === 0 - ? 0 - : requireZeroProof - ? Math.max(...capacities.map((capacity) => capacity.requiredTargetUnits)) - : Math.max( - ...capacities.map((capacity) => - Math.ceil( - sourceAssignments * - (capacity.requiredTargetUnits / capacity.sourceAssignments) - ) - ) - ) - return { - sourceAssignments, - requiredTargetUnits, - availableTargetUnits: new Map(targets.map((target, index) => [ - target.cellId, - Math.min(...rounds.map((round) => round[index].availableTargetUnits)) - ])) - } -} - -async function readTargetCapacitySnapshot( - config, - deps, - adminPost, - source, - targets, - requireRecoveryZeroProof = false -) { - for (let attempt = 0; attempt < CAPACITY_SNAPSHOT_ATTEMPTS; attempt++) { - const rounds = [] - for (let round = 0; round < 2; round++) { - const capacities = [] - for (const target of targets) { - capacities.push(await adminPost( - config.directorOrigin, - '/v1/admin/evacuation-capacity', - { v: 1, sourceCellId: source.cellId, targetCellId: target.cellId } - )) - } - rounds.push(capacities) - } - if (config.mode === 'recover-forward') { - return conservativeRecoveryCapacity(rounds, targets, requireRecoveryZeroProof) - } - const capacities = rounds.flat() - const baseline = capacities[0] - if (capacities.every((capacity) => - capacity.sourceAssignments === baseline.sourceAssignments && - capacity.requiredTargetUnits === baseline.requiredTargetUnits - )) { - return { - sourceAssignments: baseline.sourceAssignments, - requiredTargetUnits: baseline.requiredTargetUnits, - availableTargetUnits: new Map(targets.map((target, index) => [ - target.cellId, - Math.min(...rounds.map((round) => round[index].availableTargetUnits)) - ])) - } - } - if (attempt + 1 < CAPACITY_SNAPSHOT_ATTEMPTS) { - await deps.wait(CAPACITY_SNAPSHOT_RETRY_MS) - } - } - throw new Error('target capacity snapshots disagree') -} - -async function preflight( - config, - deps, - adminPost, - source, - targets, - sourceFence = null, - selector = null, - coveredSourceConnections = null -) { - const sourceStatus = sourceFence - ? await inspectFencedSource(config, deps, adminPost, source, sourceFence.mig) - : config.mode === 'fence-source' - ? { - ...await inspectDirectorObservedCell(config, deps, adminPost, source), - process: null - } - : await inspectCell(config, deps, adminPost, source) - const sourceProcess = sourceFence - ? null - : processCounts(config, deps, sourceStatus, source.cellId) - const targetStatuses = [] - for (const target of targets) { - const status = config.mode === 'fence-source' - ? { - ...await inspectDirectorObservedCell(config, deps, adminPost, target), - process: null - } - : await inspectCell(config, deps, adminPost, target) - const targetProcess = processCounts(config, deps, status, target.cellId) - const targetAdmission = selector - ? selectorCellState(selector, target.cellId) - : status.enabled - ? 'general' - : 'existing-only' - if ( - ['preflight', 'execute'].includes(config.mode) && - (selector ? targetAdmission === 'general' : status.enabled) - ) { - throw new Error(`${target.cellId} must not start in general admission`) - } - targetStatuses.push({ - ...target, - status, - process: targetProcess, - currentConnections: runtimeConnections(targetProcess, target.cellId), - availableConnectionReservations: connectionReservationHeadroom( - status, - target.cellId - ), - connectionCeiling: Math.min( - config.connectionCeiling, - status.connectionCapacity.hardCap - ) - }) - } - if (config.mode === 'audit' || config.mode === 'recover-forward') { - const statuses = await allPairStatuses(config, adminPost, false) - deps.emit({ - event: config.mode === 'audit' ? 'multi_target_audit' : 'multi_forward_recovery_preflight', - ...statusTotals(statuses) - }) - if (config.mode === 'audit') { - return { - sourceProcess, - sourceStatus, - plannedTargets: targetStatuses, - sourceAlreadyFenced: Boolean(sourceFence) - } - } - } - const capacity = await readTargetCapacitySnapshot( - config, - deps, - adminPost, - source, - targets - ) - const { sourceAssignments, requiredTargetUnits } = capacity - const observedSourceConnections = sourceFence - ? 0 - : runtimeConnections(sourceProcess, source.cellId) - const sourceConnections = - coveredSourceConnections === null - ? observedSourceConnections - : sourceAssignments + - Math.max(0, observedSourceConnections - coveredSourceConnections) - for (const target of targetStatuses) { - target.availableTargetUnits = capacity.availableTargetUnits.get(target.cellId) - } - deps.emit({ - event: 'multi_target_capacity_snapshot', - sourceConnections, - observedSourceConnections, - sourceAssignments, - requiredTargetUnits, - targets: targetStatuses.map((target) => ({ - cellId: target.cellId, - currentConnections: target.currentConnections, - availableConnectionReservations: target.availableConnectionReservations, - availableTargetUnits: target.availableTargetUnits - })) - }) - if ( - config.mode === 'execute' && - (selector - ? selectorCellState(selector, source.cellId) !== 'existing-only' - : !sourceStatus.enabled) - ) { - throw new Error(selector ? 'source cell is not existing-only' : 'source cell is not enabled') - } - const plannedTargets = allocateTargetQuotas({ - sourceAssignments, - sourceConnections, - requiredTargetUnits, - targets: targetStatuses, - connectionCeiling: config.connectionCeiling - }) - deps.emit({ - event: 'multi_target_preflight', - source: aggregateCellStatus(sourceStatus), - sourceConnections, - observedSourceConnections, - sourceAssignments, - targets: plannedTargets.map((target) => ({ - cellId: target.cellId, - quota: target.quota, - currentConnections: target.currentConnections, - projectedConnections: target.projectedConnections, - projectedUnits: target.projectedUnits - })) - }) - return { sourceProcess, sourceStatus, plannedTargets, sourceAlreadyFenced: Boolean(sourceFence) } -} - -async function assertRecoveryPreDrain( - config, - deps, - adminPost, - source, - targets, - selectorPost, - expectedSelector -) { - const statuses = await allPairStatuses(config, adminPost, false) - const totals = statusTotals(statuses) - const capacity = await readTargetCapacitySnapshot( - config, - deps, - adminPost, - source, - targets, - true - ) - if (capacity.sourceAssignments !== 0 || capacity.requiredTargetUnits !== 0) { - deps.emit({ - event: 'multi_forward_recovery_catch_up', - sourceAssignments: capacity.sourceAssignments, - requiredTargetUnits: capacity.requiredTargetUnits - }) - return false - } - for (const target of targets) await targetRuntime(config, adminPost, target) - if (expectedSelector) { - const current = await inspectAdmissionSelector(selectorPost) - if ( - current.selector.generation !== expectedSelector.generation || - JSON.stringify(current.selector.membership) !== - JSON.stringify(expectedSelector.membership) - ) { - throw new Error('admission selector changed before recovery drain') - } - } - deps.emit({ - event: 'multi_forward_recovery_ready_to_drain', - ...totals - }) - return true -} - -async function waitForRecoveryTargetOwnership(config, deps, adminPost, targets) { - const deadline = - deps.now() + Math.min(config.timeoutMs, RECOVERY_TARGET_OWNERSHIP_TIMEOUT_MS) - while (deps.now() < deadline) { - const statuses = await allPairStatuses(config, adminPost, false) - for (const target of targets) await targetRuntime(config, adminPost, target) - const totals = statusTotals(statuses) - deps.emit({ event: 'multi_recovery_target_ownership', ...totals }) - assertLeaseGate(statuses, config.minimumLeaseRemainingMs) - if (hasDurableTargetOwnership(totals)) return - if (boundedUnregisteredMigrations(totals, config.unobservedConnectionBound)) { - const unregistered = totals.inProgress - totals.targetRegistered - deps.emit({ - event: - totals.targetRegistered === 0 - ? 'multi_recovery_bounded_unregistered' - : 'multi_recovery_bounded_mixed_registration', - unobservedConnectionBound: config.unobservedConnectionBound, - unregistered, - ...totals - }) - return - } - await deps.wait(config.pollIntervalMs) - } - throw new Error('timed out waiting for recovery target ownership') -} - -async function runEvacuation( - config, - deps, - adminPost, - token, - source, - targets, - plannedTargets, - sourceStatus, - selectorActive, - selectorPost, - expectedSelector -) { - let drainAttempted = false - try { - if (!selectorActive) { - await setCellState(config, adminPost, source.cellId, false) - for (const target of targets) await setCellState(config, adminPost, target.cellId, true) - } - await publishMigrations(config, deps, adminPost, plannedTargets) - const statuses = await allPairStatuses(config, adminPost, false) - assertLeaseGate(statuses, config.minimumLeaseRemainingMs) - for (const target of targets) await targetRuntime(config, adminPost, target) - if (selectorActive) { - const current = await inspectAdmissionSelector(selectorPost) - if ( - current.selector.generation !== expectedSelector.generation || - JSON.stringify(current.selector.membership) !== - JSON.stringify(expectedSelector.membership) - ) { - throw new Error('admission selector changed before drain') - } - } - const attemptId = randomUUID() - const traceValue = randomUUID() - const prepared = await adminPost( - config.directorOrigin, - '/v1/admin/drain-attempt-prepare', - { - v: 1, - attemptId, - cellId: source.cellId, - cellIncarnation: runtimeIncarnation(sourceStatus, source.cellId), - traceValue, - graceMs: 120_000, - confirmation: 'PREPARE_LEGACY_DRAIN' - } - ) - if (prepared.state !== 'prepared') { - throw new Error('planned drain already recorded; use recover-forward') - } - drainAttempted = true - const sending = await adminPost( - config.directorOrigin, - '/v1/admin/drain-attempt-send', - { - v: 1, - attemptId, - cellId: source.cellId, - cellIncarnation: runtimeIncarnation(sourceStatus, source.cellId) - } - ) - if ( - sending.attempt?.state !== 'send-may-have-started' || - sending.attempt.shouldSend !== true || - !Number.isSafeInteger(sending.attempt.sendPermitExpiresAt) || - deps.now() >= sending.attempt.sendPermitExpiresAt - ) { - throw new Error('drain send permit unavailable') - } - const receipt = await drainSource(config, deps, token, source, 120_000, traceValue) - await adminPost(config.directorOrigin, '/v1/admin/drain-attempt-receipt', { - v: 1, - attemptId, - cellId: source.cellId, - cellIncarnation: runtimeIncarnation(sourceStatus, source.cellId), - traceValue, - ...receipt - }) - deps.emit({ event: 'source_drain_accepted', sourceCellId: source.cellId }) - await waitForMultiStatus(config, deps, adminPost, targets, false) - await waitForMultiStatus(config, deps, adminPost, targets, true) - deps.emit({ event: 'multi_target_complete', sourceCellId: source.cellId }) - } catch (error) { - if (drainAttempted) { - const statuses = await allPairStatuses(config, adminPost, false).catch(() => []) - deps.emit({ event: 'multi_forward_recovery_required', ...statusTotals(statuses) }) - } else { - await rollbackBeforeDrain(config, deps, adminPost, targets, selectorActive) - } - throw error - } -} - -async function waitForFence(config, deps, adminPost, source) { - const deadline = deps.now() + config.timeoutMs - while (deps.now() < deadline) { - const mig = deps.commandJson([ - 'compute', - 'instance-groups', - 'managed', - 'describe', - source.migName, - '--project', - config.project, - '--zone', - source.zone, - '--format=json' - ]) - const instances = deps.commandJson([ - 'compute', - 'instance-groups', - 'managed', - 'list-instances', - source.migName, - '--project', - config.project, - '--zone', - source.zone, - '--format=json' - ]) - const status = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { - v: 1, - cellId: source.cellId - }) - if ( - Number(mig.targetSize) === 0 && - instances.length === 0 && - status.status?.cellUrl === source.origin && - status.status.enabled === false && - !status.status.runtime?.heartbeatFresh - ) { - const incarnation = status.status.runtime?.cellIncarnation - if (typeof incarnation !== 'string' || incarnation.length === 0) { - throw new Error('fenced source has no exact runtime incarnation') - } - return incarnation - } - await deps.wait(config.pollIntervalMs) - } - throw new Error('timed out waiting for durable source fence') -} - -function terraformFenceConfig(config, cell, cellIncarnation) { - return { - project: config.project, - environment: config.environment, - terraformDir: config.terraformDir, - varFile: config.terraformVarFile, - lockTimeout: '5m', - fenceCommit: config.fenceCommit, - cellIncarnation, - cell - } -} - -function fenceAttemptBody(attempt) { - return { - v: 1, - attemptId: attempt.attemptId, - environment: attempt.environment, - cellId: attempt.cellId, - cellIncarnation: attempt.cellIncarnation, - migName: attempt.migName, - instanceGroup: attempt.instanceGroup, - generationIdentity: attempt.generationIdentity, - fenceCommit: attempt.fenceCommit, - planSha256: attempt.planSha256, - planObjectName: attempt.planObjectName, - planObjectGeneration: attempt.planObjectGeneration, - varFileSha256: attempt.varFileSha256, - terraformStateLineage: attempt.terraformStateLineage, - terraformStateSerial: attempt.terraformStateSerial, - terraformStateObjectGeneration: attempt.terraformStateObjectGeneration, - terraformStateObjectSha256: attempt.terraformStateObjectSha256, - requestReason: attempt.requestReason, - ...(attempt.gceOperation ? { gceOperation: attempt.gceOperation } : {}) - } -} - -async function runTerraformManagedFence( - config, - deps, - adminPost, - cell, - cellIncarnation, - alreadyFenced, - preApplyGuard, - postApplyGuard -) { - const fenceConfig = terraformFenceConfig(config, cell, cellIncarnation) - const inspectProgress = async (_expected, attempt) => - await inspectTerraformFenceProgress( - fenceConfig, - { - terraform: deps.terraform, - gcloudJson: deps.commandJson - }, - attempt - ) - const attest = async (attempt) => { - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attest', { - ...fenceAttemptBody(attempt), - confirmation: 'ATTEST_TERRAFORM_FENCED_CELL' - }) - } - const attemptResult = await adminPost( - config.directorOrigin, - '/v1/admin/cell-fence-attempt-status', - { v: 1, cellId: cell.cellId } - ) - let existingAttempt = attemptResult.attempt - const planStore = { - uploadPlan: async (planPath, attempt) => - await uploadTerraformFencePlan( - fenceConfig, - { command: deps.command, commandJson: deps.commandJson }, - planPath, - attempt - ), - downloadPlan: async (attempt, planPath) => - await downloadTerraformFencePlan( - fenceConfig, - { command: deps.command }, - attempt, - planPath - ), - deletePlan: async (attempt) => - await deleteTerraformFencePlan( - fenceConfig, - { commandResult: deps.commandResult }, - attempt - ), - stateObjectBinding: async (statePath) => - await readTerraformStateObjectBinding( - fenceConfig, - { command: deps.command, commandJson: deps.commandJson }, - statePath - ) - } - if ( - existingAttempt && - existingAttempt.fenceCommit !== config.fenceCommit && - config.completedFenceRecovery - ) { - await deps.terraformFenceRecoverCompleted( - fenceConfig, - { - terraform: deps.terraform, - assertCommittedFenceSet: async () => - assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), - loadAttempt: async () => existingAttempt, - resolvePlan: async (attempt) => - await resolveTerraformFencePlanGeneration( - fenceConfig, - { commandResult: deps.commandResult }, - attempt - ), - stateObjectBinding: planStore.stateObjectBinding, - downloadPlan: planStore.downloadPlan, - deletePlan: planStore.deletePlan, - inspectCompletedProgress: async (_expected, attempt, recovery) => - await inspectCompletedTerraformFenceProgress( - fenceConfig, - { - terraform: deps.terraform, - gcloudJson: deps.commandJson - }, - attempt, - recovery - ), - markOperation: async (attempt, invocation) => - await adminPost( - config.directorOrigin, - '/v1/admin/cell-fence-attempt-operation', - { - ...fenceAttemptBody(attempt), - invocationId: invocation.invocationId, - invocationRequestReason: invocation.requestReason, - confirmation: 'RECORD_TERRAFORM_CELL_FENCE_OPERATION' - } - ), - assertZeroDiff: async () => - assertTerraformFenceZeroDiff(fenceConfig, { - terraform: deps.terraform - }), - postApplyGuard, - attest, - emit: deps.emit - }, - config.completedFenceRecovery - ) - return - } - if ( - existingAttempt && - !existingAttempt.abortedAt && - !existingAttempt.completedAt && - existingAttempt.fenceCommit !== config.fenceCommit - ) { - await deps.terraformFenceSupersede(fenceConfig, { - assertCommittedFenceSet: async () => - assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), - loadAttempt: async () => existingAttempt, - resolvePlan: async (attempt) => - await resolveTerraformFencePlanGeneration( - fenceConfig, - { commandResult: deps.commandResult }, - attempt - ), - inspectProgress, - abortAttempt: async (attempt) => - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-abort', { - ...fenceAttemptBody(attempt), - confirmation: 'ABORT_UNSTARTED_TERRAFORM_CELL_FENCE' - }), - emit: deps.emit - }) - existingAttempt = null - } - if ( - existingAttempt && - !existingAttempt.abortedAt && - (alreadyFenced || !existingAttempt.completedAt) - ) { - await deps.terraformFenceResume(fenceConfig, { - terraform: deps.terraform, - assertCommittedFenceSet: async () => - assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), - loadAttempt: async () => existingAttempt, - resolvePlan: async (attempt) => - await resolveTerraformFencePlanGeneration( - fenceConfig, - { commandResult: deps.commandResult }, - attempt - ), - bindPlan: async (attempt) => - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-plan', { - ...fenceAttemptBody(attempt), - confirmation: 'BIND_TERRAFORM_CELL_FENCE_PLAN' - }), - inspectProgress, - assertZeroDiff: async () => - assertTerraformFenceZeroDiff(fenceConfig, { terraform: deps.terraform }), - assertStateFenced: async () => - assertTerraformFenceStateFenced(fenceConfig, { terraform: deps.terraform }), - preApplyGuard, - postApplyGuard, - markOperation: async (attempt, invocation) => - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-operation', { - ...fenceAttemptBody(attempt), - invocationId: invocation.invocationId, - invocationRequestReason: invocation.requestReason, - confirmation: 'RECORD_TERRAFORM_CELL_FENCE_OPERATION' - }), - markApplyStarted: async (attempt, invocation) => - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-start', { - ...fenceAttemptBody(attempt), - invocationId: invocation.invocationId, - invocationRequestReason: invocation.requestReason, - confirmation: 'START_TERRAFORM_CELL_FENCE' - }), - attest, - ...planStore, - emit: deps.emit - }) - return - } - if (alreadyFenced) { - await deps.terraformFenceAdopt(fenceConfig, { - loadAttempt: async () => - ( - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-status', { - v: 1, - cellId: cell.cellId - }) - ).attempt, - assertCommittedFenceSet: async () => - assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), - assertStateFenced: async () => - assertTerraformFenceStateFenced(fenceConfig, { terraform: deps.terraform }), - preApplyGuard, - postApplyGuard, - attest: async (incarnation) => - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-adopt-legacy', { - v: 1, - cellId: cell.cellId, - cellIncarnation: incarnation, - confirmation: 'ADOPT_LEGACY_TERRAFORM_CELL_FENCE' - }), - commitAdoption: async (incarnation) => - await adminPost( - config.directorOrigin, - '/v1/admin/cell-fence-commit-legacy-adoption', - { - v: 1, - cellId: cell.cellId, - cellIncarnation: incarnation, - confirmation: 'COMMIT_LEGACY_TERRAFORM_CELL_FENCE_ADOPTION' - } - ), - emit: deps.emit - }) - return - } - if (existingAttempt && !existingAttempt.abortedAt && !existingAttempt.completedAt) { - throw new Error('prepared Terraform fence attempt must be aborted before replacement') - } - await deps.terraformFenceApply(fenceConfig, { - terraform: deps.terraform, - assertCommittedFenceSet: async () => - assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), - inspectProgress, - assertZeroDiff: async () => - assertTerraformFenceZeroDiff(fenceConfig, { terraform: deps.terraform }), - preApplyGuard, - postApplyGuard, - ...planStore, - prepareAttempt: async (attempt) => - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-prepare', { - ...fenceAttemptBody(attempt), - confirmation: 'PREPARE_TERRAFORM_CELL_FENCE' - }), - bindPlan: async (attempt) => - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-plan', { - ...fenceAttemptBody(attempt), - confirmation: 'BIND_TERRAFORM_CELL_FENCE_PLAN' - }), - markApplyStarted: async (attempt, invocation) => - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-start', { - ...fenceAttemptBody(attempt), - invocationId: invocation.invocationId, - invocationRequestReason: invocation.requestReason, - confirmation: 'START_TERRAFORM_CELL_FENCE' - }), - markOperation: async (attempt, invocation) => - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-operation', { - ...fenceAttemptBody(attempt), - invocationId: invocation.invocationId, - invocationRequestReason: invocation.requestReason, - confirmation: 'RECORD_TERRAFORM_CELL_FENCE_OPERATION' - }), - attest, - emit: deps.emit - }) -} - -async function abortTerraformManagedFence(config, deps, adminPost, cell) { - const result = await adminPost( - config.directorOrigin, - '/v1/admin/cell-fence-attempt-status', - { v: 1, cellId: cell.cellId } - ) - const attempt = result.attempt - const fenceConfig = terraformFenceConfig(config, cell, attempt?.cellIncarnation) - await deps.terraformFenceAbort(fenceConfig, { - terraform: deps.terraform, - assertCommittedFenceSet: async () => - assertTerraformFenceSet(fenceConfig, { terraform: deps.terraform }), - loadAttempt: async () => attempt, - resolvePlan: async (value) => - await resolveTerraformFencePlanGeneration( - fenceConfig, - { commandResult: deps.commandResult }, - value - ), - bindPlan: async (value) => - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-plan', { - ...fenceAttemptBody(value), - confirmation: 'BIND_TERRAFORM_CELL_FENCE_PLAN' - }), - inspectProgress: async () => - await inspectTerraformFenceProgress( - fenceConfig, - { - terraform: deps.terraform, - gcloudJson: deps.commandJson - }, - attempt - ), - abortAttempt: async () => - await adminPost(config.directorOrigin, '/v1/admin/cell-fence-attempt-abort', { - ...fenceAttemptBody(attempt), - confirmation: 'ABORT_UNSTARTED_TERRAFORM_CELL_FENCE' - }), - deletePlan: async (value) => - await deleteTerraformFencePlan( - fenceConfig, - { commandResult: deps.commandResult }, - value - ), - emit: deps.emit - }) -} - -async function fenceSource( - config, - deps, - adminPost, - source, - targets, - sourceProcess, - sourceStatus, - sourceAlreadyFenced -) { - if (sourceStatus.enabled) throw new Error('source fencing requires disabled admission') - const counts = sourceProcess - if ( - (!sourceAlreadyFenced && - (!counts || - sourceStatus.runtime?.observedRequests !== 0 || - counts.controls !== 0 || - counts.splices !== 0 || - counts.pendingSplices !== 0)) || - sourceStatus.activityLeases !== 0 || - sourceStatus.reservedRequests !== 0 - ) { - throw new Error('source fencing requires zero source-owned work') - } - const statuses = await allPairStatuses(config, adminPost, false) - const totals = statusTotals(statuses) - if ( - totals.inProgress !== sourceStatus.outgoingMigrations || - !hasDurableTargetOwnership(totals) - ) { - throw new Error('source fencing requires full migration coverage and durable target ownership') - } - for (const target of targets) await targetRuntime(config, adminPost, target) - const cellIncarnation = runtimeIncarnation(sourceStatus, source.cellId) - const postApplyGuard = async (expectedIncarnation) => { - const actualIncarnation = await waitForFence(config, deps, adminPost, source) - if (actualIncarnation !== expectedIncarnation) { - throw new Error('fenced source incarnation changed') - } - const finalSourceMig = sourceMig(config, deps, source) - if (finalSourceMig.instances.length !== 0) { - throw new Error('fenced source still has an instance') - } - await inspectFencedSource(config, deps, adminPost, source, finalSourceMig.mig) - } - await runTerraformManagedFence( - config, - deps, - adminPost, - source, - cellIncarnation, - sourceAlreadyFenced, - async () => { - const latestStatus = sourceAlreadyFenced - ? await inspectFencedSource( - config, - deps, - adminPost, - source, - sourceMig(config, deps, source).mig - ) - : { - ...await inspectDirectorObservedCell(config, deps, adminPost, source), - process: null - } - const latestCounts = sourceAlreadyFenced - ? null - : processCounts(config, deps, latestStatus, source.cellId) - if ( - latestStatus.enabled || - runtimeIncarnation(latestStatus, source.cellId) !== cellIncarnation || - (!sourceAlreadyFenced && - (latestStatus.runtime?.observedRequests !== 0 || - latestCounts.controls !== 0 || - latestCounts.splices !== 0 || - latestCounts.pendingSplices !== 0)) || - latestStatus.activityLeases !== 0 || - latestStatus.reservedRequests !== 0 - ) { - throw new Error('source fencing guards changed before Terraform apply') - } - const latestStatuses = await allPairStatuses(config, adminPost, false) - const latestTotals = statusTotals(latestStatuses) - if ( - latestTotals.inProgress !== latestStatus.outgoingMigrations || - !hasDurableTargetOwnership(latestTotals) - ) { - throw new Error('source migration coverage or guards changed before Terraform apply') - } - for (const target of targets) { - await inspectDirectorObservedCell(config, deps, adminPost, target) - } - }, - postApplyGuard - ) - await waitForMultiStatus(config, deps, adminPost, targets, true, false, true) - deps.emit({ event: 'source_fenced', sourceCellId: source.cellId, targetSize: 0 }) -} - -async function runTargetSupersession( - config, - deps, - adminPost, - source, - targets, - selector -) { - const failed = targets.find((target) => target.cellId === config.failedTargetCellId) - const replacement = targets.find( - (target) => target.cellId === config.replacementTargetCellId - ) - if (!failed || !replacement) throw new Error('supersession topology is incomplete') - const sourceStatus = await adminPost(config.directorOrigin, '/v1/admin/cell-status', { - v: 1, - cellId: source.cellId - }) - if ( - sourceStatus.status?.cellUrl !== source.origin || - (selector - ? selectorCellState(selector, source.cellId) !== 'existing-only' - : sourceStatus.status.enabled) - ) { - throw new Error('supersession requires retained disabled source topology') - } - const failedMig = sourceMig(config, deps, failed) - const failedStatus = await inspectFenceCandidate( - config, - deps, - adminPost, - failed, - failedMig.mig - ) - if ( - selector - ? selectorCellState(selector, failed.cellId) !== 'existing-only' - : failedStatus.enabled - ) { - throw new Error( - selector - ? 'failed target admission must be existing-only' - : 'failed target admission must be disabled' - ) - } - const replacementStatus = await inspectDirectorObservedCell( - config, - deps, - adminPost, - replacement - ) - const migrationStatus = await pairStatus(config, adminPost, failed.cellId, false) - if ( - !Number.isSafeInteger(migrationStatus.targetRegistered) || - migrationStatus.targetRegistered < 1 - ) { - throw new Error('failed target has no registered migrations to supersede') - } - const failedConnections = failedStatus.runtime?.observedRequests - if (!Number.isSafeInteger(failedConnections) || failedConnections < 0) { - throw new Error('failed target has no exact runtime connection snapshot') - } - const replacementConnections = runtimeConnections( - replacementStatus.process, - replacement.cellId - ) - const projectedConnections = - replacementConnections + - Math.max(failedConnections, migrationStatus.targetRegistered) - if (projectedConnections >= targetConnectionCeiling(config, replacement)) { - throw new Error('replacement target lacks conservative connection headroom') - } - const cellIncarnation = runtimeIncarnation(failedStatus, failed.cellId) - await runTerraformManagedFence( - config, - deps, - adminPost, - failed, - cellIncarnation, - Number(failedMig.mig.targetSize) === 0, - async () => { - const latestMig = sourceMig(config, deps, failed) - const latestStatus = await inspectFenceCandidate( - config, - deps, - adminPost, - failed, - latestMig.mig - ) - if ( - selector - ? selectorCellState(selector, failed.cellId) !== 'existing-only' - : latestStatus.enabled - ) { - throw new Error('failed target admission changed before apply') - } - if (runtimeIncarnation(latestStatus, failed.cellId) !== cellIncarnation) { - throw new Error('failed target incarnation changed before apply') - } - const latestMigration = await pairStatus(config, adminPost, failed.cellId, false) - if (latestMigration.targetRegistered < 1) { - throw new Error('failed target migrations changed before apply') - } - await inspectDirectorObservedCell(config, deps, adminPost, replacement) - }, - async (expectedIncarnation) => { - const actualIncarnation = await waitForFence(config, deps, adminPost, failed) - if (actualIncarnation !== expectedIncarnation) { - throw new Error('failed target incarnation changed') - } - const finalFailedMig = sourceMig(config, deps, failed) - if (finalFailedMig.instances.length !== 0) { - throw new Error('failed target fence still has an instance') - } - await inspectFencedSource(config, deps, adminPost, failed, finalFailedMig.mig) - } - ) - if ( - selector && - selectorCellState(selector, replacement.cellId) !== 'migration-only' - ) { - throw new Error('replacement target admission must be migration-only') - } - if (!selector && !replacementStatus.enabled) { - await setCellState(config, adminPost, replacement.cellId, true) - } - let superseded = 0 - while (true) { - const result = await adminPost( - config.directorOrigin, - '/v1/admin/migration-supersede-cell', - { - v: 1, - sourceCellId: source.cellId, - currentTargetCellId: failed.cellId, - replacementTargetCellId: replacement.cellId, - limit: config.batchSize, - confirmation: 'SUPERSEDE_REGISTERED_CELL_MIGRATIONS' - } - ) - if ( - !Number.isSafeInteger(result.superseded) || - result.superseded < 0 || - result.superseded > config.batchSize - ) { - throw new Error('invalid registered supersession result') - } - superseded += result.superseded - if (result.superseded === 0) break - } - const remaining = await pairStatus(config, adminPost, failed.cellId, false) - if (remaining.targetRegistered !== 0) { - throw new Error('registered target supersession did not reconcile') - } - if (superseded !== migrationStatus.targetRegistered) { - throw new Error('registered target supersession count changed') - } - deps.emit({ - event: 'registered_target_superseded', - sourceCellId: source.cellId, - failedTargetCellId: failed.cellId, - replacementTargetCellId: replacement.cellId, - superseded, - remainingUnregistered: remaining.inProgress - }) -} - -export async function runMultiTargetDeployment(config, overrides = {}) { - const deps = { - commandJson: overrides.commandJson ?? defaultCommandJson, - command: overrides.command ?? defaultCommand, - commandResult: overrides.commandResult ?? defaultCommandResult, - terraform: overrides.terraform, - terraformFenceApply: overrides.terraformFenceApply ?? runTerraformFenceApply, - terraformFenceAdopt: - overrides.terraformFenceAdopt ?? adoptLegacyTerraformFence, - terraformFenceResume: overrides.terraformFenceResume ?? resumeTerraformFence, - terraformFenceRecoverCompleted: - overrides.terraformFenceRecoverCompleted ?? - recoverSupersededCompletedTerraformFence, - terraformFenceAbort: overrides.terraformFenceAbort ?? abortTerraformFenceBeforeApply, - terraformFenceSupersede: - overrides.terraformFenceSupersede ?? abortSupersededTerraformFenceBeforeUpload, - identityToken: overrides.identityToken ?? defaultIdentityToken, - mutationIdentityToken: - overrides.mutationIdentityToken ?? - (() => suppliedFenceMutationIdentityToken()), - fetch: overrides.fetch ?? fetch, - emit: overrides.emit ?? ((event) => process.stdout.write(`${JSON.stringify(event)}\n`)), - now: overrides.now ?? Date.now, - wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))), - random: overrides.random ?? Math.random, - resolve4: overrides.resolve4 ?? resolve4 - } - const topology = JSON.parse(readFileSync(config.topologyFile, 'utf8')) - const { source, targets } = selectMultiTargetDeployments( - topology, - config.sourceCellId, - config.targetCellIds - ) - const token = deps.identityToken(config.adminAudience) - const mutationToken = - ['fence-source', 'abort-fence-source', 'supersede-target'].includes(config.mode) - ? deps.mutationIdentityToken(config.adminAudience) - : null - if ( - ['fence-source', 'abort-fence-source', 'supersede-target'].includes(config.mode) && - mutationToken === null - ) { - throw new Error('Terraform fence mode requires a broker mutation identity token') - } - const adminPost = createAdminPost( - config, - deps, - (path) => (FENCE_BROKER_MUTATION_ROUTES.has(path) ? mutationToken : token) - ) - if (config.mode === 'abort-fence-source') { - await abortTerraformManagedFence(config, deps, adminPost, source) - return - } - const selectorPost = async (path, body) => - await adminPost(config.directorOrigin, path, body) - const selectorInspection = await inspectAdmissionSelector(selectorPost) - const selectorActive = selectorInspection.selector.generation > 0 - if (config.mode === 'cutover-admission') { - const membership = cutoverMembership(topology, config) - if ( - selectorActive && - JSON.stringify(selectorInspection.selector.membership) !== JSON.stringify(membership) - ) { - throw new Error('selector boundary is already active with different membership') - } - await inspectCutoverCells(topology, config, deps, adminPost, membership) - pruneIncompatibleDirectorRevisions(config, deps) - const director = verifySelectorCompatibleDirector(config, deps) - await inspectCutoverCells(topology, config, deps, adminPost, membership) - const result = await applyExactAdmissionSelector(selectorPost, membership, { - requireBoundary: false, - attemptId: config.selectorAttemptId, - expectedCurrentSelector: selectorInspection.selector - }) - deps.emit({ - event: 'admission_selector_cutover', - generation: result.selector.generation, - membership: result.selector.membership, - ...director - }) - return - } - if (config.mode === 'add-migration-cells') { - if (!selectorActive) throw new Error('admission selector boundary is not active') - pruneIncompatibleDirectorRevisions(config, deps) - const director = verifySelectorCompatibleDirector(config, deps) - if ( - targets.some( - (target) => - target.connectionHardCap === undefined || - target.connectionUnobservedBound === undefined - ) - ) { - throw new Error('migration cells require reviewed connection capacity') - } - const result = await addExactMigrationCells( - selectorPost, - { - attemptId: config.selectorAttemptId, - cells: targets.map((target) => ({ - cellId: target.cellId, - cellUrl: target.origin, - region: target.region, - capacityRequests: target.capacityRequests, - connectionHardCap: target.connectionHardCap, - connectionUnobservedBound: target.connectionUnobservedBound - })) - }, - { expectedCurrentSelector: selectorInspection.selector } - ) - deps.emit({ - event: 'migration_cells_added', - generation: result.selector.generation, - membership: result.selector.membership, - cellIds: targets.map(({ cellId }) => cellId), - ...director - }) - return - } - if (config.mode === 'promote-general-cell') { - if (!selectorActive) throw new Error('admission selector boundary is not active') - const [promoted] = targets - if ( - !promoted || - selectorCellState(selectorInspection.selector, promoted.cellId) !== 'migration-only' - ) { - throw new Error('promoted cell admission must be migration-only') - } - const director = verifyActiveSelectorDirector(config, deps, promoted.cellId) - await inspectGeneralPromotionTarget(config, deps, adminPost, promoted) - const result = await applyExactAdmissionSelector( - selectorPost, - membershipWithStates(selectorInspection.selector, { - [promoted.cellId]: 'general' - }), - { - attemptId: config.selectorAttemptId, - expectedCurrentSelector: selectorInspection.selector - } - ) - deps.emit({ - event: 'migration_cell_promoted_general', - generation: result.selector.generation, - membership: result.selector.membership, - cellId: promoted.cellId, - ...director - }) - return - } - if (config.mode === 'retire-migration-cell') { - if (!selectorActive) throw new Error('admission selector boundary is not active') - const [retiring] = targets - if ( - !retiring || - selectorCellState(selectorInspection.selector, retiring.cellId) !== 'migration-only' - ) { - throw new Error('retired cell admission must be migration-only') - } - pruneIncompatibleDirectorRevisions(config, deps) - const director = verifySelectorCompatibleDirector(config, deps) - const result = await applyExactAdmissionSelector( - selectorPost, - membershipWithStates(selectorInspection.selector, { - [retiring.cellId]: 'existing-only' - }), - { - attemptId: config.selectorAttemptId, - expectedCurrentSelector: selectorInspection.selector - } - ) - deps.emit({ - event: 'migration_cell_retired', - generation: result.selector.generation, - membership: result.selector.membership, - cellId: retiring.cellId, - ...director - }) - return - } - if (config.mode === 'supersede-target') { - await runTargetSupersession( - config, - deps, - adminPost, - source, - targets, - selectorActive ? selectorInspection.selector : null - ) - return - } - const sourceFence = config.mode === 'fence-source' ? sourceMig(config, deps, source) : null - if ( - sourceFence && - ![0, 1].includes(Number(sourceFence.mig.targetSize)) - ) { - throw new Error('source MIG must be fixed-one or already fenced') - } - const fencedResume = sourceFence && Number(sourceFence.mig.targetSize) === 0 ? sourceFence : null - const { sourceProcess, sourceStatus, plannedTargets, sourceAlreadyFenced } = await preflight( - config, - deps, - adminPost, - source, - targets, - fencedResume, - selectorActive ? selectorInspection.selector : null - ) - if (config.mode === 'audit' || config.mode === 'preflight') return - if (config.mode === 'fence-source') { - await fenceSource( - config, - deps, - adminPost, - source, - targets, - sourceProcess, - sourceStatus, - sourceAlreadyFenced - ) - return - } - if (config.mode === 'recover-forward') { - const invalidSelectorAdmission = - selectorActive && - (selectorCellState(selectorInspection.selector, source.cellId) !== 'existing-only' || - plannedTargets.some( - (target) => - selectorCellState(selectorInspection.selector, target.cellId) !== - 'migration-only' - )) - if ( - invalidSelectorAdmission || - (!selectorActive && - (sourceStatus.enabled || plannedTargets.some((target) => !target.status.enabled))) - ) { - throw new Error( - selectorActive - ? 'forward recovery requires existing-only source and migration-only targets' - : 'forward recovery requires disabled source and enabled targets' - ) - } - let coveredSourceConnections = runtimeConnections(sourceProcess, source.cellId) - let recoveryTargets = plannedTargets - for (let pass = 1; pass <= RECOVERY_CATCH_UP_PASSES; pass++) { - await publishMigrations(config, deps, adminPost, recoveryTargets) - if ( - await assertRecoveryPreDrain( - config, - deps, - adminPost, - source, - targets, - selectorPost, - selectorActive ? selectorInspection.selector : null - ) - ) { - break - } - if (pass === RECOVERY_CATCH_UP_PASSES) { - throw new Error('source assignments did not quiesce within bounded recovery catch-up') - } - const catchUp = await preflight( - config, - deps, - adminPost, - source, - targets, - null, - selectorActive ? selectorInspection.selector : null, - coveredSourceConnections - ) - coveredSourceConnections = Math.max( - coveredSourceConnections, - runtimeConnections(catchUp.sourceProcess, source.cellId) - ) - recoveryTargets = catchUp.plannedTargets - } - if (!sourceStatus.draining) { - const activeSourceTransports = - sourceProcess.controls + sourceProcess.splices + sourceProcess.pendingSplices - if (activeSourceTransports > 0) { - const recovery = await adminPost( - config.directorOrigin, - '/v1/admin/drain-attempt-recover-forward', - { - v: 1, - cellId: source.cellId, - cellIncarnation: runtimeIncarnation(sourceStatus, source.cellId), - confirmation: 'RECOVER_LEGACY_DRAIN' - } - ) - await waitForRecoveryTargetOwnership(config, deps, adminPost, targets) - if (recovery.preparedAttempt) { - const attempt = recovery.preparedAttempt - if ( - attempt.state !== 'prepared' || - attempt.cellId !== source.cellId || - attempt.cellIncarnation !== runtimeIncarnation(sourceStatus, source.cellId) || - attempt.plannedGraceMs !== 120_000 || - typeof attempt.attemptId !== 'string' || - typeof attempt.traceValue !== 'string' - ) { - throw new Error('prepared drain recovery state is invalid') - } - const sending = await adminPost( - config.directorOrigin, - '/v1/admin/drain-attempt-send', - { - v: 1, - attemptId: attempt.attemptId, - cellId: source.cellId, - cellIncarnation: attempt.cellIncarnation - } - ) - if ( - sending.attempt?.state !== 'send-may-have-started' || - sending.attempt.shouldSend !== true || - !Number.isSafeInteger(sending.attempt.sendPermitExpiresAt) || - deps.now() >= sending.attempt.sendPermitExpiresAt - ) { - throw new Error('prepared drain recovery send permit unavailable') - } - const receipt = await drainSource( - config, - deps, - token, - source, - attempt.plannedGraceMs, - attempt.traceValue - ) - await adminPost(config.directorOrigin, '/v1/admin/drain-attempt-receipt', { - v: 1, - attemptId: attempt.attemptId, - cellId: source.cellId, - cellIncarnation: attempt.cellIncarnation, - traceValue: attempt.traceValue, - ...receipt - }) - deps.emit({ - event: 'source_prepared_drain_recovered', - sourceCellId: source.cellId - }) - } else if (recovery.shouldSend === true) { - await drainSource(config, deps, token, source, 0) - deps.emit({ event: 'source_recovery_drain_accepted', sourceCellId: source.cellId }) - } else { - const latestSource = await inspectCell(config, deps, adminPost, source) - const expectedIncarnation = runtimeIncarnation(sourceStatus, source.cellId) - if (runtimeIncarnation(latestSource, source.cellId) !== expectedIncarnation) { - throw new Error('source incarnation changed before recovery drain reissue') - } - if (latestSource.draining) { - deps.emit({ - event: 'source_recovery_drain_already_applied', - sourceCellId: source.cellId - }) - } else { - const latestStatuses = await allPairStatuses(config, adminPost, false) - const latestTotals = statusTotals(latestStatuses) - assertLeaseGate(latestStatuses, config.minimumLeaseRemainingMs) - if ( - !hasDurableTargetOwnership(latestTotals) && - !boundedUnregisteredMigrations( - latestTotals, - config.unobservedConnectionBound - ) - ) { - throw new Error('recovery target ownership changed before drain reissue') - } - await drainSource(config, deps, token, source, 0) - deps.emit({ - event: 'source_recovery_drain_reissued_after_non_delivery', - sourceCellId: source.cellId, - ...latestTotals - }) - } - } - } else { - deps.emit({ - event: 'source_recovery_drain_not_needed', - sourceCellId: source.cellId - }) - } - } - await waitForMultiStatus(config, deps, adminPost, targets, false, true) - await waitForRecoveredSourceZero( - config, - deps, - adminPost, - source, - runtimeIncarnation(sourceStatus, source.cellId) - ) - await waitForMultiStatus(config, deps, adminPost, targets, true, true) - deps.emit({ event: 'multi_target_complete', sourceCellId: source.cellId }) - return - } - if ( - selectorActive && - targets.some( - (target) => - selectorCellState(selectorInspection.selector, target.cellId) !== 'migration-only' - ) - ) { - throw new Error('evacuation targets must be migration-only') - } - await runEvacuation( - config, - deps, - adminPost, - token, - source, - targets, - plannedTargets, - sourceStatus, - selectorActive, - selectorPost, - selectorInspection.selector - ) -} - -export async function main(argv = process.argv.slice(2)) { - await runMultiTargetDeployment(parseMultiTargetArguments(argv)) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/deploy-relay-gce-multi-target.test.mjs b/cloud/dev/scripts/deploy-relay-gce-multi-target.test.mjs deleted file mode 100644 index 1ad4e240d56..00000000000 --- a/cloud/dev/scripts/deploy-relay-gce-multi-target.test.mjs +++ /dev/null @@ -1,3320 +0,0 @@ -import assert from 'node:assert/strict' -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { test } from 'node:test' -import { - allocateTargetQuotas, - assertCutoverCellReady, - cutoverMembership, - parseMultiTargetArguments, - pruneIncompatibleDirectorRevisions, - runMultiTargetDeployment, - sameBackendServiceResource, - selectMultiTargetDeployments, - verifySelectorCompatibleDirector -} from './deploy-relay-gce-multi-target.mjs' - -const runtimeServiceAccount = 'orca-relay@example.iam.gserviceaccount.com' -const digest = (value) => `sha256:${value.repeat(64)}` - -test('matches only exact canonical backend-service resource forms', () => { - const resource = 'projects/onorca-cloud/global/backendServices/orca-cloud-relay-gce-c11' - assert.equal( - sameBackendServiceResource( - `https://www.googleapis.com/compute/v1/${resource}`, - resource - ), - true - ) - assert.equal( - sameBackendServiceResource( - `https://www.googleapis.com/compute/v1/${resource}`, - resource.replace('c11', 'c12') - ), - false - ) - assert.equal( - sameBackendServiceResource( - `https://compute.example/compute/v1/${resource}`, - resource - ), - false - ) -}) - -test('requires only compatible active and rollback director revisions', () => { - let rollbackInventory = ['c1'] - const revision = (minimum = 1, inventory = ['c1']) => ({ - metadata: { - annotations: { 'autoscaling.knative.dev/minScale': String(minimum) } - }, - spec: { - containers: [ - { - image: 'registry.example/relay@sha256:abc', - env: [ - { name: 'ORCA_RELAY_ROLE', value: 'director' }, - { name: 'ORCA_RELAY_ADMISSION_SELECTOR_VERSION', value: '3' }, - { - name: 'ORCA_RELAY_CELLS_JSON', - value: JSON.stringify(inventory.map((id) => ({ id }))) - } - ] - } - ] - } - }) - let names = ['active', 'rollback'] - const deps = { - command: (args) => { - const revision = args[3] - names = names.filter((name) => name !== revision) - }, - commandJson: (args) => { - if (args.includes('services')) { - return { - status: { - traffic: [ - { percent: 100, revisionName: 'active' }, - { tag: 'selector-rollback', revisionName: 'rollback' } - ] - } - } - } - if (args.includes('list')) { - return names.map((name) => ({ metadata: { name } })) - } - return args.includes('rollback') - ? revision(0, rollbackInventory) - : revision(1) - } - } - const config = { - project: 'project', - directorRegion: 'region', - directorService: 'service', - directorMinimumInstances: 1 - } - assert.deepEqual(verifySelectorCompatibleDirector(config, deps), { - activeRevision: 'active', - rollbackRevision: 'rollback' - }) - rollbackInventory = ['c1', 'c2'] - assert.throws( - () => verifySelectorCompatibleDirector(config, deps), - /director inventories do not match/ - ) - assert.throws( - () => pruneIncompatibleDirectorRevisions(config, deps), - /director compatibility pair failed/ - ) - rollbackInventory = ['c1'] - names = ['active', 'rollback', 'legacy'] - assert.throws( - () => verifySelectorCompatibleDirector(config, deps), - /old or pre-selector director revisions/ - ) - pruneIncompatibleDirectorRevisions(config, deps) - assert.deepEqual(names, ['active', 'rollback']) - assert.deepEqual(verifySelectorCompatibleDirector(config, deps), { - activeRevision: 'active', - rollbackRevision: 'rollback' - }) - names = ['active', 'rollback', null] - assert.throws( - () => verifySelectorCompatibleDirector(config, deps), - /old or pre-selector director revisions/ - ) - assert.throws( - () => pruneIncompatibleDirectorRevisions(config, deps), - /unnamed revision/ - ) -}) - -test('requires the active director to meet the configured floor', () => { - let activeMinimum = 5 - let rollbackMinimum = 0 - const revision = (minimum) => ({ - metadata: { - annotations: { 'autoscaling.knative.dev/minScale': String(minimum) } - }, - spec: { - containers: [ - { - image: 'registry.example/relay@sha256:abc', - env: [ - { name: 'ORCA_RELAY_ROLE', value: 'director' }, - { name: 'ORCA_RELAY_ADMISSION_SELECTOR_VERSION', value: '3' }, - { - name: 'ORCA_RELAY_CELLS_JSON', - value: JSON.stringify([{ id: 'c1' }]) - } - ] - } - ] - } - }) - const deps = { - command: () => {}, - commandJson: (args) => { - if (args.includes('services')) { - return { - status: { - traffic: [ - { percent: 100, revisionName: 'active' }, - { tag: 'selector-rollback', revisionName: 'rollback' } - ] - } - } - } - if (args.includes('list')) { - return ['active', 'rollback'].map((name) => ({ metadata: { name } })) - } - return args.includes('rollback') ? revision(rollbackMinimum) : revision(activeMinimum) - } - } - const config = { - project: 'project', - directorRegion: 'region', - directorService: 'service', - directorMinimumInstances: 5 - } - - assert.deepEqual(verifySelectorCompatibleDirector(config, deps), { - activeRevision: 'active', - rollbackRevision: 'rollback' - }) - pruneIncompatibleDirectorRevisions(config, deps) - - activeMinimum = 4 - assert.throws( - () => verifySelectorCompatibleDirector(config, deps), - /active selector revision is below the required floor/ - ) - assert.throws( - () => pruneIncompatibleDirectorRevisions(config, deps), - /director compatibility pair failed/ - ) - - // Every comparison against NaN is false, so negating the minimum check rejects it. - activeMinimum = 'warm' - assert.throws( - () => verifySelectorCompatibleDirector(config, deps), - /active selector revision is below the required floor/ - ) - assert.throws( - () => pruneIncompatibleDirectorRevisions(config, deps), - /director compatibility pair failed/ - ) - - activeMinimum = 5 - rollbackMinimum = 1 - assert.throws( - () => verifySelectorCompatibleDirector(config, deps), - /selector rollback revision is not scale-to-zero/ - ) - assert.throws( - () => pruneIncompatibleDirectorRevisions(config, deps), - /director compatibility pair failed/ - ) -}) - -function topology() { - const cell = (id, hostname, initiallyEnabled) => ({ - origin: `https://${hostname}.relay.example.com`, - zone: `us-central1-${hostname}`, - mig_name: `relay-${hostname}`, - instance_group: `https://compute.example/instanceGroups/relay-${hostname}`, - backend_name: `relay-${hostname}`, - backend_id: `https://compute.example/backendServices/relay-${hostname}`, - url_map_name: 'orca-relay', - generation_identity: `https://compute.example/instanceTemplates/relay-${hostname}-abc`, - image: `us-central1-docker.pkg.dev/project/repo/relay@${digest(id)}`, - capacity_requests: 4_000, - connection_hard_cap: 600, - connection_unobserved_bound: 40, - initially_enabled: initiallyEnabled, - fenced: false, - desired_target_size: 1, - target_size: 1 - }) - return { - source: cell('a', 'a', true), - target1: cell('b', 'b', false), - target2: cell('c', 'c', false), - general: cell('d', 'd', true) - } -} - -function withTopology(operation, value = topology()) { - const directory = mkdtempSync(join(tmpdir(), 'relay-gce-multi-')) - const file = join(directory, 'topology.json') - writeFileSync(file, JSON.stringify(value)) - return Promise.resolve(operation(file)).finally(() => rmSync(directory, { recursive: true })) -} - -function config(topologyFile, mode = 'preflight') { - const selectorMutation = [ - 'cutover-admission', - 'add-migration-cells', - 'promote-general-cell', - 'retire-migration-cell' - ].includes(mode) - return { - project: 'test-project', - directorOrigin: 'https://relay.example.com', - adminAudience: 'https://relay.example.com/v1/admin/drain', - topologyFile, - sourceCellId: 'source', - targetCellIds: ['promote-general-cell', 'retire-migration-cell'].includes(mode) - ? ['target1'] - : ['target1', 'target2'], - generalCellIds: mode === 'cutover-admission' ? ['general'] : [], - directorRegion: selectorMutation ? 'us-central1' : undefined, - directorService: selectorMutation ? 'relay-director' : undefined, - directorMinimumInstances: selectorMutation ? 1 : undefined, - selectorAttemptId: - mode === 'add-migration-cells' - ? 'add_cells_test' - : mode === 'promote-general-cell' - ? 'promote_cell_test' - : mode === 'retire-migration-cell' - ? 'retire_cell_test' - : undefined, - unobservedConnectionBound: - selectorMutation || mode === 'fence-source' ? 40 : undefined, - failedTargetCellId: mode === 'supersede-target' ? 'target1' : undefined, - replacementTargetCellId: mode === 'supersede-target' ? 'target2' : undefined, - runtimeServiceAccount, - environment: 'production', - fenceCommit: 'a'.repeat(40), - terraformDir: 'infra/terraform', - terraformVarFile: 'environments/production.tfvars', - mode, - batchSize: 2, - connectionCeiling: mode === 'fence-source' ? 600 : 6, - minimumLeaseRemainingMs: 600_000, - drainGraceMs: 120_000, - pollIntervalMs: 1, - timeoutMs: 1_000 - } -} - -function response(body, status = 200) { - return new Response(JSON.stringify(body), { - status, - headers: { 'content-type': 'application/json' } - }) -} - -function harness({ - leaseRemainingMs = 900_000, - refreshedLeaseRemainingMs = leaseRemainingMs, - failAfterDrain = false, - failEvacuationStatus = false, - fence = false, - allowPreFenceCompletion = false, - loseResizeResponse = false, - resumeNeedsPreApply = false, - loseDrainSendResponse = false, - loseDrainBeforeAccept = false, - preparedDrainAttempt = false, - recoverableDrain = false, - recoveryAlreadyAttempted = false, - preexistingRegisteredMigrations = 0, - supersede = false, - cleanupBeforeSupersession = false, - offlineTargetMigrations = 0, - offlineTargetMigrationsByTarget = {}, - unregisteredTargetMigrations = 0, - registeredSourceActive = 0, - recoveryRegistrationDelayReads = 0, - failedTargetEnabled = false, - replacementHeartbeatFresh = true, - replacementRuntimeCellUrl, - legacySource = false, - legacyMetricAgeMs = 0, - selectorMembership = null, - capacitySourceAssignments = [], - capacityRequiredTargetUnits = [], - headroomFailureTarget = null, - sourceAssignments = fence ? 0 : 4, - sourceRequiredTargetUnits = fence ? 0 : 8, - sourceObservedRequests = null, - sourceOutgoingMigrations = null, - existingFenceAttempt = false, - alreadyFencedSource = false, - supersededFenceAttempt = false, - completedFenceAttempt = false, - activeDirectorMinimum = 1, - misroutedCell = null, - routeOverrideCell = null, - frontendMisbound = false, - templateHardCap = 600, - templateUnobservedBound = 40, - directorHardCap = templateHardCap, - directorUnobservedBound = templateUnobservedBound, - directorCapacityOverrides = {}, - liveMigTemplateOverrides = {}, - topologyValue = topology() -} = {}) { - const directorCapacity = (cellId) => { - const hardCap = directorCapacityOverrides[cellId]?.hardCap ?? directorHardCap - const unobservedBound = - directorCapacityOverrides[cellId]?.unobservedBound ?? directorUnobservedBound - return { - hardCap, - controlRebindReserve: 100, - ordinaryConnectionLimit: hardCap - 100, - unobservedBound, - normalAdmissionPause: hardCap - 100 - unobservedBound - } - } - const cells = { - source: { - enabled: !fence && !supersede, - assignments: 4, - activityLeases: fence ? 0 : 4, - totalConnections: fence ? 1 : 5, - controls: fence ? 0 : 4, - observedRequests: sourceObservedRequests ?? (fence ? 0 : 4), - heartbeatFresh: !alreadyFencedSource, - draining: fence - }, - target1: { - enabled: failedTargetEnabled || (fence && !supersede), - assignments: fence ? 2 : 0, - activityLeases: fence ? 2 : 0, - totalConnections: fence ? 2 : 0, - controls: fence ? 2 : 0, - heartbeatFresh: !supersede, - draining: false - }, - target2: { - enabled: fence && !supersede, - assignments: fence ? 2 : 0, - activityLeases: fence ? 2 : 0, - totalConnections: fence ? 2 : 0, - controls: fence ? 2 : 0, - heartbeatFresh: replacementHeartbeatFresh, - draining: false - }, - general: { - enabled: true, - assignments: 0, - activityLeases: 0, - totalConnections: 0, - controls: 0, - heartbeatFresh: true, - draining: false - } - } - for (const cellId of Object.keys(topologyValue)) { - cells[cellId] ??= { - enabled: fence && !supersede, - assignments: fence ? 2 : 0, - activityLeases: fence ? 2 : 0, - totalConnections: fence ? 2 : 0, - controls: fence ? 2 : 0, - heartbeatFresh: true, - draining: false - } - } - const events = [] - const stateChanges = [] - const batches = [] - const publishedMigrations = new Map() - const runtimeInspections = new Map() - let drained = fence - let remainingSourceAssignments = sourceAssignments - let remainingRequiredTargetUnits = sourceRequiredTargetUnits - const migSizes = Object.fromEntries( - Object.keys(topologyValue).map((cellId) => [ - cellId, - cellId === 'source' && alreadyFencedSource - ? 0 - : cellId === 'target1' && completedFenceAttempt - ? 0 - : 1 - ]) - ) - const instanceCounts = { ...migSizes } - let supersessionRemaining = supersede ? 2 : 0 - let remainingOfflineTargetMigrations = offlineTargetMigrations - const initialOfflineTargetMigrationsByTarget = new Map( - Object.entries(offlineTargetMigrationsByTarget) - ) - const remainingOfflineTargetMigrationsByTarget = new Map( - initialOfflineTargetMigrationsByTarget - ) - const targetMigrationTotal = (cellId) => - initialOfflineTargetMigrationsByTarget.has(cellId) - ? initialOfflineTargetMigrationsByTarget.get(cellId) - : 2 - const remainingOfflineTargetMigrationCount = (cellId) => - remainingOfflineTargetMigrationsByTarget.has(cellId) - ? remainingOfflineTargetMigrationsByTarget.get(cellId) - : remainingOfflineTargetMigrations - const clearOfflineTargetMigrations = (cellId) => { - if (remainingOfflineTargetMigrationsByTarget.has(cellId)) { - remainingOfflineTargetMigrationsByTarget.set(cellId, 0) - return - } - remainingOfflineTargetMigrations = 0 - } - let drainReceiptRecorded = recoverableDrain - let drainSendStarted = false - let recoveryDrainPrepared = recoveryAlreadyAttempted - let recoveryLeasesRefreshed = false - const recoveryRegistrationReads = new Map() - let headroomFailureInjected = false - let currentLeaseRemainingMs = leaseRemainingMs - const drainGraces = [] - const timeline = [] - let fencedCompletions = 0 - let resizeAttempts = 0 - let capacityReads = 0 - let addedCells = [] - let fenceAttested = false - let fenceAttempt = existingFenceAttempt || supersededFenceAttempt || completedFenceAttempt - ? { - attemptId: '44444444-4444-4444-8444-444444444444', - environment: 'production', - cellId: 'source', - cellIncarnation: '11111111-1111-4111-8111-111111111111', - migName: topologyValue.source.mig_name, - instanceGroup: topologyValue.source.instance_group, - generationIdentity: topologyValue.source.generation_identity, - fenceCommit: (supersededFenceAttempt || completedFenceAttempt ? 'b' : 'a').repeat(40), - planSha256: 'd'.repeat(64), - planObjectName: - 'terraform/state/relay-fence-plans/production/44444444-4444-4444-8444-444444444444.tfplan', - ...(supersededFenceAttempt && !completedFenceAttempt - ? {} - : { planObjectGeneration: '123456789' }), - varFileSha256: 'e'.repeat(64), - terraformStateLineage: '55555555-5555-4555-8555-555555555555', - terraformStateSerial: 7, - terraformStateObjectGeneration: '987654321', - terraformStateObjectSha256: 'f'.repeat(64), - requestReason: - 'orca-relay-fence/44444444-4444-4444-8444-444444444444', - createdAt: Date.now(), - expiresAt: Date.now() + 3_600_000, - ...(completedFenceAttempt - ? { - applyStartedAt: 101, - applyInvocations: [ - { - invocationId: '66666666-6666-4666-8666-666666666666', - requestReason: - 'orca-relay-fence/44444444-4444-4444-8444-444444444444/66666666-6666-4666-8666-666666666666', - startedAt: 101 - } - ] - } - : {}) - } - : null - let selector = selectorMembership - ? { generation: 1, attemptId: 'existing-selector', membership: selectorMembership } - : null - const selectorIntents = new Map() - const cellForOrigin = (origin) => { - const entry = Object.entries(topologyValue).find(([, value]) => value.origin === origin) - return entry?.[0] - } - const cellForMig = (name) => { - const entry = Object.entries(topologyValue).find(([, value]) => value.mig_name === name) - return entry?.[0] - } - const commandJson = (args) => { - if (args[0] === 'run') { - if (args.includes('services')) { - return { - status: { - traffic: [ - { percent: 100, revisionName: 'active' }, - { percent: 0, tag: 'selector-rollback', revisionName: 'rollback' } - ] - } - } - } - if (args.includes('list')) { - return ['active', 'rollback'].map((name) => ({ metadata: { name } })) - } - const revisionName = args[3] - return { - metadata: { - annotations: { - 'autoscaling.knative.dev/minScale': - revisionName === 'rollback' ? '0' : String(activeDirectorMinimum) - } - }, - spec: { - containers: [ - { - image: 'registry.example/relay@sha256:abc', - env: [ - { name: 'ORCA_RELAY_ROLE', value: 'director' }, - { name: 'ORCA_RELAY_ADMISSION_SELECTOR_VERSION', value: '3' }, - { - name: 'ORCA_RELAY_CELLS_JSON', - value: JSON.stringify( - Object.keys(topologyValue).map((id) => ({ id })) - ) - } - ] - } - ] - } - } - } - if (args.includes('instance-templates')) { - const templateName = args[args.indexOf('describe') + 1] - const expected = Object.values(topologyValue).find((cell) => - cell.generation_identity.endsWith(`/instanceTemplates/${templateName}`) - ) - return { - selfLink: expected.generation_identity, - properties: { - metadata: { - items: [ - { - key: 'startup-script', - value: [ - `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${expected.image.split('@')[1]}'`, - ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${templateHardCap}'`, - ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${templateUnobservedBound}'`, - `docker pull '${expected.image}'`, - `docker run '${expected.image}'` - ].join('\n') - } - ] - } - } - } - } - if (args[0] === 'logging') { - return [ - { - timestamp: new Date(Date.now() - legacyMetricAgeMs).toISOString(), - jsonPayload: { - totalConnections: cells.source.totalConnections, - preAuthConnections: 0, - controls: cells.source.controls, - splices: 0, - pendingSplices: 0, - queuedBytes: 0 - } - } - ] - } - const describeIndex = args.indexOf('describe') - const listIndex = args.indexOf('list-instances') - const name = args[describeIndex >= 0 ? describeIndex + 1 : listIndex + 1] - const migCell = cellForMig(name) - if (args.includes('list-instances')) { - return instanceCounts[migCell] === 0 - ? [] - : [ - { - instance: `https://compute.example/instances/${name}-vm`, - instanceStatus: 'RUNNING', - currentAction: 'NONE', - version: { - name: 'primary', - instanceTemplate: topologyValue[migCell].generation_identity - } - } - ] - } - if (args.includes('instance-groups')) { - return { - targetSize: migSizes[migCell], - instanceTemplate: - liveMigTemplateOverrides[migCell] ?? topologyValue[migCell].generation_identity, - updatePolicy: { - replacementMethod: 'RECREATE', - maxSurge: { fixed: 0 }, - maxUnavailable: { fixed: 1 } - } - } - } - if (args.includes('instances')) { - return { - networkInterfaces: [{ networkIP: '10.42.0.2' }], - serviceAccounts: [{ email: runtimeServiceAccount }] - } - } - if (args.includes('target-https-proxies')) { - return { - name: 'orca-relay', - selfLink: - 'https://www.googleapis.com/compute/v1/projects/test-project/global/targetHttpsProxies/orca-relay', - urlMap: frontendMisbound - ? 'https://www.googleapis.com/compute/v1/projects/test-project/global/urlMaps/other' - : 'https://www.googleapis.com/compute/v1/projects/test-project/global/urlMaps/orca-relay' - } - } - if (args.includes('forwarding-rules')) { - return { - name: 'orca-relay', - target: - 'https://www.googleapis.com/compute/v1/projects/test-project/global/targetHttpsProxies/orca-relay', - IPAddress: '203.0.113.10', - portRange: '443-443', - loadBalancingScheme: 'EXTERNAL_MANAGED' - } - } - if (args.includes('addresses')) { - return { name: 'orca-relay', address: '203.0.113.10' } - } - if (args.includes('url-maps')) { - return { - name: 'orca-relay', - selfLink: - 'https://www.googleapis.com/compute/v1/projects/test-project/global/urlMaps/orca-relay', - hostRules: Object.entries(topologyValue).map(([cellId, cell]) => ({ - hosts: [new URL(cell.origin).hostname], - pathMatcher: cellId - })), - pathMatchers: Object.entries(topologyValue).map(([cellId, cell]) => ({ - name: cellId, - defaultService: - cellId === misroutedCell ? topologyValue.general.backend_id : cell.backend_id, - ...(cellId === routeOverrideCell - ? { - pathRules: [ - { paths: ['/v1/*'], service: topologyValue.general.backend_id } - ] - } - : {}) - })) - } - } - const id = Object.entries(topologyValue).find(([, value]) => value.backend_name === name)?.[0] - return { - protocol: 'HTTP', - timeoutSec: 86_400, - backends: [{ group: topologyValue[id].instance_group }] - } - } - const fetch = async (url, options = {}) => { - const parsed = new URL(url) - if (parsed.pathname === '/health' || parsed.pathname === '/ready') return response({ ok: true }) - const body = JSON.parse(options.body ?? '{}') - if (parsed.pathname === '/v1/admin/runtime-status') { - const id = cellForOrigin(parsed.origin) - const cell = cells[id] - const capacity = directorCapacity(id) - runtimeInspections.set(id, (runtimeInspections.get(id) ?? 0) + 1) - return response({ - v: 1, - role: 'cell', - cellId: id, - cellUrl: topologyValue[id].origin, - imageDigest: topologyValue[id].image.split('@')[1], - draining: cell.draining, - connectionCapacity: { - ...capacity - }, - runtime: - legacySource && id === 'source' - ? null - : { - totalConnections: cell.totalConnections, - preAuthConnections: 0, - enforcedConnectionUnits: cell.totalConnections, - controls: cell.controls, - splices: 0, - pendingSplices: 0, - queuedBytes: 0 - } - }) - } - if (parsed.pathname === '/v1/admin/cell-status') { - const cell = cells[body.cellId] - const capacity = directorCapacity(body.cellId) - return response({ - v: 1, - status: { - cellId: body.cellId, - cellUrl: topologyValue[body.cellId].origin, - enabled: cell.enabled, - assignments: cell.assignments, - activityLeases: cell.activityLeases, - activityRequestUnits: cell.activityLeases, - reservedRequests: cell.activityLeases, - connectionCapacity: { - ...capacity, - observedConnections: cell.totalConnections, - inFlightConnections: 0, - reservedConnectionUnits: 0, - enforcedConnectionUnits: cell.totalConnections, - pendingControlReservations: 0, - heartbeatFresh: cell.heartbeatFresh - }, - outgoingMigrations: - sourceOutgoingMigrations ?? - (fence - ? Object.keys(topologyValue) - .filter((cellId) => cellId !== 'source' && cellId !== 'general') - .reduce((total, cellId) => total + targetMigrationTotal(cellId), 0) - : 0), - incomingMigrations: 0, - runtime: { - cellUrl: - body.cellId === 'target2' && replacementRuntimeCellUrl - ? replacementRuntimeCellUrl - : topologyValue[body.cellId].origin, - cellIncarnation: '11111111-1111-4111-8111-111111111111', - ready: true, - heartbeatFresh: cell.heartbeatFresh, - observedRequests: cell.observedRequests ?? cell.controls - } - } - }) - } - if (parsed.pathname === '/v1/admin/admission-selector/status') { - selector ??= { - generation: 0, - attemptId: null, - membership: { - existingOnly: Object.keys(cells).filter((cellId) => !cells[cellId].enabled), - migrationOnly: [], - general: Object.keys(cells).filter((cellId) => cells[cellId].enabled) - } - } - return response({ - v: 1, - selector, - intent: body.attemptId ? selectorIntents.get(body.attemptId) ?? null : null - }) - } - if (parsed.pathname === '/v1/admin/admission-selector/apply') { - selector = { - generation: body.expectedGeneration + 1, - attemptId: body.attemptId, - membership: body.membership - } - selectorIntents.set(body.attemptId, { - attemptId: body.attemptId, - expectedGeneration: body.expectedGeneration, - intendedGeneration: selector.generation, - membership: body.membership, - state: 'committed' - }) - return response({ v: 1, changed: true, selector }) - } - if (parsed.pathname === '/v1/admin/admission-selector/add-migration-cells') { - addedCells = body.cells - selector = { - generation: body.expectedGeneration + 1, - attemptId: body.attemptId, - membership: { - ...selector.membership, - migrationOnly: [ - ...selector.membership.migrationOnly, - ...body.cells.map(({ cellId }) => cellId) - ].sort() - } - } - selectorIntents.set(body.attemptId, { - attemptId: body.attemptId, - expectedGeneration: body.expectedGeneration, - intendedGeneration: selector.generation, - membership: selector.membership, - state: 'committed' - }) - return response({ v: 1, changed: true, selector }) - } - if (parsed.pathname === '/v1/admin/evacuation-capacity') { - const read = capacityReads++ - return response({ - v: 1, - sourceAssignments: capacitySourceAssignments[read] ?? remainingSourceAssignments, - requiredTargetUnits: - capacityRequiredTargetUnits[read] ?? remainingRequiredTargetUnits, - availableTargetUnits: 4_000 - }) - } - if (parsed.pathname === '/v1/admin/cell-state') { - cells[body.cellId].enabled = body.enabled - stateChanges.push([body.cellId, body.enabled]) - return response({ ok: true }) - } - if (parsed.pathname === '/v1/admin/drain-attempt-prepare') { - return response({ v: 1, state: 'prepared' }) - } - if (parsed.pathname === '/v1/admin/drain-attempt-send') { - const shouldSend = !drainSendStarted - drainSendStarted = true - if (loseDrainSendResponse) throw new Error('injected_send_transition_response_loss') - return response({ - v: 1, - attempt: { - state: 'send-may-have-started', - shouldSend, - sendPermitExpiresAt: Date.now() + 30_000 - } - }) - } - if (parsed.pathname === '/v1/admin/drain-attempt-receipt') { - drainReceiptRecorded = true - return response({ v: 1, attempt: { state: 'application-receipt' } }) - } - if (parsed.pathname === '/v1/admin/drain-attempt-recover-forward') { - recoveryLeasesRefreshed = true - currentLeaseRemainingMs = refreshedLeaseRemainingMs - if (!drainReceiptRecorded) { - if (preparedDrainAttempt && !drainSendStarted) { - return response({ - v: 1, - shouldSend: false, - retryAfter: Date.now(), - preparedAttempt: { - attemptId: '33333333-3333-4333-8333-333333333333', - cellId: 'source', - cellIncarnation: '11111111-1111-4111-8111-111111111111', - traceValue: '44444444-4444-4444-8444-444444444444', - plannedGraceMs: 120_000, - state: 'prepared' - } - }) - } - return response({ error: 'drain_application_receipt_missing' }, 409) - } - const shouldSend = !recoveryDrainPrepared - recoveryDrainPrepared = true - return response({ v: 1, shouldSend, retryAfter: Date.now() }) - } - if (parsed.pathname === '/v1/admin/evacuate-cell') { - if (body.targetCellId === headroomFailureTarget && !headroomFailureInjected) { - headroomFailureInjected = true - const partiallyStarted = Math.min(1, remainingSourceAssignments) - publishedMigrations.set( - body.targetCellId, - (publishedMigrations.get(body.targetCellId) ?? 0) + partiallyStarted - ) - remainingSourceAssignments -= partiallyStarted - return response({ error: 'relay_connection_headroom_exhausted' }, 409) - } - const started = Math.min(body.limit, remainingSourceAssignments) - batches.push([body.targetCellId, started]) - publishedMigrations.set( - body.targetCellId, - (publishedMigrations.get(body.targetCellId) ?? 0) + started - ) - remainingSourceAssignments -= started - if (remainingSourceAssignments === 0) remainingRequiredTargetUnits = 0 - return response({ v: 1, started }) - } - if (parsed.pathname === '/v1/admin/cell-fence-attest') { - fenceAttested = true - return response({ v: 1, cellId: body.cellId, expiresAt: Date.now() + 300_000 }) - } - if (parsed.pathname === '/v1/admin/cell-fence-adopt-legacy') { - fenceAttested = true - return response({ v: 1, cellId: body.cellId, expiresAt: Date.now() + 300_000 }) - } - if (parsed.pathname === '/v1/admin/cell-fence-commit-legacy-adoption') { - return response({ v: 1, cellId: body.cellId, committed: true }) - } - if (parsed.pathname === '/v1/admin/cell-fence-attempt-prepare') { - fenceAttempt = body - return response({ v: 1, attempt: body }) - } - if (parsed.pathname === '/v1/admin/cell-fence-attempt-start') { - fenceAttempt = { ...body, applyStartedAt: Date.now() } - return response({ - v: 1, - attempt: fenceAttempt, - invocation: { - invocationId: body.invocationId, - requestReason: body.invocationRequestReason, - startedAt: fenceAttempt.applyStartedAt - } - }) - } - if (parsed.pathname === '/v1/admin/cell-fence-attempt-operation') { - fenceAttempt = body - return response({ - v: 1, - attempt: body, - invocation: { - invocationId: body.invocationId, - requestReason: body.invocationRequestReason, - startedAt: Date.now(), - gceOperation: body.gceOperation - } - }) - } - if (parsed.pathname === '/v1/admin/cell-fence-attempt-status') { - return response({ v: 1, attempt: fenceAttempt }) - } - if (parsed.pathname === '/v1/admin/cell-fence-attempt-abort') { - fenceAttempt = { ...body, abortedAt: Date.now() } - return response({ v: 1, attempt: fenceAttempt }) - } - if (parsed.pathname === '/v1/admin/evacuation-status') { - if (failEvacuationStatus) { - return response({ error: 'injected_status_failure' }, 500) - } - const completed = - body.completeReady && (!fence || fenceAttested || allowPreFenceCompletion) - if (completed && fenceAttested) clearOfflineTargetMigrations(body.targetCellId) - const migrationTotal = targetMigrationTotal(body.targetCellId) - const remainingOfflineTargetMigrationsForCell = - remainingOfflineTargetMigrationCount(body.targetCellId) - const supersededTarget = supersede && body.targetCellId === 'target1' - const recoveryRegistrationRead = - recoveryRegistrationReads.get(body.targetCellId) ?? 0 - if (recoveryLeasesRefreshed) { - recoveryRegistrationReads.set(body.targetCellId, recoveryRegistrationRead + 1) - } - const recoveryRegistrationSettled = - recoveryRegistrationRead >= recoveryRegistrationDelayReads - const remainingMigrations = supersededTarget - ? supersessionRemaining - : completed - ? remainingOfflineTargetMigrationsForCell + unregisteredTargetMigrations - : migrationTotal - const registeredMigrations = supersededTarget - ? supersessionRemaining - : drained - ? remainingMigrations - unregisteredTargetMigrations - : Math.min( - remainingMigrations, - preexistingRegisteredMigrations + - (recoveryLeasesRefreshed && recoveryRegistrationSettled - ? publishedMigrations.get(body.targetCellId) ?? 0 - : 0) - ) - const completableMigrations = - drained && !completed - ? migrationTotal - - remainingOfflineTargetMigrationsForCell - - unregisteredTargetMigrations - - registeredSourceActive - : recoveryLeasesRefreshed - ? Math.max( - 0, - registeredMigrations - - remainingOfflineTargetMigrationsForCell - - registeredSourceActive - ) - : 0 - if (completed) { - if (failAfterDrain) return response({ error: 'injected_completion_failure' }, 500) - if (fenceAttested) fencedCompletions++ - cells.source.activityLeases = 0 - for (const targetCellId of Object.keys(cells).filter((id) => id !== 'source')) { - cells[targetCellId].activityLeases = 2 - } - } - timeline.push({ - kind: 'evacuation_status', - targetCellId: body.targetCellId, - completeReady: body.completeReady, - inProgress: remainingMigrations, - registeredTargetInactive: remainingOfflineTargetMigrationsForCell - }) - return response({ - v: 1, - inProgress: remainingMigrations, - oldestExpiresAt: - remainingMigrations === 0 ? null : Date.now() + currentLeaseRemainingMs, - oldestRemainingMs: - remainingMigrations === 0 ? null : currentLeaseRemainingMs, - targetRegistered: registeredMigrations, - registeredSourceActive, - registeredCompletable: completableMigrations, - registeredTargetInactive: remainingOfflineTargetMigrationsForCell, - completed: - completed - ? migrationTotal - - remainingOfflineTargetMigrationsForCell - - unregisteredTargetMigrations - : 0, - blocked: completed ? remainingOfflineTargetMigrationsForCell : 0, - expiredUnregistered: 0, - repairableExpiredUnregistered: 0, - abortableExpiredUnregistered: 0, - blockedExpiredUnregistered: 0, - blockedExpiredOnNewerTargetAssignment: 0 - }) - } - if (parsed.pathname === '/v1/admin/migration-supersede-cell') { - const superseded = cleanupBeforeSupersession ? 0 : supersessionRemaining - supersessionRemaining = 0 - return response({ v: 1, superseded }) - } - if (parsed.pathname === '/v1/admin/drain') { - drainGraces.push(body.graceMs) - if (loseDrainBeforeAccept && body.graceMs === 120_000) { - throw new Error('injected_drain_response_loss') - } - drained = true - cells.source.draining = true - cells.source.activityLeases = 0 - cells.source.controls = 0 - for (const targetCellId of Object.keys(cells).filter((id) => id !== 'source')) { - cells[targetCellId].totalConnections = 2 - } - return response({ ok: true }) - } - return response({ error: 'unexpected_request' }, 500) - } - return { - overrides: { - commandJson, - terraform: (args) => { - if (args.includes('console')) return 'true\n' - if (args.includes('plan')) return '' - if (args.includes('show') && args.length > 3) { - return JSON.stringify({ resource_changes: [] }) - } - if (args.includes('show')) { - return JSON.stringify({ - values: { - root_module: { - resources: [ - { - address: - 'google_compute_instance_group_manager.relay_gce_cell["source"]', - values: { - name: topologyValue.source.mig_name, - zone: topologyValue.source.zone, - instance_group: topologyValue.source.instance_group, - target_size: migSizes.source, - version: [ - { instance_template: topologyValue.source.generation_identity } - ] - } - } - ] - } - } - }) - } - throw new Error(`unexpected terraform command: ${args.join(' ')}`) - }, - command: (args) => { - throw new Error(`unexpected direct gcloud mutation: ${args.join(' ')}`) - }, - terraformFenceApply: async (fenceConfig, callbacks) => { - const attempt = { - attemptId: '44444444-4444-4444-8444-444444444444', - environment: fenceConfig.environment, - cellId: fenceConfig.cell.cellId, - cellIncarnation: fenceConfig.cellIncarnation, - migName: fenceConfig.cell.migName, - instanceGroup: fenceConfig.cell.instanceGroup, - generationIdentity: fenceConfig.cell.generationIdentity, - fenceCommit: fenceConfig.fenceCommit, - planSha256: 'd'.repeat(64), - planObjectName: - `terraform/state/relay-fence-plans/${fenceConfig.environment}/44444444-4444-4444-8444-444444444444.tfplan`, - planObjectGeneration: '123456789', - varFileSha256: 'e'.repeat(64), - terraformStateLineage: '55555555-5555-4555-8555-555555555555', - terraformStateSerial: 7, - terraformStateObjectGeneration: '987654321', - terraformStateObjectSha256: 'f'.repeat(64), - requestReason: - 'orca-relay-fence/44444444-4444-4444-8444-444444444444' - } - await callbacks.prepareAttempt(attempt) - await callbacks.preApplyGuard() - const invocation = { - invocationId: '66666666-6666-4666-8666-666666666666', - requestReason: - `${attempt.requestReason}/66666666-6666-4666-8666-666666666666` - } - await callbacks.markApplyStarted(attempt, invocation) - const cellId = fenceConfig.cell.cellId - migSizes[cellId] = 0 - instanceCounts[cellId] = 0 - cells[cellId].heartbeatFresh = false - resizeAttempts++ - if (loseResizeResponse && resizeAttempts === 1) { - throw new Error('injected_resize_response_loss') - } - const completed = { ...attempt, gceOperation: 'operation-1' } - await callbacks.markOperation(completed, invocation) - await callbacks.postApplyGuard(attempt.cellIncarnation) - await callbacks.attest(completed) - }, - terraformFenceAdopt: async (fenceConfig, callbacks) => { - assert.equal(await callbacks.loadAttempt(), null) - await callbacks.assertCommittedFenceSet() - await callbacks.preApplyGuard() - await callbacks.assertStateFenced() - await callbacks.postApplyGuard(fenceConfig.cellIncarnation) - assert.equal(await callbacks.loadAttempt(), null) - await callbacks.attest(fenceConfig.cellIncarnation) - await callbacks.postApplyGuard(fenceConfig.cellIncarnation) - await callbacks.commitAdoption(fenceConfig.cellIncarnation) - callbacks.emit({ - event: 'terraform_cell_fence_legacy_adopted', - cellId: fenceConfig.cell.cellId - }) - }, - terraformFenceResume: async (fenceConfig, callbacks) => { - const attempt = await callbacks.loadAttempt() - if (resumeNeedsPreApply) await callbacks.preApplyGuard() - const completed = { - ...attempt, - cellIncarnation: fenceConfig.cellIncarnation, - gceOperation: 'operation-1' - } - await callbacks.postApplyGuard(completed.cellIncarnation) - await callbacks.attest(completed) - }, - terraformFenceRecoverCompleted: async ( - fenceConfig, - callbacks, - recovery - ) => { - const attempt = await callbacks.loadAttempt() - callbacks.emit({ - event: 'terraform_cell_fence_completed_attempt_recovered', - cellId: fenceConfig.cell.cellId, - attemptId: attempt.attemptId, - gceOperation: recovery.gceOperation - }) - }, - terraformFenceAbort: async (fenceConfig, callbacks) => { - await callbacks.abortAttempt() - callbacks.emit({ - event: 'terraform_fence_aborted_before_apply', - cellId: fenceConfig.cell.cellId - }) - }, - terraformFenceSupersede: async (fenceConfig, callbacks) => { - const attempt = await callbacks.loadAttempt() - await callbacks.abortAttempt(attempt) - callbacks.emit({ - event: 'terraform_fence_superseded_before_upload', - cellId: fenceConfig.cell.cellId, - previousFenceCommit: attempt.fenceCommit, - fenceCommit: fenceConfig.fenceCommit - }) - }, - identityToken: () => 'aaa.bbb.ccc', - mutationIdentityToken: () => 'ddd.eee.fff', - resolve4: async () => ['203.0.113.10'], - fetch, - emit: (event) => { - events.push(event) - timeline.push({ kind: 'event', event }) - }, - wait: async () => undefined, - random: () => 0 - }, - batches, - events, - stateChanges, - drainGraces, - timeline, - fencedCompletions: () => fencedCompletions, - capacityReads: () => capacityReads, - selector: () => selector, - addedCells: () => addedCells, - runtimeInspections - } -} - -test('parses deterministic target sets with single-cell selector exceptions', () => { - const common = [ - '--project', - 'project', - '--director-origin', - 'https://relay.example.com', - '--admin-audience', - 'https://relay.example.com/v1/admin/drain', - '--topology-file', - 'topology.json', - '--source-cell-id', - 'source', - '--runtime-service-account', - runtimeServiceAccount, - '--mode', - 'preflight' - ] - assert.deepEqual( - parseMultiTargetArguments([...common, '--target-cell-ids', 'target2,target1']).targetCellIds, - ['target1', 'target2'] - ) - assert.throws( - () => parseMultiTargetArguments([...common, '--target-cell-ids', 'target1']), - /at least 2/ - ) - const cutover = [ - ...common.slice(0, -2), - '--mode', - 'cutover-admission', - '--target-cell-ids', - 'target1,target2', - '--general-cell-ids', - 'general', - '--director-region', - 'us-central1', - '--director-service', - 'relay-director', - '--director-min-instances', - '5' - ] - assert.throws( - () => parseMultiTargetArguments(cutover), - /unobserved-connection-bound/ - ) - assert.equal( - parseMultiTargetArguments([ - ...cutover, - '--unobserved-connection-bound', - '40' - ]).unobservedConnectionBound, - 40 - ) - const recovery = [ - ...common.slice(0, -2), - '--mode', - 'recover-forward', - '--target-cell-ids', - 'target1,target2' - ] - assert.throws( - () => parseMultiTargetArguments(recovery), - /unobserved-connection-bound/ - ) - assert.equal( - parseMultiTargetArguments([ - ...recovery, - '--unobserved-connection-bound', - '60' - ]).unobservedConnectionBound, - 60 - ) - const fenceSource = [ - ...common.slice(0, -2), - '--mode', - 'fence-source', - '--target-cell-ids', - 'target1,target2', - '--fence-commit', - 'a'.repeat(40) - ] - assert.throws( - () => parseMultiTargetArguments(fenceSource), - /unobserved-connection-bound/ - ) - assert.equal( - parseMultiTargetArguments([ - ...fenceSource, - '--unobserved-connection-bound', - '60' - ]).unobservedConnectionBound, - 60 - ) - const addCells = [ - ...common.slice(0, -2), - '--mode', - 'add-migration-cells', - '--target-cell-ids', - 'target1,target2', - '--director-region', - 'us-central1', - '--director-service', - 'relay-director', - '--director-min-instances', - '5', - '--unobserved-connection-bound', - '40' - ] - assert.throws( - () => parseMultiTargetArguments(addCells), - /selector-attempt-id/ - ) - assert.equal( - parseMultiTargetArguments([ - ...addCells, - '--target-cell-ids', - 'target1', - '--selector-attempt-id', - 'add_cells_parse' - ]).targetCellIds.length, - 1 - ) - const retireCell = [ - ...common.slice(0, -2), - '--mode', - 'retire-migration-cell', - '--target-cell-ids', - 'target1', - '--director-region', - 'us-central1', - '--director-service', - 'relay-director', - '--director-min-instances', - '5', - '--selector-attempt-id', - 'retire_cell_parse' - ] - assert.equal( - parseMultiTargetArguments(retireCell).mode, - 'retire-migration-cell' - ) - assert.throws( - () => - parseMultiTargetArguments([ - ...retireCell, - '--target-cell-ids', - 'target1,target2' - ]), - /exactly one/ - ) - const promoteCell = [ - ...common.slice(0, -2), - '--mode', - 'promote-general-cell', - '--target-cell-ids', - 'target1', - '--director-region', - 'us-central1', - '--director-service', - 'relay-director', - '--director-min-instances', - '5', - '--selector-attempt-id', - 'promote_cell_parse' - ] - assert.equal( - parseMultiTargetArguments(promoteCell).mode, - 'promote-general-cell' - ) - assert.throws( - () => - parseMultiTargetArguments([ - ...promoteCell, - '--target-cell-ids', - 'target1,target2' - ]), - /exactly one/ - ) -}) - -test('requires a complete exact completed-fence recovery pin set', () => { - const args = [ - '--project', - 'project', - '--director-origin', - 'https://relay.example.com', - '--admin-audience', - 'https://relay.example.com/v1/admin/drain', - '--topology-file', - 'topology.json', - '--source-cell-id', - 'source', - '--target-cell-ids', - 'target1,target2', - '--runtime-service-account', - runtimeServiceAccount, - '--mode', - 'supersede-target', - '--failed-target-cell-id', - 'target1', - '--replacement-target-cell-id', - 'target2', - '--fence-commit', - 'a'.repeat(40), - '--completed-fence-attempt-id', - '44444444-4444-4444-8444-444444444444', - '--completed-fence-commit', - 'b'.repeat(40), - '--completed-fence-operation', - 'operation-1', - '--completed-fence-state-serial', - '61', - '--completed-fence-plan-generation', - '123', - '--completed-fence-state-generation', - '456', - '--completed-fence-state-sha256', - 'c'.repeat(64), - '--fence-broker-service-account', - 'fence-broker@example.gserviceaccount.com' - ] - assert.equal( - parseMultiTargetArguments(args).completedFenceRecovery - .terraformStateSerial, - 61 - ) - assert.throws( - () => parseMultiTargetArguments(args.slice(0, -2)), - /recovery inputs are invalid/ - ) -}) - -test('requires the cutover source to remain existing-only', () => { - assert.throws( - () => - cutoverMembership(topology(), { - sourceCellId: 'source', - targetCellIds: ['target1'], - generalCellIds: ['source', 'target2'] - }), - /source must remain existing-only/ - ) -}) - -test('requires exact cutover connection-capacity evidence and live headroom', () => { - const status = { - draining: false, - connectionCapacity: { - hardCap: 600, - controlRebindReserve: 100, - ordinaryConnectionLimit: 500, - unobservedBound: 40, - normalAdmissionPause: 460, - pendingControlReservations: 20, - heartbeatFresh: true - }, - runtimeConnectionCapacity: { - hardCap: 600, - controlRebindReserve: 100, - ordinaryConnectionLimit: 500, - unobservedBound: 40, - normalAdmissionPause: 460 - }, - process: { - enforcedConnectionUnits: 400, - preAuthConnections: 3 - } - } - assert.doesNotThrow(() => assertCutoverCellReady('target1', status, 40)) - assert.throws( - () => - assertCutoverCellReady( - 'target1', - { - ...status, - runtimeConnectionCapacity: { - ...status.runtimeConnectionCapacity, - unobservedBound: 39 - } - }, - 40 - ), - /reviewed connection-capacity policy/ - ) - assert.throws( - () => - assertCutoverCellReady( - 'target1', - { - ...status, - connectionCapacity: { - ...status.connectionCapacity, - pendingControlReservations: 60 - } - }, - 40 - ), - /normal-admission connection headroom/ - ) - assert.throws( - () => - assertCutoverCellReady( - 'target1', - { - ...status, - process: { ...status.process, preAuthConnections: 45 } - }, - 40 - ), - /pre-auth connection headroom/ - ) -}) - -test('cuts over only after every proposed cell passes exact readiness evidence', async () => { - await withTopology(async (file) => { - const testHarness = harness() - await runMultiTargetDeployment( - config(file, 'cutover-admission'), - testHarness.overrides - ) - assert.deepEqual(testHarness.selector(), { - generation: 1, - attemptId: testHarness.selector().attemptId, - membership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - assert.equal(testHarness.events.at(-1).event, 'admission_selector_cutover') - assert.deepEqual(Object.fromEntries(testHarness.runtimeInspections), { - target1: 2, - target2: 2, - general: 2 - }) - }) -}) - -test('rejects a different active selector before readiness checks or revision mutation', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - selectorMembership: { - existingOnly: ['source', 'target2'], - migrationOnly: ['target1'], - general: ['general'] - } - }) - await assert.rejects( - runMultiTargetDeployment( - config(file, 'cutover-admission'), - testHarness.overrides - ), - /already active with different membership/ - ) - assert.equal(testHarness.runtimeInspections.size, 0) - }) -}) - -test('registers new Terraform targets as one selector generation', async () => { - await withTopology(async (file) => { - const reviewed = topology() - reviewed.target1.connection_hard_cap = 1_000 - reviewed.target1.connection_unobserved_bound = 60 - writeFileSync(file, JSON.stringify(reviewed)) - const testHarness = harness({ - selectorMembership: { - existingOnly: ['source'], - migrationOnly: [], - general: ['general'] - }, - activeDirectorMinimum: 5 - }) - await runMultiTargetDeployment( - { ...config(file, 'add-migration-cells'), directorMinimumInstances: 5 }, - testHarness.overrides - ) - assert.deepEqual(testHarness.selector(), { - generation: 2, - attemptId: 'add_cells_test', - membership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - assert.equal(testHarness.events.at(-1).event, 'migration_cells_added') - assert.deepEqual(testHarness.addedCells(), [ - { - cellId: 'target1', - cellUrl: topology().target1.origin, - capacityRequests: 4_000, - region: 'us-central1', - connectionHardCap: 1_000, - connectionUnobservedBound: 60 - }, - { - cellId: 'target2', - cellUrl: topology().target2.origin, - capacityRequests: 4_000, - region: 'us-central1', - connectionHardCap: 600, - connectionUnobservedBound: 40 - } - ]) - assert.equal(testHarness.runtimeInspections.size, 0) - }) -}) - -test('promotes exactly one healthy migration cell to general admission', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'promote-general-cell'), - testHarness.overrides - ) - assert.deepEqual(testHarness.selector(), { - generation: 2, - attemptId: 'promote_cell_test', - membership: { - existingOnly: ['source'], - migrationOnly: ['target2'], - general: ['general', 'target1'] - } - }) - assert.equal(testHarness.events.at(-1).event, 'migration_cell_promoted_general') - }) -}) - -test('rejects promotion below the configured director floor', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await assert.rejects( - runMultiTargetDeployment( - { ...config(file, 'promote-general-cell'), directorMinimumInstances: 2 }, - testHarness.overrides - ), - /active director is not compatible/ - ) - assert.equal(testHarness.events.length, 0) - }) -}) - -test('rejects general promotion unless the exact cell is migration-only', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target2'], - general: ['general', 'target1'] - } - }) - await assert.rejects( - runMultiTargetDeployment( - config(file, 'promote-general-cell'), - testHarness.overrides - ), - /must be migration-only/ - ) - assert.equal(testHarness.events.length, 0) - }) -}) - -test('retires exactly one migration cell without changing other membership', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'retire-migration-cell'), - testHarness.overrides - ) - assert.deepEqual(testHarness.selector(), { - generation: 2, - attemptId: 'retire_cell_test', - membership: { - existingOnly: ['source', 'target1'], - migrationOnly: ['target2'], - general: ['general'] - } - }) - assert.equal(testHarness.events.at(-1).event, 'migration_cell_retired') - assert.equal(testHarness.runtimeInspections.size, 0) - }) -}) - -test('rejects retirement unless the exact cell is migration-only', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target2'], - general: ['general', 'target1'] - } - }) - await assert.rejects( - runMultiTargetDeployment( - config(file, 'retire-migration-cell'), - testHarness.overrides - ), - /must be migration-only/ - ) - assert.equal(testHarness.events.length, 0) - }) -}) - -test('rejects overlapping generations and allocates below the connection ceiling', () => { - const selected = selectMultiTargetDeployments(topology(), 'source', ['target1', 'target2']) - assert.equal(selected.targets.length, 2) - const planned = allocateTargetQuotas({ - sourceAssignments: 4, - sourceConnections: 5, - requiredTargetUnits: 8, - connectionCeiling: 6, - targets: [ - { - cellId: 'target1', - currentConnections: 0, - availableConnectionReservations: 10, - availableTargetUnits: 10 - }, - { - cellId: 'target2', - currentConnections: 0, - availableConnectionReservations: 10, - availableTargetUnits: 10 - } - ] - }) - assert.deepEqual( - planned.map(({ cellId, quota, projectedConnections }) => ({ - cellId, - quota, - projectedConnections - })), - [ - { cellId: 'target1', quota: 2, projectedConnections: 3 }, - { cellId: 'target2', quota: 2, projectedConnections: 3 } - ] - ) - assert.throws(() => - allocateTargetQuotas({ - sourceAssignments: 4, - sourceConnections: 20, - requiredTargetUnits: 8, - connectionCeiling: 4, - targets: [ - { - cellId: 'target1', - currentConnections: 0, - availableConnectionReservations: 10, - availableTargetUnits: 10 - }, - { - cellId: 'target2', - currentConnections: 0, - availableConnectionReservations: 10, - availableTargetUnits: 10 - } - ] - }) - ) - assert.deepEqual( - allocateTargetQuotas({ - sourceAssignments: 1, - sourceConnections: 700, - requiredTargetUnits: 1, - connectionCeiling: 1_000, - targets: [ - { - cellId: 'target-600', - currentConnections: 0, - connectionCeiling: 600, - availableConnectionReservations: 1, - availableTargetUnits: 1 - }, - { - cellId: 'target-1000', - currentConnections: 0, - connectionCeiling: 1_000, - availableConnectionReservations: 1, - availableTargetUnits: 1 - } - ] - }).map(({ cellId, projectedConnections }) => ({ cellId, projectedConnections })), - [ - { cellId: 'target-600', projectedConnections: 699 }, - { cellId: 'target-1000', projectedConnections: 700 } - ] - ) -}) - -test('assumes every unbound source connection can land on one target', () => { - const targets = [ - { - cellId: 'target1', - currentConnections: 0, - availableConnectionReservations: 10, - availableTargetUnits: 10 - }, - { - cellId: 'target2', - currentConnections: 0, - availableConnectionReservations: 10, - availableTargetUnits: 10 - } - ] - const planned = allocateTargetQuotas({ - sourceAssignments: 4, - sourceConnections: 9, - requiredTargetUnits: 8, - connectionCeiling: 8, - targets - }) - assert.deepEqual( - planned.map(({ quota, projectedConnections }) => ({ quota, projectedConnections })), - [ - { quota: 2, projectedConnections: 7 }, - { quota: 2, projectedConnections: 7 } - ] - ) - assert.throws(() => - allocateTargetQuotas({ - sourceAssignments: 4, - sourceConnections: 9, - requiredTargetUnits: 8, - connectionCeiling: 7, - targets - }) - ) -}) - -test('serializes deterministic target quotas and completes after drain acceptance', async () => { - await withTopology(async (file) => { - const testHarness = harness() - await runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides) - assert.deepEqual(testHarness.batches, [ - ['target1', 2], - ['target2', 2] - ]) - assert.deepEqual(testHarness.stateChanges.slice(0, 3), [ - ['source', false], - ['target1', true], - ['target2', true] - ]) - assert.equal(testHarness.events.some((event) => event.event === 'source_drain_accepted'), true) - assert.equal(testHarness.events.at(-1).event, 'multi_target_complete') - }) -}) - -test('uses fresh aggregate logs for a legacy source without runtime counts', async () => { - await withTopology(async (file) => { - const testHarness = harness({ legacySource: true }) - await runMultiTargetDeployment(config(file), testHarness.overrides) - assert.equal(testHarness.events.at(-1).sourceConnections, 5) - }) -}) - -test('rejects stale legacy source telemetry', async () => { - await withTopology(async (file) => { - const testHarness = harness({ legacySource: true, legacyMetricAgeMs: 90_001 }) - await assert.rejects( - runMultiTargetDeployment(config(file), testHarness.overrides), - /runtime metrics are stale/ - ) - }) -}) - -test('refuses a drain with an expiring lease and restores pre-drain admission', async () => { - await withTopology(async (file) => { - const testHarness = harness({ leaseRemainingMs: 599_999 }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), - /insufficient time/ - ) - assert.deepEqual(testHarness.stateChanges.slice(-3), [ - ['source', true], - ['target1', false], - ['target2', false] - ]) - assert.equal( - testHarness.events.some((event) => event.event === 'source_drain_accepted'), - false - ) - }) -}) - -test('preserves forward recovery when target registration status is unavailable', async () => { - await withTopology(async (file) => { - const testHarness = harness({ failEvacuationStatus: true }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), - /cannot prove zero target registrations/ - ) - assert.equal( - testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), - false - ) - assert.equal( - testHarness.stateChanges.some( - ([cellId, enabled]) => cellId.startsWith('target') && !enabled - ), - false - ) - assert.deepEqual(testHarness.events.at(-1), { - event: 'multi_forward_recovery_required', - targetRegistered: null, - reason: 'registration_status_unavailable' - }) - }) -}) - -test('audits partial migration state without requiring new migration headroom', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - capacitySourceAssignments: [1_000], - capacityRequiredTargetUnits: [2_000] - }) - await runMultiTargetDeployment(config(file, 'audit'), testHarness.overrides) - assert.equal(testHarness.capacityReads(), 0) - assert.deepEqual(testHarness.events.at(-1), { - event: 'multi_target_audit', - inProgress: 4, - targetRegistered: 0, - registeredSourceActive: 0, - registeredCompletable: 0, - registeredTargetInactive: 0, - completed: 0, - blocked: 0, - expiredUnregistered: 0, - repairableExpiredUnregistered: 0, - abortableExpiredUnregistered: 0, - blockedExpiredUnregistered: 0, - blockedExpiredOnNewerTargetAssignment: 0 - }) - }) -}) - -test('never restores source admission after drain acceptance', async () => { - await withTopology(async (file) => { - const testHarness = harness({ failAfterDrain: true }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), - /injected_completion_failure/ - ) - assert.equal(testHarness.events.some((event) => event.event === 'source_drain_accepted'), true) - assert.equal( - testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), - false - ) - assert.equal(testHarness.events.at(-1).event, 'multi_forward_recovery_required') - }) -}) - -test('never restores source admission after the send transition becomes ambiguous', async () => { - await withTopology(async (file) => { - const testHarness = harness({ loseDrainSendResponse: true }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), - /injected_send_transition_response_loss/ - ) - assert.equal( - testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), - false - ) - assert.equal(testHarness.events.at(-1).event, 'multi_forward_recovery_required') - }) -}) - -test('freezes forward recovery when a legacy drain response is ambiguous', async () => { - await withTopology(async (file) => { - const testHarness = harness({ loseDrainBeforeAccept: true }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), - /injected_drain_response_loss/ - ) - assert.equal( - testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), - false - ) - await assert.rejects( - runMultiTargetDeployment(config(file, 'recover-forward'), testHarness.overrides), - /drain_application_receipt_missing/ - ) - assert.deepEqual(testHarness.drainGraces, [120_000]) - }) -}) - -test('resumes an exact prepared drain without allocating new migrations', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - preparedDrainAttempt: true, - preexistingRegisteredMigrations: 2, - sourceAssignments: 0, - sourceRequiredTargetUnits: 0, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'recover-forward'), - testHarness.overrides - ) - assert.deepEqual(testHarness.drainGraces, [120_000]) - assert.deepEqual(testHarness.batches, []) - assert.equal(testHarness.capacityReads(), 8) - assert.equal( - testHarness.events.some( - (event) => event.event === 'source_prepared_drain_recovered' - ), - true - ) - assert.deepEqual(testHarness.events.at(-1), { - event: 'multi_target_complete', - sourceCellId: 'source' - }) - }) -}) - -test('registers only remaining assignments before recovering a replacement drain', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - preexistingRegisteredMigrations: 1, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'recover-forward'), - testHarness.overrides - ) - assert.deepEqual(testHarness.batches, [ - ['target1', 2], - ['target2', 2] - ]) - assert.equal(testHarness.capacityReads(), 8) - assert.deepEqual(testHarness.drainGraces, [0]) - const eventNames = testHarness.events.map((event) => event.event) - assert.equal( - testHarness.events.find( - (event) => event.event === 'multi_forward_recovery_preflight' - ).targetRegistered, - 2 - ) - assert.ok( - eventNames.lastIndexOf('multi_migration_batch') < - eventNames.indexOf('multi_forward_recovery_ready_to_drain') - ) - assert.ok( - eventNames.indexOf('multi_forward_recovery_ready_to_drain') < - eventNames.indexOf('source_recovery_drain_accepted') - ) - }) -}) - -test('refreshes registered migration leases before the recovery drain gate', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - preexistingRegisteredMigrations: 2, - sourceAssignments: 0, - sourceRequiredTargetUnits: 0, - leaseRemainingMs: 599_999, - refreshedLeaseRemainingMs: 900_000, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'recover-forward'), - testHarness.overrides - ) - assert.deepEqual(testHarness.drainGraces, [0]) - }) -}) - -test('waits for catch-up migrations to gain durable target ownership', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - recoveryRegistrationDelayReads: 2, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'recover-forward'), - testHarness.overrides - ) - const ownershipEvents = testHarness.events.filter( - (event) => event.event === 'multi_recovery_target_ownership' - ) - assert.equal(ownershipEvents.length, 3) - assert.equal(ownershipEvents[0].inProgress > ownershipEvents[0].targetRegistered, true) - assert.equal(ownershipEvents.at(-1).inProgress, ownershipEvents.at(-1).targetRegistered) - assert.deepEqual(testHarness.drainGraces, [0]) - }) -}) - -test('times out before drain when catch-up migrations remain unregistered', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - let now = 0 - testHarness.overrides.now = () => now - testHarness.overrides.wait = async (ms) => { - now += ms - } - await assert.rejects( - runMultiTargetDeployment( - { ...config(file, 'recover-forward'), timeoutMs: 3 }, - testHarness.overrides - ), - /timed out waiting for recovery target ownership/ - ) - assert.deepEqual(testHarness.drainGraces, []) - }) -}) - -test('drains a fully unregistered recovery within the reviewed bound', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - { - ...config(file, 'recover-forward'), - unobservedConnectionBound: 4 - }, - testHarness.overrides - ) - assert.deepEqual(testHarness.drainGraces, [0]) - assert.equal( - testHarness.events.some( - (event) => event.event === 'multi_recovery_bounded_unregistered' - ), - true - ) - }) -}) - -test('drains mixed target registrations within the unobserved bound', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - preexistingRegisteredMigrations: 1, - recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - { - ...config(file, 'recover-forward'), - unobservedConnectionBound: 2 - }, - testHarness.overrides - ) - assert.deepEqual(testHarness.drainGraces, [0]) - const event = testHarness.events.find( - (entry) => entry.event === 'multi_recovery_bounded_mixed_registration' - ) - assert.equal(event.unregistered, 2) - assert.equal(event.targetRegistered, 2) - }) -}) - -test('reissues a proven non-delivered recovery drain and settles bounded offline clients', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - recoveryAlreadyAttempted: true, - preexistingRegisteredMigrations: 1, - unregisteredTargetMigrations: 1, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - { - ...config(file, 'recover-forward'), - unobservedConnectionBound: 2 - }, - testHarness.overrides - ) - assert.deepEqual(testHarness.drainGraces, [0]) - assert.equal( - testHarness.events.some( - (event) => event.event === 'source_recovery_drain_reissued_after_non_delivery' - ), - true - ) - assert.equal( - testHarness.events.some( - (event) => event.event === 'multi_migration_bounded_offline_complete' - ), - true - ) - }) -}) - -test('rejects mixed target registrations above the unobserved bound', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - preexistingRegisteredMigrations: 1, - recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - let now = 0 - testHarness.overrides.now = () => now - testHarness.overrides.wait = async (ms) => { - now += ms - } - await assert.rejects( - runMultiTargetDeployment( - { - ...config(file, 'recover-forward'), - timeoutMs: 3, - unobservedConnectionBound: 1 - }, - testHarness.overrides - ), - /timed out waiting for recovery target ownership/ - ) - assert.deepEqual(testHarness.drainGraces, []) - }) -}) - -test('times out before drain while a target registration remains source-active', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - preexistingRegisteredMigrations: 1, - registeredSourceActive: 1, - recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - let now = 0 - testHarness.overrides.now = () => now - testHarness.overrides.wait = async (ms) => { - now += ms - } - await assert.rejects( - runMultiTargetDeployment( - { - ...config(file, 'recover-forward'), - timeoutMs: 3, - unobservedConnectionBound: 2 - }, - testHarness.overrides - ), - /timed out waiting for recovery target ownership/ - ) - assert.deepEqual(testHarness.drainGraces, []) - }) -}) - -test('allows recovery drain with durable offline target registrations', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - offlineTargetMigrations: 1, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'recover-forward'), - testHarness.overrides - ) - assert.deepEqual(testHarness.drainGraces, [0]) - }) -}) - -test('catches up source assignments that arrive during recovery publication', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - sourceAssignments: 5, - sourceRequiredTargetUnits: 10, - capacitySourceAssignments: [4, 4, 4, 4, 1, 1, 1, 1], - capacityRequiredTargetUnits: [8, 8, 8, 8, 2, 2, 2, 2], - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'recover-forward'), - testHarness.overrides - ) - assert.equal( - testHarness.batches.reduce((total, [, limit]) => total + limit, 0), - 5 - ) - assert.equal( - testHarness.events.some( - (event) => event.event === 'multi_forward_recovery_catch_up' - ), - true - ) - assert.deepEqual( - testHarness.events - .filter((event) => event.event === 'multi_target_preflight') - .map((event) => ({ - sourceConnections: event.sourceConnections, - observedSourceConnections: event.observedSourceConnections - })), - [ - { sourceConnections: 5, observedSourceConnections: 5 }, - { sourceConnections: 1, observedSourceConnections: 5 } - ] - ) - assert.deepEqual(testHarness.drainGraces, [0]) - }) -}) - -test('replans recover-forward after transactional target headroom rejection', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - headroomFailureTarget: 'target1', - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'recover-forward'), - testHarness.overrides - ) - assert.equal( - testHarness.events.some( - (event) => - event.event === 'multi_recovery_target_headroom_paused' && - event.targetCellId === 'target1' - ), - true - ) - assert.equal( - testHarness.events.some( - (event) => event.event === 'multi_forward_recovery_catch_up' - ), - true - ) - assert.deepEqual(testHarness.drainGraces, [0]) - }) -}) - -test('uses conservative changing capacity samples during recovery', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - sourceAssignments: 5, - sourceRequiredTargetUnits: 10, - capacitySourceAssignments: [5, 4, 5, 4], - capacityRequiredTargetUnits: [10, 8, 10, 8], - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'recover-forward'), - testHarness.overrides - ) - assert.equal( - testHarness.batches.reduce((total, [, limit]) => total + limit, 0), - 5 - ) - assert.deepEqual(testHarness.drainGraces, [0]) - }) -}) - -test('requires every final recovery sample to prove zero source assignments', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - preexistingRegisteredMigrations: 2, - sourceAssignments: 1, - sourceRequiredTargetUnits: 2, - capacitySourceAssignments: [1, 1, 1, 1, 0, 0, 0, 1], - capacityRequiredTargetUnits: [2, 2, 2, 2, 0, 0, 0, 2], - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'recover-forward'), - testHarness.overrides - ) - assert.equal( - testHarness.events.some( - (event) => event.event === 'multi_forward_recovery_catch_up' - ), - true - ) - assert.deepEqual(testHarness.drainGraces, [0]) - }) -}) - -test('stops after bounded recovery catch-up cannot quiesce the source', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - recoverableDrain: true, - sourceAssignments: 10, - sourceRequiredTargetUnits: 20, - capacitySourceAssignments: Array.from({ length: 40 }, () => 1), - capacityRequiredTargetUnits: Array.from({ length: 40 }, () => 2), - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await assert.rejects( - runMultiTargetDeployment( - config(file, 'recover-forward'), - testHarness.overrides - ), - /did not quiesce within bounded recovery catch-up/ - ) - assert.equal( - testHarness.batches.reduce((total, [, limit]) => total + limit, 0), - 5 - ) - assert.deepEqual(testHarness.drainGraces, []) - }) -}) - -test('settles a drained source while registered target users remain offline', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - allowPreFenceCompletion: true, - offlineTargetMigrations: 1, - selectorMembership: { - existingOnly: ['source'], - migrationOnly: ['target1', 'target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'recover-forward'), - testHarness.overrides - ) - assert.deepEqual(testHarness.drainGraces, []) - assert.deepEqual(testHarness.events.at(-1), { - event: 'multi_target_complete', - sourceCellId: 'source' - }) - }) -}) - -test('fences a quiescent disabled source and completes through stale-source checks', async () => { - await withTopology(async (file) => { - const testHarness = harness({ fence: true }) - const authorizationByOrigin = new Map() - const fetch = testHarness.overrides.fetch - testHarness.overrides.fetch = async (url, options) => { - const parsed = new URL(url) - if (parsed.pathname.startsWith('/v1/admin/')) { - authorizationByOrigin.set( - parsed.origin, - new Set([ - ...(authorizationByOrigin.get(parsed.origin) ?? []), - options?.headers?.authorization - ]) - ) - } - return await fetch(url, options) - } - await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) - assert.equal(testHarness.fencedCompletions(), 2) - assert.deepEqual( - authorizationByOrigin.get('https://relay.example.com'), - new Set(['Bearer aaa.bbb.ccc', 'Bearer ddd.eee.fff']) - ) - assert.equal(authorizationByOrigin.has('https://a.relay.example.com'), false) - assert.equal(authorizationByOrigin.has('https://b.relay.example.com'), false) - assert.equal(authorizationByOrigin.has('https://c.relay.example.com'), false) - assert.deepEqual(testHarness.events.at(-1), { - event: 'source_fenced', - sourceCellId: 'source', - targetSize: 0 - }) - }) -}) - -test('adopts an already-fenced source only through the legacy no-op path', async () => { - await withTopology(async (file) => { - const testHarness = harness({ fence: true, alreadyFencedSource: true }) - await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) - assert.equal(testHarness.fencedCompletions(), 2) - assert.deepEqual( - testHarness.events.find( - ({ event }) => event === 'terraform_cell_fence_legacy_adopted' - ), - { event: 'terraform_cell_fence_legacy_adopted', cellId: 'source' } - ) - assert.deepEqual(testHarness.events.at(-1), { - event: 'source_fenced', - sourceCellId: 'source', - targetSize: 0 - }) - }) -}) - -test('refuses to attest a fence when selected targets omit an outgoing migration', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - alreadyFencedSource: true, - sourceOutgoingMigrations: 5 - }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), - /full migration coverage/ - ) - assert.equal(testHarness.fencedCompletions(), 0) - }) -}) - -test('clears the complete 156-row legacy backlog before reporting the source fenced', async () => { - const rollout = topology() - const additionalTarget = (id, hostname) => ({ - ...rollout.target2, - origin: `https://${hostname}.relay.example.com`, - zone: `us-central1-${hostname}`, - mig_name: `relay-${hostname}`, - instance_group: `https://compute.example/instanceGroups/relay-${hostname}`, - backend_name: `relay-${hostname}`, - backend_id: `https://compute.example/backendServices/relay-${hostname}`, - generation_identity: `https://compute.example/instanceTemplates/relay-${hostname}-abc`, - image: `us-central1-docker.pkg.dev/project/repo/relay@${digest(id)}` - }) - rollout.target3 = additionalTarget('e', 'e') - rollout.target4 = additionalTarget('f', 'f') - rollout.target5 = additionalTarget('1', 'g') - rollout.target6 = additionalTarget('2', 'h') - const migrationCounts = { - target1: 58, - target2: 63, - target3: 24, - target4: 10, - target5: 1, - target6: 0 - } - - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - alreadyFencedSource: true, - offlineTargetMigrationsByTarget: migrationCounts, - topologyValue: rollout - }) - const fenceConfig = config(file, 'fence-source') - fenceConfig.targetCellIds = Object.keys(migrationCounts) - - await runMultiTargetDeployment(fenceConfig, testHarness.overrides) - - const sourceFencedIndex = testHarness.timeline.findIndex( - ({ kind, event }) => kind === 'event' && event.event === 'source_fenced' - ) - assert.notEqual(sourceFencedIndex, -1) - let observedInitialTotal = 0 - for (const [targetCellId, initialCount] of Object.entries(migrationCounts)) { - const initialIndex = testHarness.timeline.findIndex( - (entry) => - entry.kind === 'evacuation_status' && - entry.targetCellId === targetCellId && - entry.inProgress === initialCount && - entry.registeredTargetInactive === initialCount - ) - const clearedIndex = testHarness.timeline.findIndex( - (entry) => - entry.kind === 'evacuation_status' && - entry.targetCellId === targetCellId && - entry.inProgress === 0 - ) - assert.notEqual(initialIndex, -1) - observedInitialTotal += testHarness.timeline[initialIndex].inProgress - if (initialCount > 0) assert.ok(clearedIndex > initialIndex) - else assert.ok(clearedIndex >= initialIndex) - assert.ok(clearedIndex < sourceFencedIndex) - } - assert.equal(observedInitialTotal, 156) - assert.equal(testHarness.fencedCompletions(), 6) - }, rollout) -}) - -test('refuses legacy adoption when the live MIG template changed', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - alreadyFencedSource: true, - liveMigTemplateOverrides: { - source: `${topology().source.generation_identity}-other` - } - }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), - /source MIG fence topology is unsafe/ - ) - assert.equal(testHarness.fencedCompletions(), 0) - }) -}) - -test('fences with reviewed 600 templates during the declared 1000 rollout', async () => { - const rollout = topology() - for (const [cellId, cell] of Object.entries(rollout)) { - cell.connection_unobserved_bound = 60 - if (['target1', 'target2'].includes(cellId)) cell.connection_hard_cap = 1_000 - } - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - templateUnobservedBound: 60, - directorUnobservedBound: 60 - }) - const fenceConfig = config(file, 'fence-source') - fenceConfig.unobservedConnectionBound = 60 - await runMultiTargetDeployment(fenceConfig, testHarness.overrides) - assert.equal(testHarness.fencedCompletions(), 2) - }, rollout) -}) - -test('fences exact configured capacity when the saved Terraform output is legacy', async () => { - const rollout = topology() - for (const cell of Object.values(rollout)) { - delete cell.connection_hard_cap - delete cell.connection_unobserved_bound - } - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - templateUnobservedBound: 60, - directorUnobservedBound: 60 - }) - const fenceConfig = config(file, 'fence-source') - fenceConfig.unobservedConnectionBound = 60 - await runMultiTargetDeployment(fenceConfig, testHarness.overrides) - assert.equal(testHarness.fencedCompletions(), 2) - }, rollout) -}) - -test('refuses legacy Terraform output when live capacity differs from broker config', async () => { - const rollout = topology() - for (const cell of Object.values(rollout)) { - delete cell.connection_hard_cap - delete cell.connection_unobserved_bound - } - await withTopology(async (file) => { - const testHarness = harness({ fence: true }) - const fenceConfig = config(file, 'fence-source') - fenceConfig.unobservedConnectionBound = 60 - await assert.rejects( - runMultiTargetDeployment(fenceConfig, testHarness.overrides), - /instance template capacity differs from Terraform/ - ) - assert.equal(testHarness.fencedCompletions(), 0) - }, rollout) -}) - -test('refuses explicit null capacity as a legacy Terraform output', async () => { - const rollout = topology() - for (const cell of Object.values(rollout)) { - cell.connection_hard_cap = null - cell.connection_unobserved_bound = null - } - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - templateUnobservedBound: 60, - directorUnobservedBound: 60 - }) - const fenceConfig = config(file, 'fence-source') - fenceConfig.unobservedConnectionBound = 60 - await assert.rejects( - runMultiTargetDeployment(fenceConfig, testHarness.overrides), - /instance template capacity differs from Terraform/ - ) - assert.equal(testHarness.fencedCompletions(), 0) - }, rollout) -}) - -test('refuses a mixed legacy and declared capacity topology', async () => { - const rollout = topology() - for (const cell of Object.values(rollout)) { - delete cell.connection_hard_cap - delete cell.connection_unobserved_bound - } - rollout.general.connection_hard_cap = null - rollout.general.connection_unobserved_bound = null - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - templateUnobservedBound: 60, - directorUnobservedBound: 60 - }) - const fenceConfig = config(file, 'fence-source') - fenceConfig.unobservedConnectionBound = 60 - await assert.rejects( - runMultiTargetDeployment(fenceConfig, testHarness.overrides), - /instance template capacity differs from Terraform/ - ) - assert.equal(testHarness.fencedCompletions(), 0) - }, rollout) -}) - -test('fences after the active template reaches the declared 1000 capacity', async () => { - const rollout = topology() - for (const cellId of Object.keys(rollout)) { - rollout[cellId].connection_hard_cap = 1_000 - } - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - templateHardCap: 1_000, - directorHardCap: 1_000 - }) - await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) - assert.equal(testHarness.fencedCompletions(), 2) - }, rollout) -}) - -test('refuses a rollout predecessor whose template has the wrong bound', async () => { - const rollout = topology() - for (const cellId of ['target1', 'target2']) { - rollout[cellId].connection_hard_cap = 1_000 - rollout[cellId].connection_unobserved_bound = 60 - } - await withTopology(async (file) => { - const testHarness = harness({ fence: true }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), - /instance template capacity is outside reviewed rollout/ - ) - assert.equal(testHarness.fencedCompletions(), 0) - }, rollout) -}) - -test('refuses a rollout predecessor when the director reports another cap', async () => { - const rollout = topology() - for (const cellId of ['target1', 'target2']) { - rollout[cellId].connection_hard_cap = 1_000 - } - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - directorCapacityOverrides: { - target1: { hardCap: 1_000, unobservedBound: 40 } - } - }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), - /connection capacity differs from Terraform/ - ) - assert.equal(testHarness.fencedCompletions(), 0) - }, rollout) -}) - -test('preserves direct target runtime reads outside fence mode', async () => { - await withTopology(async (file) => { - const testHarness = harness({}) - const fetch = testHarness.overrides.fetch - const authorizationByOrigin = new Map() - testHarness.overrides.fetch = async (url, options) => { - const parsed = new URL(url) - if (parsed.pathname.startsWith('/v1/admin/')) { - authorizationByOrigin.set( - parsed.origin, - new Set([ - ...(authorizationByOrigin.get(parsed.origin) ?? []), - options?.headers?.authorization - ]) - ) - } - return await fetch(url, options) - } - - await runMultiTargetDeployment(config(file, 'preflight'), testHarness.overrides) - assert.deepEqual( - authorizationByOrigin.get('https://relay.example.com'), - new Set(['Bearer aaa.bbb.ccc']) - ) - assert.deepEqual( - authorizationByOrigin.get('https://b.relay.example.com'), - new Set(['Bearer aaa.bbb.ccc']) - ) - assert.deepEqual( - authorizationByOrigin.get('https://c.relay.example.com'), - new Set(['Bearer aaa.bbb.ccc']) - ) - }) -}) - -test('refuses to fence when a cell hostname routes to the wrong backend', async () => { - await withTopology(async (file) => { - const testHarness = harness({ fence: true, misroutedCell: 'target1' }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), - /retained route topology mismatch/ - ) - assert.equal(testHarness.fencedCompletions(), 0) - }) -}) - -test('refuses to fence when a cell route overrides the reviewed backend', async () => { - await withTopology(async (file) => { - const testHarness = harness({ fence: true, routeOverrideCell: 'target1' }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), - /retained route topology mismatch/ - ) - assert.equal(testHarness.fencedCompletions(), 0) - }) -}) - -test('refuses to fence when the live HTTPS frontend uses another URL map', async () => { - await withTopology(async (file) => { - const testHarness = harness({ fence: true, frontendMisbound: true }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), - /live frontend topology mismatch/ - ) - assert.equal(testHarness.fencedCompletions(), 0) - }) -}) - -test('refuses to fence when the fresh source heartbeat reports active work', async () => { - await withTopology(async (file) => { - const testHarness = harness({ fence: true, sourceObservedRequests: 1 }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), - /source fencing requires zero source-owned work/ - ) - assert.equal(testHarness.fencedCompletions(), 0) - }) -}) - -test('fences a quiescent source while registered target users remain offline', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - alreadyFencedSource: true, - offlineTargetMigrations: 1 - }) - await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) - assert.equal(testHarness.fencedCompletions(), 2) - assert.deepEqual(testHarness.events.at(-1), { - event: 'source_fenced', - sourceCellId: 'source', - targetSize: 0 - }) - }) -}) - -test('refuses to fence source-active or unregistered migrations', async () => { - for (const migrationState of [ - { registeredSourceActive: 1 }, - { unregisteredTargetMigrations: 1 } - ]) { - await withTopology(async (file) => { - const testHarness = harness({ fence: true, ...migrationState }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), - /full migration coverage and durable target ownership/ - ) - }) - } -}) - -test('resumes fenced completion after losing the resize response', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - fence: true, - loseResizeResponse: true, - resumeNeedsPreApply: true - }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), - /injected_resize_response_loss/ - ) - await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) - assert.equal(testHarness.fencedCompletions(), 2) - assert.deepEqual(testHarness.events.at(-1), { - event: 'source_fenced', - sourceCellId: 'source', - targetSize: 0 - }) - }) -}) - -test('records a proven pre-apply Terraform fence abort', async () => { - await withTopology(async (file) => { - const testHarness = harness({ existingFenceAttempt: true }) - await runMultiTargetDeployment( - config(file, 'abort-fence-source'), - testHarness.overrides - ) - assert.deepEqual(testHarness.events.at(-1), { - event: 'terraform_fence_aborted_before_apply', - cellId: 'source' - }) - }) -}) - -test('fences a failed registered target before aggregate supersession', async () => { - await withTopology(async (file) => { - const testHarness = harness({ supersede: true }) - await runMultiTargetDeployment( - config(file, 'supersede-target'), - testHarness.overrides - ) - assert.equal( - testHarness.stateChanges.some( - ([cellId, enabled]) => cellId === 'target2' && enabled - ), - true - ) - assert.deepEqual(testHarness.events.at(-1), { - event: 'registered_target_superseded', - sourceCellId: 'source', - failedTargetCellId: 'target1', - replacementTargetCellId: 'target2', - superseded: 2, - remainingUnregistered: 0 - }) - assert.equal(testHarness.runtimeInspections.get('target2') ?? 0, 0) - }) -}) - -test('fails closed when cleanup wins before aggregate supersession selects rows', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - supersede: true, - cleanupBeforeSupersession: true - }) - - await assert.rejects( - runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), - /registered target supersession count changed/ - ) - assert.deepEqual(testHarness.events, []) - }) -}) - -test('does not accept a capacity predecessor for target supersession', async () => { - const rollout = topology() - rollout.target2.connection_hard_cap = 1_000 - await withTopology(async (file) => { - const testHarness = harness({ supersede: true }) - await assert.rejects( - runMultiTargetDeployment( - config(file, 'supersede-target'), - testHarness.overrides - ), - /instance template capacity is outside reviewed rollout/ - ) - }, rollout) -}) - -test('does not accept capacity-bearing templates from legacy output for supersession', async () => { - const rollout = topology() - for (const cell of Object.values(rollout)) { - delete cell.connection_hard_cap - delete cell.connection_unobserved_bound - } - await withTopology(async (file) => { - const testHarness = harness({ supersede: true }) - await assert.rejects( - runMultiTargetDeployment( - config(file, 'supersede-target'), - testHarness.overrides - ), - /instance template capacity differs from Terraform/ - ) - }, rollout) -}) - -test('replaces a superseded unuploaded fence attempt before target fencing', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - supersede: true, - supersededFenceAttempt: true - }) - await runMultiTargetDeployment( - config(file, 'supersede-target'), - testHarness.overrides - ) - assert.deepEqual(testHarness.events[0], { - event: 'terraform_fence_superseded_before_upload', - cellId: 'target1', - previousFenceCommit: 'b'.repeat(40), - fenceCommit: 'a'.repeat(40) - }) - assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') - }) -}) - -test('recovers a pinned completed older fence before registered supersession', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - supersede: true, - completedFenceAttempt: true - }) - const deploymentConfig = config(file, 'supersede-target') - deploymentConfig.completedFenceRecovery = { - attemptId: '44444444-4444-4444-8444-444444444444', - fenceCommit: 'b'.repeat(40), - gceOperation: 'operation-1', - terraformStateSerial: 7, - planObjectGeneration: '123456789', - terraformStateObjectGeneration: '222222222', - terraformStateObjectSha256: 'c'.repeat(64), - principalEmail: 'fence-broker@example.gserviceaccount.com' - } - - await runMultiTargetDeployment(deploymentConfig, testHarness.overrides) - - assert.equal( - testHarness.events[0].event, - 'terraform_cell_fence_completed_attempt_recovered' - ) - assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') - }) -}) - -test('fails supersession on stale replacement heartbeat without calling its admin API', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - supersede: true, - replacementHeartbeatFresh: false - }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), - /no fresh matching director runtime snapshot/ - ) - assert.equal(testHarness.runtimeInspections.get('target2') ?? 0, 0) - assert.deepEqual(testHarness.events, []) - }) -}) - -test('fails supersession on mismatched director runtime without calling cell admin', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - supersede: true, - replacementRuntimeCellUrl: 'https://wrong.relay.example.com' - }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), - /no fresh matching director runtime snapshot/ - ) - assert.equal(testHarness.runtimeInspections.get('target2') ?? 0, 0) - assert.deepEqual(testHarness.events, []) - }) -}) - -test('reads the broker mutation token without treating the audience as the environment', async () => { - await withTopology(async (file) => { - const testHarness = harness({ supersede: true }) - delete testHarness.overrides.mutationIdentityToken - const previous = process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN - process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN = 'ddd.eee.fff' - try { - await runMultiTargetDeployment( - config(file, 'supersede-target'), - testHarness.overrides - ) - } finally { - if (previous === undefined) { - delete process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN - } else { - process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN = previous - } - } - assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') - }) -}) - -test('fails closed when the broker child has no mutation token', async () => { - await withTopology(async (file) => { - const testHarness = harness({ supersede: true }) - delete testHarness.overrides.mutationIdentityToken - const previous = process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN - delete process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN - try { - await assert.rejects( - runMultiTargetDeployment( - config(file, 'supersede-target'), - testHarness.overrides - ), - /requires a broker mutation identity token/ - ) - } finally { - if (previous !== undefined) { - process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN = previous - } - } - assert.equal(testHarness.events.length, 0) - }) -}) - -test('uses Terraform fencing for selector-era target supersession', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - supersede: true, - selectorMembership: { - existingOnly: ['source', 'target1'], - migrationOnly: ['target2'], - general: ['general'] - } - }) - await runMultiTargetDeployment( - config(file, 'supersede-target'), - testHarness.overrides - ) - assert.deepEqual(testHarness.stateChanges, []) - assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') - }) -}) - -test('resumes failed-target supersession after losing the resize response', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - supersede: true, - loseResizeResponse: true - }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), - /injected_resize_response_loss/ - ) - await runMultiTargetDeployment( - config(file, 'supersede-target'), - testHarness.overrides - ) - assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') - }) -}) - -test('refuses to fence a failed target while its admission remains enabled', async () => { - await withTopology(async (file) => { - const testHarness = harness({ supersede: true, failedTargetEnabled: true }) - await assert.rejects( - runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), - /failed target admission must be disabled/ - ) - assert.equal(testHarness.events.length, 0) - }) -}) - -test('retries until two complete target-capacity rounds agree', async () => { - await withTopology(async (file) => { - const testHarness = harness({ capacitySourceAssignments: [4, 3] }) - await runMultiTargetDeployment(config(file), testHarness.overrides) - assert.equal(testHarness.capacityReads(), 8) - assert.equal(testHarness.events.at(-1).sourceAssignments, 4) - }) -}) - -test('fails closed after bounded inconsistent target-capacity rounds', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - capacitySourceAssignments: Array.from( - { length: 12 }, - (_, index) => index % 2 === 0 ? 4 : 3 - ) - }) - await assert.rejects( - runMultiTargetDeployment(config(file), testHarness.overrides), - /target capacity snapshots disagree/ - ) - assert.equal(testHarness.capacityReads(), 12) - assert.deepEqual(testHarness.stateChanges, []) - }) -}) - -test('emits aggregate capacity inputs before rejecting insufficient headroom', async () => { - await withTopology(async (file) => { - const testHarness = harness({ - capacityRequiredTargetUnits: Array.from({ length: 4 }, () => 20_000) - }) - await assert.rejects( - runMultiTargetDeployment(config(file), testHarness.overrides), - /multi-target connection or request-unit headroom exhausted/ - ) - const snapshot = testHarness.events.at(-1) - assert.equal(snapshot.event, 'multi_target_capacity_snapshot') - assert.equal(snapshot.sourceConnections, 5) - assert.equal(snapshot.sourceAssignments, 4) - assert.equal(snapshot.requiredTargetUnits, 20_000) - assert.deepEqual(snapshot.targets, [ - { - cellId: 'target1', - currentConnections: 0, - availableConnectionReservations: 460, - availableTargetUnits: 4_000 - }, - { - cellId: 'target2', - currentConnections: 0, - availableConnectionReservations: 460, - availableTargetUnits: 4_000 - } - ]) - }) -}) - -test('rejects quotas above enforced target reservation headroom', () => { - assert.throws( - () => - allocateTargetQuotas({ - sourceAssignments: 3, - sourceConnections: 3, - requiredTargetUnits: 3, - connectionCeiling: 600, - targets: [ - { - cellId: 'target1', - currentConnections: 0, - availableConnectionReservations: 1, - availableTargetUnits: 10 - }, - { - cellId: 'target2', - currentConnections: 0, - availableConnectionReservations: 1, - availableTargetUnits: 10 - } - ] - }), - /multi-target connection or request-unit headroom exhausted/ - ) -}) diff --git a/cloud/dev/scripts/drive-relay-director-deploy.mjs b/cloud/dev/scripts/drive-relay-director-deploy.mjs deleted file mode 100644 index 060f35418e4..00000000000 --- a/cloud/dev/scripts/drive-relay-director-deploy.mjs +++ /dev/null @@ -1,1012 +0,0 @@ -// Operator-local driver for a production relay director deploy. It only dispatches the existing -// audited workflows and reads their results back; every safety check stays in those workflows. -// -// It keeps no state between runs. Every decision comes from live state read at the start: the -// serving director, its configured cells, and the rehome control. The one fact live state cannot -// show, that a pause is this driver's own, is the run that made it (`--pause-run`), checked -// against that run's log and the live generation. On any stop it prints the command that -// finishes the deploy from wherever it got to. -import { spawnSync } from 'node:child_process' -import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' -import { homedir, tmpdir } from 'node:os' -import { join, resolve } from 'node:path' -import { createInterface } from 'node:readline/promises' -import { pathToFileURL } from 'node:url' -import { verifyDryRunAuthority } from './relay-monitor-evidence.mjs' -import { - DIRECTOR_SERVICE, - IMAGE_REPOSITORY, - PROJECT, - REGION, - REPOSITORY, - WORKFLOW_REF, - WORKFLOWS, - admissionInspectInputs, - admissionInspectResult, - blocksDeploy, - configureInputs, - directorDeployInputs, - directorRevisions, - logConfirmsPublishedDigest, - monitorDryRunInputs, - parseConfigureWave, - pausedGeneration, - publishInputs, - rehomeEnableInputs, - rehomeInspectInputs, - rehomePauseInputs, - rehomeResultFromLog, - requireCommit, - requireDigest, - revisionDigest, - validateDispatchInputs -} from './relay-director-deploy-plan.mjs' - -const ACTIVE_RUN_STATUSES = ['queued', 'in_progress', 'waiting', 'requested', 'pending'] -// The enable job verifies the monitor within 5 minutes of completion after ~2.5 minutes of setup. -export const MONITOR_MAX_AGE_AT_ENABLE_MS = 150_000 -// The manual procedure watched the new director for 5+ minutes before configuring cells. -export const SOAK_MS = 5 * 60_000 -// Traffic moves about a minute before the deploy run completes (ops-log 05:00:33Z vs 05:01:34Z). -const TRAFFIC_SWITCH_LEAD_MS = 60_000 -// Request logs land up to a minute late; reading at the window's end would undercount it. -const LOG_INGESTION_LAG_MS = 60_000 -const DIRECTOR_5XX_FILTER = [ - 'resource.type="cloud_run_revision"', - `resource.labels.service_name="${DIRECTOR_SERVICE}"`, - `logName="projects/${PROJECT}/logs/run.googleapis.com%2Frequests"`, - 'httpRequest.status>=500' -].join(' AND ') -const LOG_COUNT_LIMIT = 5_000 -const LOG_ATTEMPTS = 6 -const LOG_INTERVAL_MS = 10_000 -const WATCH_INTERVAL_MS = 10_000 -// A run still going is reported this often, so a long monitor never looks hung. -const WATCH_REPORT_MS = 5 * 60_000 -const REHOME_HISTORY_RUNS = 5 -const RUN_ID = /^[1-9][0-9]*$/ - -export class DriverStop extends Error {} - -export function parseDriverArguments(argv, home = homedir()) { - const config = { - dryRun: false, - leaveRehomePaused: false, - configure: [], - logDirectory: join(home, '.orca', 'relay-director-deploy') - } - const runId = (value, key) => { - if (!RUN_ID.test(value)) throw new Error(`${key} must be a run ID`) - return Number(value) - } - for (let index = 0; index < argv.length; index += 1) { - const key = argv[index] - if (key === '--dry-run' || key === '--leave-rehome-paused') { - config[key === '--dry-run' ? 'dryRun' : 'leaveRehomePaused'] = true - continue - } - const value = argv[index + 1] - if (value === undefined || value.startsWith('--')) throw new Error(`${key} needs a value`) - index += 1 - if (key === '--commit') config.commit = requireCommit(value, '--commit') - else if (key === '--configure') config.configure.push(parseConfigureWave(value)) - else if (key === '--publish-run') config.publishRun = runId(value, key) - else if (key === '--pause-run') config.pauseRun = runId(value, key) - else if (key === '--rehome-generation') { - if (!/^(0|[1-9][0-9]*)$/.test(value)) throw new Error('--rehome-generation is invalid') - config.rehomeGeneration = Number(value) - } else if (key === '--log-directory') config.logDirectory = resolve(value) - else throw new Error(`unsupported argument ${key}`) - } - if (!config.commit) throw new Error('missing --commit <40-character main commit to publish>') - if (config.pauseRun && config.leaveRehomePaused) { - throw new Error('--pause-run and --leave-rehome-paused contradict each other') - } - return config -} - -const timestamp = (ms) => new Date(ms).toISOString() -const runUrl = (runId) => `https://github.com/${REPOSITORY}/actions/runs/${runId}` -// Argument lists whose values never contain spaces. -const words = (text) => text.split(' ') - -export function createDriver(config, deps) { - let logPath - const typedPhrases = new Map() - // Everything learned this run; nothing outlives the process. - const known = { director: undefined, selector: undefined, control: undefined } - let rollbackPoint - let published = config.publishRun ? { runId: config.publishRun } : undefined - let owned // { generation, runId }: the pause this driver made, the only rehome state it owns - // A pause or enable this run cannot vouch for: `changing` while it may still apply on its own, - // `unconfirmed` once it finished without a usable result. { kind, name, runId?, url? } - let uncertain - let movedBuild // { commit, runId }: a publish that built a newer main than the reviewed commit - let login - const ownRunIds = new Set() - let enabled = false - - function log(message) { - const line = `${timestamp(deps.now())} ${message}` - deps.print(line) - if (logPath) appendFileSync(logPath, `${line}\n`) - } - - function command(program, args, input) { - const result = deps.run(program, args, input) - if (result.status !== 0) { - const detail = String(result.stderr ?? '') - .trim() - .split('\n') - .slice(-3) - .join(' | ') - throw new Error(`${program} ${args.slice(0, 4).join(' ')} failed: ${detail}`) - } - return String(result.stdout ?? '') - } - - const gh = (args, input) => command('gh', args, input) - const ghJson = (args) => JSON.parse(gh(args)) - const gcloudJson = (args) => - JSON.parse(command('gcloud', [...args, '--project', PROJECT, '--format=json'])) - - function readDirector() { - const service = gcloudJson( - words(`run services describe ${DIRECTOR_SERVICE} --region ${REGION}`) - ) - const { servingRevision, rollbackRevision } = directorRevisions(service) - const describe = (revision) => - gcloudJson(words(`run revisions describe ${revision} --region ${REGION}`)) - const serving = describe(servingRevision) - const cellsJson = serving.spec.containers[0].env?.find( - (variable) => variable.name === 'ORCA_RELAY_CELLS_JSON' - )?.value - return { - servingRevision, - servingDigest: revisionDigest(serving, `revision ${servingRevision}`), - servingCreatedAt: serving.metadata?.creationTimestamp, - cells: new Set(JSON.parse(cellsJson ?? '[]').map((cell) => cell.id)), - rollbackRevision, - rollbackDigest: revisionDigest(describe(rollbackRevision), `revision ${rollbackRevision}`) - } - } - - function describeDirector(director) { - return `serving ${director.servingRevision} ${director.servingDigest}; rollback ${director.rollbackRevision} ${director.rollbackDigest}` - } - - // Paginated per status: the repository-wide first page can hide an in-flight relay run. - function activeRuns() { - const active = [] - for (const status of ACTIVE_RUN_STATUSES) { - const lines = gh( - words( - `api --paginate -X GET repos/${REPOSITORY}/actions/runs -f status=${status} -f per_page=100 --jq .workflow_runs[]|{id,path,name,status}` - ) - ) - for (const run of lines.split('\n').filter(Boolean).map(JSON.parse)) { - if (blocksDeploy(run.path) && !ownRunIds.has(run.id)) - active.push(`${run.name} ${runUrl(run.id)} (${run.status})`) - } - } - return active - } - - function requireQuietLane() { - const active = activeRuns() - if (active.length > 0) { - throw new DriverStop( - `relay workflows are in flight; wait for them:\n ${active.join('\n ')}` - ) - } - } - - function viewRun(runId) { - return ghJson( - words( - `run view ${runId} -R ${REPOSITORY} --json status,conclusion,attempt,headSha,headBranch,event,workflowName` - ) - ) - } - - // One line per status change and one every WATCH_REPORT_MS, never a stream of job steps. - async function waitForRun(run) { - const started = deps.now() - let reported - let reportedAt - for (;;) { - const view = viewRun(run.runId) - if (view.status === 'completed') { - log(`${run.name}: ${view.conclusion} ${run.url}`) - return { ...run, conclusion: view.conclusion, attempt: view.attempt } - } - if (view.status !== reported || deps.now() - reportedAt >= WATCH_REPORT_MS) { - const minutes = Math.floor((deps.now() - started) / 60_000) - log(`${run.name}: ${view.status}, ${minutes} min`) - reported = view.status - reportedAt = deps.now() - } - await deps.sleep(WATCH_INTERVAL_MS) - } - } - - // Dispatches and watches one workflow run. The run ID comes only from the URL `gh workflow run` - // prints, so another dispatch by the same account can never be mistaken for this one. - async function dispatch({ name, workflow, inputs: build, changesRehome = false }) { - const inputs = validateDispatchInputs(build(view())) - // Lets a signal that arrived during synchronous work stop the driver before it dispatches. - await new Promise((resolveYield) => setImmediate(resolveYield)) - requireQuietLane() - log(`dispatch ${name}: ${workflow.file} ${JSON.stringify(inputs)}`) - if (changesRehome) uncertain = { kind: 'changing', name } - const output = gh( - words(`workflow run ${workflow.file} -R ${REPOSITORY} --ref ${WORKFLOW_REF} --json`), - JSON.stringify(inputs) - ) - const printed = output.match(/\/actions\/runs\/([0-9]+)/) - if (!printed) { - throw new DriverStop( - `gh printed no run URL for ${workflow.file}; upgrade gh. The run may exist: find it before re-running` - ) - } - const runId = Number(printed[1]) - const url = runUrl(runId) - ownRunIds.add(runId) - if (changesRehome) Object.assign(uncertain, { runId, url }) - log(`${name}: run ${url}`) - const dispatched = viewRun(runId) - if ( - dispatched.workflowName !== workflow.name || - dispatched.event !== 'workflow_dispatch' || - dispatched.headBranch !== WORKFLOW_REF - ) { - throw new DriverStop( - `run ${runUrl(runId)} is not a ${workflow.file} dispatch on ${WORKFLOW_REF}` - ) - } - // `uncertain` is cleared by the step only once it has read the run's result. - return await waitForRun({ name, runId, url, headSha: dispatched.headSha }) - } - - function requireSuccess(run) { - if (run.conclusion !== 'success') { - throw new DriverStop(`${run.name} run ${run.url} concluded ${run.conclusion}`) - } - } - - async function runLog(runId) { - for (let attempt = 1; ; attempt += 1) { - try { - const text = gh(words(`run view ${runId} -R ${REPOSITORY} --log`)) - if (text.trim()) return text - } catch (error) { - if (attempt >= LOG_ATTEMPTS) - throw new Error(`log for ${runUrl(runId)} is unavailable: ${error.message}`) - } - if (attempt >= LOG_ATTEMPTS) throw new Error(`log for ${runUrl(runId)} is empty`) - await deps.sleep(LOG_INTERVAL_MS) - } - } - - async function rehomeResult(runId) { - try { - return rehomeResultFromLog(await runLog(runId)) - } catch { - return undefined - } - } - - async function withArtifact(runId, artifact, read) { - const directory = mkdtempSync(join(tmpdir(), 'relay-director-deploy-')) - try { - gh(words(`run download ${runId} -R ${REPOSITORY} -n ${artifact} -D ${directory}`)) - return await read(directory) - } finally { - rmSync(directory, { recursive: true, force: true }) - } - } - - async function typed(phrase, meaning = '') { - if (typedPhrases.has(phrase)) return phrase - // Ends in a newline, so the prompt is never left mid-line where it can be missed. - const answer = (await deps.prompt(`Type ${phrase} to continue${meaning}:\n`)).trim() - if (answer !== phrase) - throw new DriverStop(`expected ${phrase}; nothing further was dispatched`) - typedPhrases.set(phrase, answer) - log(`operator typed ${phrase}`) - return answer - } - - // The digest the publish run pushed: the registry's digest for the commit's tag, which the push - // line in the run's own log must name too. - async function publishedDigest(runId) { - const view = viewRun(runId) - if (view.workflowName !== WORKFLOWS.publish.name || view.conclusion !== 'success') { - throw new DriverStop(`${runUrl(runId)} is not a successful ${WORKFLOWS.publish.file} run`) - } - if (view.headSha !== config.commit) { - movedBuild = { commit: view.headSha, runId } - throw new DriverStop( - `publish ${runUrl(runId)} built ${view.headSha}, not the reviewed ${config.commit}: main moved` - ) - } - const tag = `${IMAGE_REPOSITORY}:sha-${config.commit}` - const digest = command( - 'gcloud', - words( - `artifacts docker images describe ${tag} --project ${PROJECT} --format=value(image_summary.digest)` - ) - ).trim() - requireDigest(digest, 'registry digest of the published tag') - if (!logConfirmsPublishedDigest(await runLog(runId), config.commit, digest)) { - throw new DriverStop( - `registry digest ${digest} is not the digest ${runUrl(runId)} pushed; the tag moved` - ) - } - return digest - } - - async function lastKnownRehomeGeneration() { - const runs = ghJson( - words( - `run list -R ${REPOSITORY} --workflow ${WORKFLOWS.rehome.file} --status completed --limit ${REHOME_HISTORY_RUNS} --json databaseId` - ) - ) - for (const run of runs) { - const result = await rehomeResult(run.databaseId) - if (result) { - log( - `rehome generation candidate ${result.control.generation} from ${runUrl(run.databaseId)}` - ) - return result.control.generation - } - } - throw new DriverStop( - 'no recent rehome run printed the control generation; pass --rehome-generation' - ) - } - - // Values every input builder reads. A dry run shows what is not known yet as ``, - // which validateDispatchInputs refuses, so a placeholder can never be dispatched. - function view() { - const unknown = (label) => `<${label}>` - const placeholder = (label) => new Proxy({}, { get: () => unknown(label) }) - const fromAdmission = [unknown('from admission inspect')] - const control = known.control - return { - director: known.director, - selector: known.selector ?? { - generation: fromAdmission[0], - membership: new Proxy({}, { get: () => fromAdmission }) - }, - control: control ?? placeholder('inspected'), - pausedGeneration: - owned?.generation ?? - (control?.enabled === false - ? control.generation - : unknown('rehome generation after pause')), - published: published?.digest ?? unknown('published digest'), - rollbackDigest: rollbackPoint?.digest ?? known.director?.servingDigest, - monitor: placeholder('monitor run'), - afterDeploy: placeholder('digest read from gcloud after the deploy'), - notBefore: config.dryRun ? unknown('now, epoch ms') : Math.floor(deps.now() / 1000) * 1000, - confirmation: (phrase) => - typedPhrases.has(phrase) ? phrase : unknown(`operator types ${phrase}`) - } - } - - const deployPending = () => known.director.servingDigest !== published?.digest - const pendingWaves = () => - config.configure.filter((wave) => wave.cells.some((cell) => !known.director.cells.has(cell))) - const work = () => deployPending() || pendingWaves().length > 0 - - function count5xx(fromMs, toMs) { - const filter = `${DIRECTOR_5XX_FILTER} AND timestamp>="${timestamp(fromMs)}" AND timestamp<"${timestamp(toMs)}"` - return command('gcloud', [ - 'logging', - 'read', - filter, - ...words(`--project ${PROJECT} --limit ${LOG_COUNT_LIMIT} --format=value(timestamp)`) - ]) - .split('\n') - .filter(Boolean).length - } - - // Director 5xx over SOAK_MS on the new image against the same span before its revision existed. - // The window starts at the traffic switch when this run deployed, otherwise now, so a re-run - // after a tripped soak judges fresh traffic. - async function soak(deployedAt) { - const created = Date.parse(known.director.servingCreatedAt) - const start = - deployedAt === undefined ? deps.now() : Math.max(deployedAt - TRAFFIC_SWITCH_LEAD_MS, created) - const end = start + SOAK_MS - const readAt = end + LOG_INGESTION_LAG_MS - if (deps.now() < readAt) { - log( - `soak: watching the new director until ${timestamp(end)}, reading at ${timestamp(readAt)}` - ) - await deps.sleep(readAt - deps.now()) - } - const before = count5xx(created - SOAK_MS, created) - const after = count5xx(start, end) - log( - `soak: director 5xx ${after} in ${SOAK_MS / 60_000} min on the new image, ${before} before it` - ) - if (after >= LOG_COUNT_LIMIT || after > 2 * before + 25) { - throw new DriverStop( - `director 5xx rose from ${before} to ${after} after the deploy; investigate before configuring cells` - ) - } - } - - // The same audited check the enable job runs, on the same sealed files. - async function monitorCompletedAt(run) { - const incidentId = `relay-${run.runId}-dry-run` - return await withArtifact( - run.runId, - `relay-monitor-dry-run-${run.runId}-${run.attempt}`, - async (directory) => { - try { - const argv = Object.entries({ - directory, - 'incident-id': incidentId, - 'run-id': run.runId, - 'run-attempt': run.attempt, - 'commit-sha': run.headSha, - mode: 'dry-run', - 'required-migration-policy': 'strict' - }).flatMap(([key, value]) => [`--${key}`, String(value)]) - const { state } = await verifyDryRunAuthority(argv, deps.now) - log(`monitor GREEN, completed ${state.completedAt}`) - return state.completedAt - } catch (error) { - // The freeze reason lives only in the sealed state, never in the run log. - const sealed = (() => { - try { - return JSON.parse(readFileSync(join(directory, `${incidentId}.state.json`), 'utf8')) - } catch { - return {} - } - })() - const failures = (sealed.failures ?? []).map((failure) => - [ - failure.source, - failure.code, - failure.signal, - failure.observed, - failure.threshold - ].join(' ') - ) - throw new DriverStop( - [ - `monitor ${run.url} is not usable: ${error.message}`, - `frozenAt ${sealed.frozenAt}`, - ...failures - ].join('\n ') - ) - } - } - ) - } - - async function preflight() { - log('preflight: relay workflow lane, target commit, serving director') - if (config.dryRun) { - const active = activeRuns() - if (active.length > 0) - log( - `WARNING relay workflows are in flight; a real run would stop:\n ${active.join('\n ')}` - ) - } else { - requireQuietLane() - } - let claim - if (config.pauseRun) { - // Only this operator's own rehome-control run can prove a pause belongs to this driver. - const claimed = ghJson( - words( - `api repos/${REPOSITORY}/actions/runs/${config.pauseRun} --jq {path,actor:.triggering_actor.login}` - ) - ) - if ( - !claimed.path?.split('@')[0].endsWith(`/${WORKFLOWS.rehome.file}`) || - claimed.actor !== login - ) { - throw new DriverStop( - `${runUrl(config.pauseRun)} is not a ${WORKFLOWS.rehome.file} run by ${login}, so it cannot prove a pause is this driver's` - ) - } - const result = await rehomeResult(config.pauseRun) - claim = result && pausedGeneration(result) - if (claim === undefined) { - throw new DriverStop( - `${runUrl(config.pauseRun)} printed no pause of its own, so it cannot prove a pause is this driver's` - ) - } - } - if (published) { - published.digest = await publishedDigest(published.runId) - } else { - const main = gh(['api', `repos/${REPOSITORY}/commits/${WORKFLOW_REF}`, '--jq', '.sha']).trim() - if (main !== config.commit) { - throw new DriverStop( - `${WORKFLOW_REF} is at ${main}, not the reviewed ${config.commit}; review the difference and run with --commit ${main}` - ) - } - // The workflow builds main's head at dispatch, so it is dispatched seconds after the check - // rather than after the inspects and the typed phrase; publishing changes nothing serving. - if (!config.dryRun) await publishStep.run(publishStep) - } - known.director = readDirector() - if (known.director.servingDigest !== published?.digest) { - rollbackPoint = { - revision: known.director.servingRevision, - digest: known.director.servingDigest - } - } - log(describeDirector(known.director)) - // A director safety pause moves the generation without a run; the inspect then fails closed. - const generation = config.rehomeGeneration ?? (await lastKnownRehomeGeneration()) - if (config.dryRun) { - known.control = undefined - return generation - } - const [admissionStep, inspectStep] = readSteps(generation) - const admission = await dispatch(admissionStep) - requireSuccess(admission) - known.selector = await withArtifact( - admission.runId, - `relay-asia-admission-result-${admission.runId}-${admission.attempt}`, - (directory) => - admissionInspectResult(JSON.parse(readFileSync(join(directory, 'result.json'), 'utf8'))) - ) - const inspect = await dispatch(inspectStep) - if (inspect.conclusion !== 'success') { - throw new DriverStop( - `rehome inspect at generation ${generation} failed (${inspect.url}): the generation, selector or a digest moved. A director safety pause bumps the generation. Read the log, then pass --rehome-generation` - ) - } - known.control = (await rehomeResult(inspect.runId))?.control - if (!known.control) throw new DriverStop(`rehome inspect ${inspect.url} printed no control`) - const control = known.control - log( - `rehome: generation ${control.generation}, ${control.enabled ? 'ENABLED' : 'disabled'}; selector generation ${known.selector.generation}` - ) - if (claim !== undefined) { - if (!control.enabled && control.generation === claim) - owned = { generation: claim, runId: config.pauseRun } - else if (control.enabled && control.generation === claim + 1) enabled = true - else - throw new DriverStop( - `rehome is generation ${control.generation} ${control.enabled ? 'enabled' : 'paused'}, not the pause ${runUrl(config.pauseRun)} made at ${claim}. Something else changed it, such as a director safety pause; this driver will not touch it` - ) - } else if (control.enabled && config.leaveRehomePaused) { - throw new DriverStop( - 'rehome is enabled; --leave-rehome-paused accepts only a paused switch. Drop it' - ) - } else if (!control.enabled && !config.leaveRehomePaused) { - throw new DriverStop( - `rehome is paused at generation ${control.generation}, and this run did not pause it. If an earlier run of this driver did, re-run with the --pause-run its log printed. Otherwise pass --leave-rehome-paused to deploy and leave it paused` - ) - } - if (control.hostCooldownMs === undefined && (control.enabled || owned)) { - throw new DriverStop( - 'the director reports no per-host rehome cooldown, so enable would refuse; not pausing' - ) - } - return generation - } - - // The two read-only inspects every real run starts with; never resumed, always run fresh. - function readSteps(generation) { - return [ - { - name: 'preflight-admission', - workflow: WORKFLOWS.admission, - inputs: (v) => admissionInspectInputs(v.director.servingDigest) - }, - { - name: 'preflight-rehome', - workflow: WORKFLOWS.rehome, - inputs: (v) => rehomeInspectInputs({ ...v, controlGeneration: generation }) - } - ] - } - - const publishStep = { - name: 'publish', - workflow: WORKFLOWS.publish, - inputs: publishInputs, - when: () => !published, - run: async (step) => { - const run = await dispatch(step) - requireSuccess(run) - published = { runId: run.runId } - published.digest = await publishedDigest(run.runId) - log(`published ${IMAGE_REPOSITORY}@${published.digest}`) - } - } - - // The one ordered plan. `when` reads live state, so a re-run skips what is already done; a dry run - // prints every step whose need it cannot know yet. - function plan() { - let deployedAt - let verified - let monitor - return [ - publishStep, - { - name: 'pause', - workflow: WORKFLOWS.rehome, - changesRehome: true, - inputs: (v) => - rehomePauseInputs({ ...v, confirmation: v.confirmation('PAUSE_REGIONAL_REHOMING') }), - when: () => known.control?.enabled === true && !owned && work(), - run: async (step) => { - await typed('PAUSE_REGIONAL_REHOMING') - const run = await dispatch(step) - const result = await rehomeResult(run.runId) - const generation = result && pausedGeneration(result) - if (generation !== known.control.generation + 1) { - uncertain.kind = 'unconfirmed' - throw new DriverStop( - `${run.url} (${run.conclusion}) printed no pause at generation ${known.control.generation + 1}` - ) - } - owned = { generation, runId: run.runId } - uncertain = undefined - log( - `REHOME PAUSED at generation ${generation}. To finish from here after any stop: ${rerunCommand()}` - ) - } - }, - { - name: 'deploy', - workflow: WORKFLOWS.director, - inputs: (v) => - directorDeployInputs({ - imageDigest: v.published, - predecessorDigest: v.rollbackDigest, - rehomeGeneration: v.pausedGeneration - }), - when: () => !published || deployPending(), - run: async (step) => { - requireSuccess(await dispatch(step)) - deployedAt = deps.now() - known.director = readDirector() - if (deployPending()) - throw new DriverStop( - `deploy: ${describeDirector(known.director)}, not ${published.digest}` - ) - } - }, - ...config.configure.slice(0, 1).map(() => ({ - name: 'soak', - description: `wait ${SOAK_MS / 60_000} min, then compare director 5xx`, - // A wave already configured means an earlier run passed the soak on this image. - when: () => pendingWaves().length === config.configure.length, - run: () => soak(deployedAt) - })), - ...config.configure.map((wave) => ({ - name: `configure:${wave.cells.join(',')}`, - workflow: WORKFLOWS.admission, - inputs: (v) => - configureInputs({ - ...wave, - directorDigest: v.published, - selectorGeneration: v.selector.generation, - confirmation: v.confirmation('CONFIGURE_ASIA_DIRECTOR') - }), - when: () => pendingWaves().includes(wave), - run: async (step) => { - await typed('CONFIGURE_ASIA_DIRECTOR') - requireSuccess(await dispatch(step)) - known.director = readDirector() - if (deployPending()) - throw new DriverStop( - `${step.name}: ${describeDirector(known.director)}, not ${published.digest}` - ) - } - })), - // Inspect binds the exact serving and rollback digests, so a wrong one fails here, read-only, - // before 15 minutes of monitor evidence is spent on it. - { - name: 'verify-identities', - workflow: WORKFLOWS.rehome, - inputs: (v) => - rehomeInspectInputs({ - director: verified ?? v.afterDeploy, - selector: v.selector, - controlGeneration: v.pausedGeneration - }), - when: () => Boolean(owned), - run: async (step) => { - verified = readDirector() - const run = await dispatch(step) - const control = (await rehomeResult(run.runId))?.control - if ( - run.conclusion !== 'success' || - control?.enabled !== false || - control.generation !== owned.generation - ) { - throw new DriverStop( - `verify-identities ${run.url} (${run.conclusion}) did not find rehome paused at ${owned.generation} with ${describeDirector(verified)}` - ) - } - } - }, - { - name: 'monitor', - workflow: WORKFLOWS.monitor, - inputs: (v) => monitorDryRunInputs(v.selector), - when: () => Boolean(owned), - run: async (step) => { - // The operator arms the enable before the 15-minute watch; it still dispatches only on green. - await typed( - 'ENABLE_REGIONAL_REHOMING', - ` (this arms an automatic enable, sent about 17 min from now and only if the monitor is green and its evidence is at most ${MONITOR_MAX_AGE_AT_ENABLE_MS / 1000} s old)` - ) - const run = await dispatch(step) - requireSuccess(run) - monitor = { - runId: run.runId, - attempt: run.attempt, - completedAt: await monitorCompletedAt(run) - } - } - }, - { - name: 'enable', - workflow: WORKFLOWS.rehome, - changesRehome: true, - inputs: (v) => - rehomeEnableInputs({ - ...v, - director: verified ?? v.afterDeploy, - controlGeneration: v.pausedGeneration, - monitor: monitor ?? v.monitor, - confirmation: v.confirmation('ENABLE_REGIONAL_REHOMING') - }), - when: () => Boolean(owned), - run: async (step) => { - const ageMs = deps.now() - Date.parse(monitor.completedAt) - if (ageMs > MONITOR_MAX_AGE_AT_ENABLE_MS) { - throw new DriverStop( - `monitor evidence is ${Math.round(ageMs / 1000)} s old, past the ${MONITOR_MAX_AGE_AT_ENABLE_MS / 1000} s budget; re-run for a fresh monitor` - ) - } - const now = readDirector() - if ( - now.servingDigest !== verified.servingDigest || - now.rollbackDigest !== verified.rollbackDigest - ) { - throw new DriverStop( - `the director changed after its digests were verified: ${describeDirector(now)}` - ) - } - if (known.control.ratePerMinute !== 10) - log( - `enable starts at the job's fixed 10 hosts/min (was ${known.control.ratePerMinute})` - ) - const run = await dispatch(step) - const result = await rehomeResult(run.runId) - if (result) known.control = result.control - // A failed run is never an enable, whatever it printed last. - if ( - run.conclusion === 'success' && - result?.mode === 'enable' && - result.control.enabled && - result.control.generation === owned.generation + 1 - ) { - owned = undefined - uncertain = undefined - enabled = true - log(`REHOME RE-ENABLED at generation ${result.control.generation}`) - return - } - if (result?.mode !== 'recover-enable') { - uncertain.kind = 'unconfirmed' - throw new DriverStop(`${run.url} (${run.conclusion}) printed no enable result`) - } - // Recovery that disabled rehome itself is this driver's pause too. Recovery that found it - // already disabled at another generation found someone else's pause, such as a director - // safety pause: this driver gives up ownership and will never lift it. - uncertain = undefined - const recovered = pausedGeneration(result) - if (recovered !== undefined) owned = { generation: recovered, runId: run.runId } - else if (result.control.generation !== owned.generation) owned = undefined - throw new DriverStop( - `enable ${run.url} failed; the job's recovery left rehome paused at generation ${result.control.generation}` - ) - } - } - ] - } - - function rerunCommand( - pauseRun = owned?.runId, - build = published?.digest ? { commit: config.commit, runId: published.runId } : undefined - ) { - return [ - 'node dev/scripts/drive-relay-director-deploy.mjs', - `--commit ${build?.commit ?? config.commit}`, - ...(build ? [`--publish-run ${build.runId}`] : []), - ...(pauseRun ? [`--pause-run ${pauseRun}`] : []), - ...(config.leaveRehomePaused ? ['--leave-rehome-paused'] : []), - ...config.configure.map( - (wave) => `--configure ${wave.cells.join(',')}=${wave.cellImageDigest}` - ) - ].join(' ') - } - - function stopReport(reason) { - const lines = [`STOPPED: ${reason}`, '', 'State now:'] - const workflowPage = `https://github.com/${REPOSITORY}/actions/workflows/${WORKFLOWS.rehome.file}` - const where = uncertain?.url ?? `(gh printed no URL; find it at ${workflowPage})` - if (uncertain?.name === 'pause') { - lines.push( - uncertain.kind === 'changing' - ? `- *** REHOME IS CHANGING: the pause run ${where} was dispatched and applies on its own, if it has not already. ***` - : `- *** PAUSE UNCONFIRMED: ${where} printed no pause. Rehome MAY BE PAUSED. ***`, - ' Inspect rehome before walking away.' - ) - if (uncertain.runId) { - lines.push( - ` If it paused rehome at generation ${known.control.generation + 1}, finish with: cd cloud && ${rerunCommand(uncertain.runId)}` - ) - } - } else if (uncertain?.name === 'enable') { - lines.push( - uncertain.kind === 'changing' - ? `- *** REHOME IS CHANGING: the enable run ${where} applies on its own: it enables rehome, or disables it again if it fails. ***` - : `- *** ENABLE UNCONFIRMED: ${where} did not confirm the enable. Rehome may be enabled, or paused. ***`, - ` If it enabled rehome, or failed before applying, finish with: cd cloud && ${rerunCommand(owned.runId)}` - ) - if (uncertain.runId) { - lines.push( - ` If its recovery paused rehome again, finish with: cd cloud && ${rerunCommand(uncertain.runId)}` - ) - } - } else if (owned) { - lines.push( - `- *** REHOME IS PAUSED by this driver at generation ${owned.generation} (${runUrl(owned.runId)}). It stays paused until the re-run below enables it. ***` - ) - } else if (known.control) { - const state = - enabled || known.control.enabled - ? 'enabled' - : 'PAUSED, not by this driver; it will not be re-enabled here' - lines.push(`- rehome: generation ${known.control.generation} as last read, ${state}`) - } else { - lines.push('- rehome: not read; this run did not change it') - } - try { - lines.push(`- director now: ${describeDirector(readDirector())}`) - } catch (error) { - lines.push(`- director: could not re-read (${error.message})`) - } - if (published?.digest) - lines.push(`- published: ${published.digest} (${runUrl(published.runId)})`) - if (rollbackPoint) { - lines.push( - `- rollback point: ${rollbackPoint.revision} ${rollbackPoint.digest}. To undo the deploy, while rehome is paused:` - ) - lines.push( - ` ${ghCommand(WORKFLOWS.director, directorDeployInputs({ imageDigest: rollbackPoint.digest, predecessorDigest: rollbackPoint.digest, rehomeGeneration: owned?.generation ?? '' }))}` - ) - } - if (movedBuild) { - // Re-running the reviewed commit would only build the moved main again. - lines.push( - '', - `Review ${config.commit}..${movedBuild.commit}, then deploy that build without rebuilding:`, - ` cd cloud && ${rerunCommand(undefined, movedBuild)}` - ) - } else if (!config.dryRun && !uncertain) { - lines.push( - '', - `Re-run to finish from here (it re-reads everything and skips what is done):`, - ` cd cloud && ${rerunCommand()}` - ) - } - return lines.join('\n') - } - - function ghCommand(workflow, inputs) { - const fields = Object.entries(inputs).map(([key, value]) => `-f ${key}=${value}`) - return `gh workflow run ${workflow.file} -R ${REPOSITORY} --ref ${WORKFLOW_REF} ${fields.join(' ')}` - } - - async function run() { - const stamp = timestamp(deps.now()).replace(/[:.]/g, '-') - mkdirSync(config.logDirectory, { recursive: true, mode: 0o700 }) - logPath = join( - config.logDirectory, - `${stamp}-${config.commit.slice(0, 12)}${config.dryRun ? '-dry-run' : ''}.log` - ) - log(`${config.dryRun ? 'dry run' : 'start'}: ${rerunCommand()}`) - try { - login = gh(['api', 'user', '--jq', '.login']).trim() - const generation = await preflight() - const steps = plan() - for (const step of [...(config.dryRun ? readSteps(generation) : []), ...steps]) { - const needed = config.dryRun || step.when() ? '' : ' (done or not needed)' - const detail = step.workflow - ? `${step.workflow.file} ${JSON.stringify(step.inputs({ ...view(), control: { ...view().control, generation } }))}` - : step.description - log(`plan ${step.name}${needed}: ${detail}`) - } - if (config.dryRun) { - log('dry run: nothing dispatched') - return { dryRun: true } - } - if (steps.some((step) => step.when())) await typed(`DEPLOY ${config.commit.slice(0, 12)}`) - for (const step of steps) if (step.when()) await step.run(step) - const rehome = - enabled || known.control.enabled ? 'enabled' : 'left paused (--leave-rehome-paused)' - log( - `DONE: ${describeDirector(known.director)}; rehome ${rehome}${rollbackPoint ? `; rollback point ${rollbackPoint.revision} ${rollbackPoint.digest}` : ''}` - ) - return { done: true, logPath } - } catch (error) { - for (const line of stopReport(error.message).split('\n')) log(line) - throw Object.assign(error instanceof DriverStop ? error : new DriverStop(error.message), { - reported: true, - logPath - }) - } - } - - // Ctrl-C, SIGTERM or a closed terminal still leaves the operator the live state and the re-run. - function interrupt(signal) { - if (logPath) for (const line of stopReport(`interrupted by ${signal}`).split('\n')) log(line) - } - - return { run, interrupt } -} - -function defaultDependencies() { - return { - run: (program, args, input) => - spawnSync(program, args, { - encoding: 'utf8', - input, - maxBuffer: 256 * 1024 * 1024, - stdio: ['pipe', 'pipe', 'pipe'] - }), - now: () => Date.now(), - sleep: (ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)), - print: (line) => process.stdout.write(`${line}\n`), - prompt: async (question) => { - const reader = createInterface({ input: process.stdin, output: process.stdout }) - try { - return await reader.question(question) - } finally { - reader.close() - } - } - } -} - -export async function main(argv = process.argv.slice(2), deps = defaultDependencies()) { - const driver = createDriver(parseDriverArguments(argv), deps) - for (const [signal, code] of [ - ['SIGINT', 130], - ['SIGTERM', 143], - ['SIGHUP', 129] - ]) { - process.once(signal, () => { - driver.interrupt(signal) - process.exit(code) - }) - } - return await driver.run() -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - if (!error.reported) - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/drive-relay-director-deploy.test.mjs b/cloud/dev/scripts/drive-relay-director-deploy.test.mjs deleted file mode 100644 index db80ef22c7b..00000000000 --- a/cloud/dev/scripts/drive-relay-director-deploy.test.mjs +++ /dev/null @@ -1,1090 +0,0 @@ -import assert from 'node:assert/strict' -import { createHash } from 'node:crypto' -import { mkdtempSync, readFileSync, readdirSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { test } from 'node:test' -import { - MONITOR_MAX_AGE_AT_ENABLE_MS, - createDriver, - parseDriverArguments -} from './drive-relay-director-deploy.mjs' -import { - IMAGE_REPOSITORY, - REPOSITORY, - WORKFLOWS, - blocksDeploy, - parseConfigureWave, - pausedGeneration, - rehomeResultFromLog, - validateDispatchInputs -} from './relay-director-deploy-plan.mjs' -import { RELAY_WORKFLOW_FILE_PREFIX, relayWorkflowPath } from './relay-repository.mjs' - -const COMMIT = 'a'.repeat(40) -const OLD = `sha256:${'1'.repeat(64)}` -const NEW = `sha256:${'2'.repeat(64)}` -const CELL = `sha256:${'3'.repeat(64)}` -const START = Date.parse('2026-10-05T04:50:00Z') -const MEMBERSHIP = { - existingOnly: ['production-gce-c1'], - migrationOnly: ['production-gce-c34'], - general: ['production-gce-c27', 'production-gce-c7'] -} -const CONTROL = { - generation: 39, - enabled: true, - observationStartedAt: 1_786_687_676_179, - notBefore: 1_790_934_023_000, - ratePerMinute: 10, - preferenceMaxAgeMs: 86_400_000, - hostCooldownMs: 604_800_000, - drainGraceMs: 3_600_000 -} - -function controlLine(mode, control, extra = {}) { - const result = { event: 'relay_regional_rehome_control', mode, ...extra, control } - return `operate / control\tUNKNOWN STEP\t2026-10-05T04:51:27Z ${JSON.stringify(result)}` -} - -function monitorFiles(run, now, overrides = {}) { - const incidentId = `relay-${run.id}-dry-run` - const at = (offset) => new Date(now - offset).toISOString() - const state = JSON.stringify({ - schemaVersion: 4, - incidentId, - environment: 'production', - preDrainDryRun: true, - migrationPolicy: 'strict', - recoverySourceCellId: null, - capacityCellId: null, - durationMinutes: 15, - intervalMs: 60_000, - sampleCount: 16, - frozenAt: null, - failures: [], - startedAt: at(16 * 60_000), - windowStartedAt: at(15 * 60_000), - lastSampleAt: at(0), - completedAt: at(0), - ...overrides - }) - const manifest = { - schemaVersion: 1, - incidentId, - runId: String(run.id), - runAttempt: 1, - commitSha: run.headSha, - mode: 'dry-run', - files: { [`${incidentId}.state.json`]: createHash('sha256').update(state).digest('hex') } - } - return { - [`${incidentId}.state.json`]: state, - 'evidence-manifest.json': JSON.stringify(manifest) - } -} - -// A model of GitHub Actions and Cloud Run: each dispatch does what the real workflow would do to -// `world`, including the side effects a failing run leaves behind. -function fakeWorld(overrides = {}) { - const world = { - now: START, - main: COMMIT, - nextRunId: 37_000_000_000, - runs: [], - active: [], - printUrl: true, - revision: 700, - serving: { revision: 'orca-cloud-relay-00700-qor', digest: OLD, createdAt: START - 86_400_000 }, - rollback: { revision: 'orca-cloud-relay-00699-das', digest: OLD }, - cells: ['production-gce-c1'], - registry: {}, - selector: { generation: 345, attemptId: 'x', membership: MEMBERSHIP }, - control: CONTROL, - publishedDigest: NEW, - pushLogDigest: undefined, - fail: {}, - monitorState: {}, - director5xx: () => 10, - prompts: [], - answer: (question) => question.match(/^Type (\S+(?: [0-9a-f]{12})?) to continue/)[1], - printed: [], - ...overrides - } - // The last rehome run before this deploy, where the driver finds the current generation. - world.runs.push({ - id: 1, - file: WORKFLOWS.rehome.file, - conclusion: 'success', - log: controlLine('enable', world.control) - }) - world.dispatches = () => world.runs.filter((run) => run.dispatched) - return world -} - -function promote(world, digest) { - world.revision += 5 - world.rollback = { revision: `orca-cloud-relay-00${world.revision - 1}-rbk`, digest } - world.serving = { - revision: `orca-cloud-relay-00${world.revision}-new`, - digest, - createdAt: world.now - } -} - -const nextGeneration = (world, enabled, step = 1) => { - world.control = { ...world.control, generation: world.control.generation + step, enabled } -} - -function simulate(world, run) { - const { inputs } = run - const failure = world.fail[run.key] - delete world.fail[run.key] - run.conclusion = failure ? 'failure' : 'success' - if (failure === 'before-apply') return - if (run.file === WORKFLOWS.admission.file && inputs.mode === 'inspect') { - const result = { - v: 1, - mode: 'inspect', - generation: world.selector.generation, - membership: world.selector.membership - } - run.artifacts[`relay-asia-admission-result-${run.id}-1`] = { - 'result.json': JSON.stringify(result) - } - } else if (run.file === WORKFLOWS.admission.file) { - assert.equal(inputs.confirmation, 'CONFIGURE_ASIA_DIRECTOR') - promote(world, inputs['director-image-digest']) - world.cells = [...world.cells, ...inputs['cell-ids'].split(',')] - } else if (run.file === WORKFLOWS.rehome.file) { - const identities = - inputs['director-image-digest'] === world.serving.digest && - inputs['rollback-image-digest'] === world.rollback.digest - const matches = - Number(inputs['expected-control-generation']) === world.control.generation && - Number(inputs['expected-selector-generation']) === world.selector.generation - if (inputs.mode === 'pause' && matches && world.control.enabled) { - assert.equal(inputs.confirmation, 'PAUSE_REGIONAL_REHOMING') - // The real job applies the pause before anything else can fail. - nextGeneration(world, false) - run.log = controlLine('pause', world.control) - } else if (inputs.mode === 'enable') { - assert.equal(inputs.confirmation, 'ENABLE_REGIONAL_REHOMING') - assert.equal( - world.runs.find((other) => other.id === Number(inputs['monitor-run-id']))?.file, - WORKFLOWS.monitor.file - ) - if (matches && identities && !failure) { - nextGeneration(world, true) - run.log = controlLine('enable', world.control) - return - } - run.conclusion = 'failure' - if (failure === 'applied-silent') { - // Applied, then a later step failed and the recovery printed nothing. - nextGeneration(world, true) - run.log = controlLine('enable', world.control) - return - } - // Applied, then the job's own recovery disabled it again; or a director safety pause got - // there first and recovery found it disabled. - if (failure === 'applied-then-recovered') nextGeneration(world, false, 2) - if (failure === 'safety-pause') nextGeneration(world, false, 3) - run.log = controlLine('recover-enable', world.control, { - recovered: failure === 'applied-then-recovered' - }) - } else if (!matches || (inputs.mode === 'inspect' && !identities)) { - run.conclusion = 'failure' - } else { - run.log = controlLine(inputs.mode, world.control) - } - } else if (run.file === WORKFLOWS.publish.file) { - world.registry[`${IMAGE_REPOSITORY}:sha-${run.headSha}`] = world.publishedDigest - run.log = `publish\tBuild\tsha-${run.headSha}: digest: ${world.pushLogDigest ?? world.publishedDigest} size: 3241` - } else if (run.file === WORKFLOWS.director.file) { - if ( - world.control.enabled || - Number(inputs['expected-rehome-generation']) !== world.control.generation - ) { - run.conclusion = 'failure' - return - } - promote(world, inputs['image-digest']) - // Blue/green takes minutes; traffic moves about one before the run completes. - world.now += 4 * 60_000 - } else if (run.file === WORKFLOWS.monitor.file) { - world.now += 16 * 60_000 - run.artifacts[`relay-monitor-dry-run-${run.id}-1`] = monitorFiles( - run, - world.now, - world.monitorState - ) - } -} - -function flag(args, name) { - const index = args.indexOf(name) - return index < 0 ? undefined : args[index + 1] -} - -function gh(world, args, input) { - const ok = (value) => ({ - status: 0, - stdout: typeof value === 'string' ? value : JSON.stringify(value), - stderr: '' - }) - if (args[0] === 'api' && args[1] === 'user') return ok('operator\n') - if (args[0] === 'api' && args.includes('--paginate')) { - const status = flag(args, '-f').slice('status='.length) - return ok( - world.active - .filter((run) => run.status === status) - .map((run) => `${JSON.stringify(run)}\n`) - .join('') - ) - } - if (args[0] === 'api' && args[1].includes('/actions/runs/')) { - const claimed = world.runs.find( - (candidate) => candidate.id === Number(args[1].split('/').at(-1)) - ) - return ok({ - path: relayWorkflowPath(claimed.file.slice(RELAY_WORKFLOW_FILE_PREFIX.length)), - actor: claimed.actor ?? 'operator' - }) - } - if (args[0] === 'api') return ok(`${world.main}\n`) - if (args[0] === 'workflow') { - const workflow = Object.values(WORKFLOWS).find((candidate) => candidate.file === args[2]) - const inputs = JSON.parse(input) - const run = { - id: world.nextRunId++, - file: workflow.file, - name: workflow.name, - inputs, - key: `${workflow.file}:${inputs.mode ?? 'deploy'}`, - dispatched: true, - headSha: world.main, - polls: world.polls?.[`${workflow.file}:${inputs.mode ?? 'deploy'}`] ?? 0, - artifacts: {}, - log: '' - } - world.runs.push(run) - simulate(world, run) - return ok( - world.printUrl ? `https://github.com/${REPOSITORY}/actions/runs/${run.id}\n` : 'Created\n' - ) - } - const run = world.runs.find((candidate) => candidate.id === Number(args[2])) - if (args[1] === 'list') { - const runs = world.runs - .filter((candidate) => candidate.file === flag(args, '--workflow')) - .reverse() - return ok(runs.map((candidate) => ({ databaseId: candidate.id }))) - } - if (args[1] === 'view' && args.includes('--log')) { - return world.unreadableLogs?.(run) ? { status: 1, stdout: '', stderr: 'HTTP 502' } : ok(run.log) - } - if (args[1] === 'view') { - world.onView?.(run) - const running = run.polls > 0 - if (running) run.polls -= 1 - return ok({ - status: running ? 'in_progress' : 'completed', - conclusion: running ? '' : run.conclusion, - attempt: 1, - headSha: run.headSha, - headBranch: 'main', - event: 'workflow_dispatch', - workflowName: run.name - }) - } - if (args[1] === 'download') { - const files = run.artifacts[flag(args, '-n')] - if (!files) return { status: 1, stdout: '', stderr: 'no artifact' } - for (const [name, content] of Object.entries(files)) - writeFileSync(join(flag(args, '-D'), name), content) - return ok('') - } - throw new Error(`unexpected gh ${args.join(' ')}`) -} - -function gcloud(world, args) { - const ok = (value) => ({ - status: 0, - stdout: typeof value === 'string' ? value : JSON.stringify(value), - stderr: '' - }) - if (args[1] === 'services') { - return ok({ - status: { - traffic: [ - { revisionName: world.serving.revision, percent: 100 }, - { revisionName: world.rollback.revision, percent: 0, tag: 'selector-rollback' } - ] - } - }) - } - if (args[1] === 'revisions') { - const revision = [world.serving, world.rollback].find( - (candidate) => candidate.revision === args[3] - ) - const cells = JSON.stringify(world.cells.map((id) => ({ id }))) - return ok({ - metadata: { creationTimestamp: new Date(revision.createdAt ?? START).toISOString() }, - spec: { - containers: [ - { - image: `${IMAGE_REPOSITORY}@${revision.digest}`, - env: [{ name: 'ORCA_RELAY_CELLS_JSON', value: cells }] - } - ] - } - }) - } - if (args[0] === 'artifacts') return ok(`${world.registry[args[4]] ?? ''}\n`) - if (args[0] === 'logging') { - const [, from, to] = args[2].match(/timestamp>="([^"]+)" AND timestamp<"([^"]+)"/) - world.reads = [ - ...(world.reads ?? []), - { from: Date.parse(from), to: Date.parse(to), readAt: world.now } - ] - return ok('t\n'.repeat(world.director5xx(Date.parse(from), Date.parse(to)))) - } - throw new Error(`unexpected gcloud ${args.join(' ')}`) -} - -function dependencies(world) { - return { - run: (program, args, input) => - program === 'gh' ? gh(world, args, input) : gcloud(world, args), - now: () => world.now, - sleep: async (ms) => { - world.now += ms - }, - print: (line) => { - // Every command the driver prints must be one its own parser accepts. - for (const [, argv] of line.matchAll(/drive-relay-director-deploy\.mjs ([^*]+?)\s*$/g)) { - assert.doesNotThrow(() => parseDriverArguments(argv.split(' ')), line) - } - world.printed.push(line) - }, - prompt: async (question) => { - world.questions = [...(world.questions ?? []), question] - const answer = world.answer(question) - world.prompts.push(question.match(/^Type (\S+(?: [0-9a-f]{12})?) to continue/)[1]) - return answer - } - } -} - -const logDirectory = () => mkdtempSync(join(tmpdir(), 'relay-deploy-test-')) - -function drive(world, argv, deps = dependencies(world)) { - return createDriver( - parseDriverArguments([...argv, '--log-directory', logDirectory()]), - deps - ).run() -} - -const start = (world, extra = [], deps) => drive(world, ['--commit', COMMIT, ...extra], deps) - -// Runs the last printed command, or the last one on a line containing `marker`. -function rerun(world, marker = '') { - const command = world.printed - .filter((line) => line.includes(marker) && line.includes('drive-relay-director-deploy.mjs ')) - .at(-1) - const argv = command - .slice(command.indexOf('.mjs ') + 5) - .trim() - .split(' ') - world.printed.length = 0 - return drive(world, argv) -} - -const stopped = (promise) => - promise.then( - () => assert.fail('expected the driver to stop'), - (error) => error - ) - -function keys(world) { - return world - .dispatches() - .map((run) => run.key.slice(RELAY_WORKFLOW_FILE_PREFIX.length).replace('.yml', '')) -} - -const dispatched = (world, key) => world.dispatches().filter((run) => run.key === key) -const REHOME = (mode) => `${WORKFLOWS.rehome.file}:${mode}` -const DEPLOY = `${WORKFLOWS.director.file}:deploy` -const PUBLISH = `${WORKFLOWS.publish.file}:publish` -const MONITOR = `${WORKFLOWS.monitor.file}:dry-run` -const report = (world) => world.printed.join('\n') -const live = (world) => [world.control.generation, world.control.enabled] - -test('publishes first, types every phrase, and enables on the digests now serving', async () => { - const world = fakeWorld() - assert.equal((await start(world)).done, true) - assert.deepEqual(keys(world), [ - 'publish-relay-production:publish', - 'operate-relay-asia-admission:inspect', - 'operate-relay-production-rehome:inspect', - 'operate-relay-production-rehome:pause', - 'deploy-relay-production-director:deploy', - 'operate-relay-production-rehome:inspect', - 'monitor-relay-production:dry-run', - 'operate-relay-production-rehome:enable' - ]) - assert.deepEqual(world.prompts, [ - 'DEPLOY aaaaaaaaaaaa', - 'PAUSE_REGIONAL_REHOMING', - 'ENABLE_REGIONAL_REHOMING' - ]) - assert.ok(world.questions.every((question) => question.endsWith('\n'))) - assert.ok( - world.questions.some((question) => - question.includes( - 'arms an automatic enable, sent about 17 min from now and only if the monitor is green and its evidence is at most 150 s old' - ) - ) - ) - const [deploy] = dispatched(world, DEPLOY) - assert.deepEqual( - [ - deploy.inputs['image-digest'], - deploy.inputs['predecessor-image-digest'], - deploy.inputs['expected-rehome-generation'] - ], - [NEW, OLD, '40'] - ) - const [enable] = dispatched(world, REHOME('enable')) - const [monitor] = dispatched(world, MONITOR) - assert.deepEqual( - [ - enable.inputs['director-image-digest'], - enable.inputs['rollback-image-digest'], - enable.inputs['expected-control-generation'], - enable.inputs['monitor-run-id'] - ], - [NEW, NEW, '40', String(monitor.id)] - ) - assert.equal(monitor.inputs['expected-general-cells'], 'production-gce-c27,production-gce-c7') - assert.deepEqual(live(world), [41, true]) -}) - -test('a wrong phrase stops before rehome or the director is touched', async () => { - const world = fakeWorld({ answer: () => 'yes' }) - assert.match((await stopped(start(world))).message, /expected DEPLOY aaaaaaaaaaaa/) - assert.deepEqual(keys(world), [ - 'publish-relay-production:publish', - 'operate-relay-asia-admission:inspect', - 'operate-relay-production-rehome:inspect' - ]) -}) - -test('F2: rehome found paused is never adopted; --leave-rehome-paused deploys and leaves it paused', async () => { - const world = fakeWorld({ control: { ...CONTROL, generation: 52, enabled: false } }) - assert.match( - (await stopped(start(world))).message, - /did not pause it.*--pause-run.*--leave-rehome-paused/s - ) - assert.match(report(world), /drive-relay-director-deploy\.mjs .*--publish-run \d+/) - await rerun(world).catch(() => {}) - assert.equal(dispatched(world, PUBLISH).length, 1) - await start(world, ['--leave-rehome-paused']) - assert.equal( - dispatched(world, REHOME('pause')).length + dispatched(world, REHOME('enable')).length, - 0 - ) - assert.equal(dispatched(world, DEPLOY)[0].inputs['expected-rehome-generation'], '52') - assert.deepEqual(live(world), [52, false]) -}) - -test('F2: main moving is caught before any rehome change, and after the pause it changes nothing', async () => { - const moved = fakeWorld({ main: 'b'.repeat(40) }) - assert.match((await stopped(start(moved))).message, /not the reviewed/) - assert.equal(moved.dispatches().length, 0) - - const world = fakeWorld() - const deps = dependencies(world) - const run = deps.run - deps.run = (program, args, input) => { - const result = run(program, args, input) - if (args[0] === 'workflow' && JSON.parse(input).mode === 'pause') world.main = 'b'.repeat(40) - return result - } - assert.equal((await start(world, [], deps)).done, true) - assert.deepEqual(live(world), [41, true]) -}) - -test('ops-log 22:59Z: main moving during the inspects and the typed phrase no longer stops the deploy', async () => { - const world = fakeWorld() - const deps = dependencies(world) - const run = deps.run - deps.run = (program, args, input) => { - const result = run(program, args, input) - if (args[0] === 'workflow' && JSON.parse(input).mode === 'inspect') world.main = 'b'.repeat(40) - return result - } - assert.equal((await start(world, [], deps)).done, true) - assert.equal(dispatched(world, PUBLISH)[0].headSha, COMMIT) - assert.equal(dispatched(world, DEPLOY)[0].inputs['image-digest'], NEW) -}) - -test('main moving between the check and the publish stops untouched, naming the reuse command', async () => { - const world = fakeWorld() - const deps = dependencies(world) - const run = deps.run - deps.run = (program, args, input) => { - if (args[0] === 'workflow') world.main = 'b'.repeat(40) - return run(program, args, input) - } - assert.match((await stopped(start(world, [], deps))).message, /main moved/) - const publishRun = String(dispatched(world, PUBLISH)[0].id) - assert.deepEqual(keys(world), ['publish-relay-production:publish']) - // The only command printed is the one that reuses the build. - const commands = world.printed.filter((line) => line.includes('&& node dev/scripts/')) - assert.equal(commands.length, 1) - assert.match(commands[0], new RegExp(`--commit ${world.main} --publish-run ${publishRun}$`)) - assert.equal((await rerun(world)).done, true) - assert.equal(dispatched(world, PUBLISH).length, 1) - assert.deepEqual(live(world), [41, true]) -}) - -test('a running workflow is summarised, not streamed', async () => { - const world = fakeWorld({ polls: { [PUBLISH]: 40 } }) - assert.equal((await start(world)).done, true) - assert.deepEqual( - world.printed - .filter((line) => / publish: (in_progress|success)/.test(line)) - .map((line) => line.slice(25)), - [ - 'publish: in_progress, 0 min', - 'publish: in_progress, 5 min', - `publish: success https://github.com/${REPOSITORY}/actions/runs/${dispatched(world, PUBLISH)[0].id}` - ] - ) -}) - -test('a publish whose log disagrees with the registry stops before rehome is touched', async () => { - const world = fakeWorld({ pushLogDigest: CELL }) - assert.match((await stopped(start(world))).message, /tag moved/) - assert.equal(dispatched(world, REHOME('pause')).length, 0) - assert.match(report(world), /rehome: not read; this run did not change it/) -}) - -test('dry run dispatches nothing and prints every step', async () => { - const world = fakeWorld() - const directory = logDirectory() - await createDriver( - parseDriverArguments([ - '--commit', - COMMIT, - '--dry-run', - '--configure', - `production-gce-c34=${CELL}`, - '--log-directory', - directory - ]), - dependencies(world) - ).run() - assert.equal(world.dispatches().length, 0) - assert.equal(world.prompts.length, 0) - const plan = world.printed.filter((line) => line.includes(' plan ')) - assert.equal(plan.length, 10) - assert.ok(plan.some((line) => line.includes('"expected-control-generation":"39"'))) - assert.ok( - plan.some((line) => line.includes('"confirmation":""')) - ) - assert.deepEqual( - readdirSync(directory).map((name) => name.endsWith('-dry-run.log')), - [true] - ) -}) - -test('an in-flight relay workflow stops the preflight; no printed run URL stops the dispatch', async () => { - const busy = fakeWorld({ - active: [ - { - id: 9, - path: relayWorkflowPath('deploy-relay-production-same-cap.yml'), - name: 'Same cap', - status: 'waiting' - } - ] - }) - assert.match((await stopped(start(busy))).message, /in flight/) - assert.equal(busy.dispatches().length, 0) - const silent = fakeWorld({ printUrl: false }) - assert.match((await stopped(start(silent))).message, /printed no run URL/) -}) - -test('F1: an interrupt while the pause is in flight says rehome is changing, and the printed command finishes', async () => { - const world = fakeWorld() - const deps = dependencies(world) - let driver - world.onView = () => { - if (world.dispatches().at(-1)?.inputs.mode === 'pause' && !world.interrupted) { - world.interrupted = true - driver.interrupt('SIGINT') - throw new Error('killed') - } - } - driver = createDriver( - parseDriverArguments(['--commit', COMMIT, '--log-directory', logDirectory()]), - deps - ) - await driver.run().catch(() => {}) - assert.match( - report(world), - /STOPPED: interrupted by SIGINT[\s\S]*REHOME IS CHANGING: the pause run .* applies on its own/ - ) - assert.match(report(world), /finish with: cd cloud && .* --publish-run \d+ --pause-run \d+/) - assert.equal(await rerun(world).then((result) => result.done), true) - assert.equal(dispatched(world, REHOME('pause')).length, 1) - assert.equal(dispatched(world, PUBLISH).length, 1) - assert.deepEqual(live(world), [41, true]) -}) - -test('F1: an interrupt while the enable is in flight never says rehome is paused', async () => { - const world = fakeWorld() - const deps = dependencies(world) - let driver - world.onView = () => { - if (world.dispatches().at(-1)?.inputs.mode === 'enable') driver.interrupt('SIGTERM') - } - driver = createDriver( - parseDriverArguments(['--commit', COMMIT, '--log-directory', logDirectory()]), - deps - ) - await driver.run() - const text = report(world) - assert.match( - text, - /REHOME IS CHANGING: the enable run .* it enables rehome, or disables it again if it fails/ - ) - assert.doesNotMatch(text.slice(text.indexOf('interrupted')), /REHOME IS PAUSED/) -}) - -test('a pause run that printed nothing is PAUSE UNCONFIRMED, and a later safety pause is never lifted', async () => { - const world = fakeWorld({ fail: { [REHOME('pause')]: 'before-apply' } }) - await stopped(start(world)) - assert.match(report(world), /PAUSE UNCONFIRMED: .* printed no pause. Rehome MAY BE PAUSED/) - // A director safety pause lands; the operator follows the report's command. - nextGeneration(world, false) - assert.match((await stopped(rerun(world))).message, /printed no pause of its own/) - assert.equal(dispatched(world, REHOME('enable')).length, 0) -}) - -test('a failed deploy leaves its pause owned, and the printed command finishes without pausing or publishing again', async () => { - const world = fakeWorld({ fail: { [DEPLOY]: 'before-apply' } }) - await stopped(start(world)) - assert.match(report(world), /REHOME IS PAUSED by this driver at generation 40/) - assert.match(report(world), /rollback point: orca-cloud-relay-00700-qor/) - await rerun(world) - assert.equal(dispatched(world, REHOME('pause')).length, 1) - assert.equal(dispatched(world, PUBLISH).length, 1) - assert.equal(dispatched(world, DEPLOY).length, 2) - assert.deepEqual(live(world), [41, true]) -}) - -test('a frozen monitor stops with its failures; the re-run runs a fresh one', async () => { - const failures = [ - { source: 'auth', code: 'threshold_equal', signal: 'health', observed: 0, threshold: 1 } - ] - const world = fakeWorld({ monitorState: { frozenAt: '2026-10-05T05:30:00Z', failures } }) - assert.match( - (await stopped(start(world))).message, - /incomplete or stale[\s\S]*auth threshold_equal health 0 1/ - ) - world.monitorState = {} - await rerun(world) - assert.equal(dispatched(world, MONITOR).length, 2) - assert.equal(dispatched(world, DEPLOY).length, 1) - assert.deepEqual(live(world), [41, true]) -}) - -test('stale monitor evidence is not spent on an enable', async () => { - const world = fakeWorld() - const deps = dependencies(world) - const prompt = deps.prompt - deps.prompt = async (question) => { - const answer = await prompt(question) - if (answer === 'ENABLE_REGIONAL_REHOMING') { - const completed = world.now + 16 * 60_000 - MONITOR_MAX_AGE_AT_ENABLE_MS - 1000 - const at = (offset) => new Date(completed - offset).toISOString() - world.monitorState = { - completedAt: at(0), - lastSampleAt: at(0), - windowStartedAt: at(15 * 60_000), - startedAt: at(16 * 60_000) - } - } - return answer - } - assert.match((await stopped(start(world, [], deps))).message, /past the 150 s budget/) - assert.equal(dispatched(world, REHOME('enable')).length, 0) -}) - -test('the ops-log 05:41Z case: an enable that failed before applying is finished by the printed command', async () => { - const world = fakeWorld({ fail: { [REHOME('enable')]: 'not-applied' } }) - await stopped(start(world)) - assert.match(report(world), /REHOME IS PAUSED by this driver at generation 40/) - await rerun(world) - assert.deepEqual(live(world), [41, true]) -}) - -test('an enable whose own recovery paused rehome again hands ownership to that run', async () => { - const world = fakeWorld({ fail: { [REHOME('enable')]: 'applied-then-recovered' } }) - await stopped(start(world)) - const [enable] = dispatched(world, REHOME('enable')) - assert.match( - report(world), - new RegExp(`REHOME IS PAUSED by this driver at generation 42 \\(.*${enable.id}\\)`) - ) - await rerun(world) - assert.deepEqual(live(world), [43, true]) -}) - -test('F3: a director safety pause found by the enable recovery is never adopted or lifted', async () => { - const world = fakeWorld({ fail: { [REHOME('enable')]: 'safety-pause' } }) - await stopped(start(world)) - assert.match(report(world), /rehome: generation 43 as last read, PAUSED, not by this driver/) - const stop = report(world).slice(report(world).indexOf('STOPPED')) - assert.doesNotMatch(stop, /--pause-run/) - assert.match((await stopped(rerun(world))).message, /did not pause it/) - const [pause] = dispatched(world, REHOME('pause')) - const claim = await stopped( - drive(world, [ - '--commit', - COMMIT, - '--publish-run', - String(dispatched(world, PUBLISH)[0].id), - '--pause-run', - String(pause.id) - ]) - ) - assert.match(claim.message, /not the pause .* made at 40/) - assert.equal(dispatched(world, REHOME('enable')).length, 1) - assert.deepEqual(live(world), [43, false]) -}) - -test('P1: a green enable with an unreadable log is ENABLE UNCONFIRMED, and the printed command settles it', async () => { - const world = fakeWorld({ unreadableLogs: (run) => run.inputs?.mode === 'enable' }) - await stopped(start(world)) - assert.match( - report(world), - /ENABLE UNCONFIRMED: .* did not confirm the enable. Rehome may be enabled, or paused/ - ) - world.unreadableLogs = undefined - assert.equal((await rerun(world, 'If it enabled rehome')).done, true) - assert.equal(dispatched(world, REHOME('enable')).length, 1) - assert.deepEqual(live(world), [41, true]) -}) - -test('configure waits out a soak anchored at the traffic switch, then takes a typed phrase', async () => { - const world = fakeWorld() - await start(world, ['--configure', `production-gce-c34=${CELL}`]) - const [configure] = dispatched(world, `${WORKFLOWS.admission.file}:configure`) - assert.deepEqual( - [ - configure.inputs['cell-ids'], - configure.inputs['image-digest'], - configure.inputs['director-image-digest'], - configure.inputs['selector-generation'] - ], - ['production-gce-c34', CELL, NEW, '345'] - ) - assert.ok(world.prompts.includes('CONFIGURE_ASIA_DIRECTOR')) - const [before, after] = world.reads - assert.equal(after.to - after.from, 5 * 60_000) - // The deploy ran 4 minutes; traffic moved a minute before it completed. - assert.equal(after.from, START + 3 * 60_000) - assert.equal(before.to, START) - assert.ok(after.readAt >= after.to + 60_000) -}) - -test('F5: a tripped soak is judged again on fresh traffic by the printed command', async () => { - const world = fakeWorld({ director5xx: (from) => (from >= START ? 200 : 10) }) - assert.match( - (await stopped(start(world, ['--configure', `production-gce-c34=${CELL}`]))).message, - /5xx rose from 10 to 200/ - ) - assert.match(report(world), /REHOME IS PAUSED by this driver at generation 40/) - world.director5xx = () => 10 - world.now += 30 * 60_000 - const rerunAt = world.now - await rerun(world) - assert.equal(dispatched(world, `${WORKFLOWS.admission.file}:configure`).length, 1) - assert.ok(world.reads.at(-1).from >= rerunAt) - assert.deepEqual(live(world), [41, true]) -}) - -test('P8 and G1: the printed command never reports DONE without reading rehome', async () => { - const world = fakeWorld() - await start(world) - const before = world.dispatches().length - world.prompts.length = 0 - const publishRun = String(dispatched(world, PUBLISH)[0].id) - assert.equal((await drive(world, ['--commit', COMMIT, '--publish-run', publishRun])).done, true) - assert.deepEqual(keys(world).slice(before), [ - 'operate-relay-asia-admission:inspect', - 'operate-relay-production-rehome:inspect' - ]) - assert.equal(world.prompts.length, 0) - assert.match(report(world), /DONE: .* rehome enabled/) - - // G1: the deploy is done but the driver's pause still holds; dropping --pause-run must not hide it. - const paused = fakeWorld({ monitorState: { frozenAt: '2026-10-05T05:30:00Z' } }) - await stopped(start(paused)) - const run = String(dispatched(paused, PUBLISH)[0].id) - assert.match( - (await stopped(drive(paused, ['--commit', COMMIT, '--publish-run', run]))).message, - /did not pause it/ - ) - assert.deepEqual(live(paused), [40, false]) -}) - -test("G2: --pause-run accepts only this operator's own rehome-control run", async () => { - const world = fakeWorld({ control: { ...CONTROL, generation: 40, enabled: false } }) - // Another operator paused rehome by hand. - world.runs.push({ - id: 900, - file: WORKFLOWS.rehome.file, - actor: 'someone-else', - conclusion: 'success', - log: controlLine('pause', world.control) - }) - assert.match( - (await stopped(start(world, ['--pause-run', '900']))).message, - /is not a .* run by operator/ - ) - // A run of another workflow cannot prove a pause either. - world.runs.push({ - id: 901, - file: WORKFLOWS.monitor.file, - conclusion: 'success', - log: controlLine('pause', world.control) - }) - assert.match( - (await stopped(start(world, ['--pause-run', '901']))).message, - /is not a .* run by operator/ - ) - assert.equal(world.dispatches().length, 0) - assert.deepEqual(live(world), [40, false]) -}) - -test('G3: a failed enable run is never reported RE-ENABLED, whatever it printed last', async () => { - const world = fakeWorld({ fail: { [REHOME('enable')]: 'applied-silent' } }) - await stopped(start(world)) - assert.doesNotMatch(report(world), /RE-ENABLED|DONE/) - assert.match(report(world), /ENABLE UNCONFIRMED: .* did not confirm the enable/) - // The printed command re-reads live state: rehome is at the pause's generation + 1, enabled. - const pauseRun = String(dispatched(world, REHOME('pause'))[0].id) - const publishRun = String(dispatched(world, PUBLISH)[0].id) - assert.equal( - (await drive(world, ['--commit', COMMIT, '--publish-run', publishRun, '--pause-run', pauseRun])) - .done, - true - ) - assert.equal(dispatched(world, REHOME('enable')).length, 1) -}) - -test('Q1 and C1: after a hard kill mid-pause, the re-run names the pause its log recorded and finishes with it', async () => { - const world = fakeWorld() - const deps = dependencies(world) - const directory = logDirectory() - let logAtKill - world.onView = () => { - if (world.dispatches().at(-1)?.key !== REHOME('pause')) return - // SIGKILL writes no stop report: only what was logged before the watch survives. - logAtKill = readFileSync(join(directory, readdirSync(directory)[0]), 'utf8') - world.onView = undefined - throw new Error('SIGKILL') - } - await stopped( - createDriver( - parseDriverArguments(['--commit', COMMIT, '--log-directory', directory]), - deps - ).run() - ) - world.printed.length = 0 - const [pause] = dispatched(world, REHOME('pause')) - assert.match(logAtKill, new RegExp(`runs/${pause.id}`)) - const publishRun = String(dispatched(world, PUBLISH)[0].id) - assert.match( - (await stopped(drive(world, ['--commit', COMMIT, '--publish-run', publishRun]))).message, - /did not pause it.*--pause-run its log printed/ - ) - assert.deepEqual(live(world), [40, false]) - const argv = ['--commit', COMMIT, '--publish-run', publishRun, '--pause-run', String(pause.id)] - assert.equal((await drive(world, argv)).done, true) - assert.deepEqual(live(world), [41, true]) -}) - -test('Q5: a failed enable followed by an unexplained disable is never adopted', async () => { - const world = fakeWorld() - const deps = dependencies(world) - const run = deps.run - deps.run = (program, args, input) => { - const result = run(program, args, input) - if (args[0] === 'workflow' && JSON.parse(input).mode === 'enable') { - world.runs.at(-1).conclusion = 'failure' - world.control = { ...world.control, generation: 42, enabled: false } - } - return result - } - await stopped(start(world, [], deps)) - assert.match(report(world), /ENABLE UNCONFIRMED/) - assert.doesNotMatch(report(world), /RE-ENABLED|DONE/) - const commands = ['failed before applying', 'paused rehome again'].map((marker) => { - const line = world.printed.findLast((printed) => printed.includes(marker)) - return line.slice(line.indexOf('.mjs ') + 5).split(' ') - }) - for (const argv of commands) { - assert.match((await stopped(drive(world, argv))).message, /--rehome-generation|cannot prove/) - const pinned = await stopped(drive(world, [...argv, '--rehome-generation', '42'])) - assert.match(pinned.message, /not the pause|cannot prove/) - } - assert.equal(dispatched(world, REHOME('enable')).length, 1) - assert.deepEqual(live(world), [42, false]) -}) - -test('C1: a pause whose run cannot be viewed, or printed no URL, is reported as REHOME IS CHANGING', async () => { - const world = fakeWorld() - const deps = dependencies(world) - const run = deps.run - deps.run = (program, args, input) => { - const pause = dispatched(world, REHOME('pause'))[0] - if (pause && args[1] === 'view' && args[2] === String(pause.id) && args.includes('--json')) { - return { status: 1, stdout: '', stderr: 'HTTP 502' } - } - return run(program, args, input) - } - await stopped(start(world, [], deps)) - const [pause] = dispatched(world, REHOME('pause')) - assert.match(report(world), new RegExp(`REHOME IS CHANGING: the pause run .*${pause.id}`)) - assert.match(report(world), new RegExp(`finish with: cd cloud && .*--pause-run ${pause.id}`)) - assert.doesNotMatch(report(world), /Re-run to finish/) - - const silent = fakeWorld() - const silentDeps = dependencies(silent) - const silentRun = silentDeps.run - silentDeps.run = (program, args, input) => { - const result = silentRun(program, args, input) - return args[0] === 'workflow' && JSON.parse(input).mode === 'pause' - ? { ...result, stdout: 'Created\n' } - : result - } - await stopped(start(silent, [], silentDeps)) - assert.match( - report(silent), - /REHOME IS CHANGING: the pause run \(gh printed no URL; find it at https:/ - ) - assert.doesNotMatch(report(silent), /Re-run to finish/) -}) - -test('C2: --leave-rehome-paused refuses an enabled switch instead of pausing it', async () => { - const world = fakeWorld() - assert.match( - (await stopped(start(world, ['--leave-rehome-paused']))).message, - /accepts only a paused switch/ - ) - assert.equal(dispatched(world, REHOME('pause')).length, 0) -}) - -test('G4: an interrupt during the enable prints both commands that can finish', async () => { - const world = fakeWorld() - const deps = dependencies(world) - let driver - world.onView = () => { - if (world.dispatches().at(-1)?.inputs.mode === 'enable') driver.interrupt('SIGINT') - } - driver = createDriver( - parseDriverArguments(['--commit', COMMIT, '--log-directory', logDirectory()]), - deps - ) - await driver.run() - const [pause] = dispatched(world, REHOME('pause')) - const [enable] = dispatched(world, REHOME('enable')) - assert.match( - report(world), - new RegExp( - `If it enabled rehome, or failed before applying, finish with: cd cloud && .*--pause-run ${pause.id}` - ) - ) - assert.match( - report(world), - new RegExp( - `If its recovery paused rehome again, finish with: cd cloud && .*--pause-run ${enable.id}` - ) - ) -}) - -test("G5: the driver's own run, still listed as in progress, does not block its next step", async () => { - const world = fakeWorld() - const deps = dependencies(world) - const run = deps.run - deps.run = (program, args, input) => { - const result = run(program, args, input) - if (args[0] === 'workflow' && JSON.parse(input).mode === 'dry-run') { - const monitor = world.runs.at(-1) - world.active.push({ - id: monitor.id, - path: relayWorkflowPath('monitor-relay-production.yml'), - name: 'Monitor', - status: 'in_progress' - }) - } - return result - } - assert.equal((await start(world, [], deps)).done, true) - assert.deepEqual(live(world), [41, true]) -}) - -test('argument parsing and plan helpers fail closed', () => { - assert.throws(() => parseDriverArguments([]), /missing --commit/) - assert.throws(() => parseDriverArguments(['--commit', 'abc']), /full commit SHA/) - assert.throws( - () => parseDriverArguments(['--commit', COMMIT, '--pause-run', 'x']), - /must be a run ID/ - ) - assert.throws( - () => parseDriverArguments(['--commit', COMMIT, '--pause-run', '5', '--leave-rehome-paused']), - /contradict/ - ) - assert.throws( - () => parseDriverArguments(['--commit', COMMIT, '--configure', 'production-gce-c34']), - /--configure must be/ - ) - assert.deepEqual(parseConfigureWave(`production-gce-c27,production-gce-c28=${CELL}`).cells, [ - 'production-gce-c27', - 'production-gce-c28' - ]) - assert.throws( - () => validateDispatchInputs({ confirmation: '' }), - /not resolved/ - ) - assert.throws( - () => validateDispatchInputs({ 'image-digest': 'sha256:abc' }), - /not a sha256 digest/ - ) - assert.ok(blocksDeploy(relayWorkflowPath('push-deploy.yml'))) - assert.ok(blocksDeploy(`${relayWorkflowPath('push-deploy.yml')}@refs/heads/main`)) - assert.ok(!blocksDeploy(relayWorkflowPath('monitor-relay-clock-skew.yml'))) - const disabled = { ...CONTROL, enabled: false } - assert.equal(pausedGeneration(rehomeResultFromLog(controlLine('pause', disabled))), 39) - assert.equal( - pausedGeneration( - rehomeResultFromLog(controlLine('recover-enable', disabled, { recovered: true })) - ), - 39 - ) - assert.equal( - pausedGeneration( - rehomeResultFromLog(controlLine('recover-enable', disabled, { recovered: false })) - ), - undefined - ) - assert.equal(pausedGeneration(rehomeResultFromLog(controlLine('inspect', disabled))), undefined) - assert.throws(() => rehomeResultFromLog('nothing'), /printed no control/) -}) diff --git a/cloud/dev/scripts/github-smoke-token.mjs b/cloud/dev/scripts/github-smoke-token.mjs deleted file mode 100644 index af4c61fd52b..00000000000 --- a/cloud/dev/scripts/github-smoke-token.mjs +++ /dev/null @@ -1,136 +0,0 @@ -const REQUEST_TIMEOUT_MS = 15_000 -const JWT_PATTERN = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/ - -export async function requestGitHubSmokeTokens( - authOrigin, - fetchImpl = fetch, - environment = process.env, - options = {} -) { - const origin = canonicalHttpsOrigin(authOrigin) - const requestUrl = environment.ACTIONS_ID_TOKEN_REQUEST_URL - const requestToken = environment.ACTIONS_ID_TOKEN_REQUEST_TOKEN - if (!requestUrl || !requestToken) throw new Error('GitHub OIDC request context is unavailable') - const audience = `${origin}/v1/internal/github-smoke-token` - const oidcUrl = new URL(requestUrl) - oidcUrl.searchParams.set('audience', audience) - const oidcResponse = await fetchImpl(oidcUrl, { - headers: { authorization: `Bearer ${requestToken}` }, - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) - }) - const oidc = await readJson(oidcResponse, 'GitHub OIDC request') - if (!oidcResponse.ok || !validJwt(oidc.value)) { - throw new Error(`GitHub OIDC request failed with ${oidcResponse.status}`) - } - const exchangeResponse = await fetchImpl(audience, { - method: 'POST', - headers: { - authorization: `Bearer ${oidc.value}`, - ...(options.relayAsiaLoad ? { 'content-type': 'application/json' } : {}) - }, - ...(options.relayAsiaLoad - ? { body: JSON.stringify({ relayAsiaLoad: parseRelayAsiaLoadOptions(options.relayAsiaLoad) }) } - : {}), - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) - }) - const exchange = await readJson(exchangeResponse, 'Orca smoke identity exchange') - if (!exchangeResponse.ok) { - throw new Error(`Orca smoke identity exchange failed with ${exchangeResponse.status}`) - } - return { - ...parseAccessTokens(exchange.accessTokens), - ...(options.relayAsiaLoad - ? { - relayAsiaLoadPrincipals: parseRelayAsiaLoadPrincipals( - exchange.relayAsiaLoadPrincipals, - options.relayAsiaLoad.principalCount - ) - } - : {}) - } -} - -// Staging auth mints Asia load principal sets for this many shards. -export const RELAY_ASIA_LOAD_PRINCIPAL_SHARDS = 4 - -function parseRelayAsiaLoadOptions(value) { - if ( - !value || typeof value !== 'object' || - !Number.isSafeInteger(value.shardIndex) || value.shardIndex < 0 || value.shardIndex >= RELAY_ASIA_LOAD_PRINCIPAL_SHARDS || - !Number.isSafeInteger(value.principalCount) || value.principalCount < 1 || - value.principalCount > 32 - ) throw new Error('Relay Asia load principal request is invalid') - return { v: 1, shardIndex: value.shardIndex, principalCount: value.principalCount } -} - -function canonicalHttpsOrigin(value) { - const url = new URL(value) - if (url.protocol !== 'https:' || url.pathname !== '/' || url.search || url.hash) { - throw new Error('auth origin must be canonical HTTPS') - } - return url.origin -} - -function validJwt(value) { - return typeof value === 'string' && value.length <= 8192 && JWT_PATTERN.test(value) -} - -function parseAccessTokens(value) { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error('Orca smoke identity response is malformed') - } - const expected = ['outsider', 'owner', 'recipient'] - if (Object.keys(value).sort().join(',') !== expected.join(',')) { - throw new Error('Orca smoke identity response principals are invalid') - } - return Object.fromEntries( - expected.map((name) => { - const principal = value[name] - if ( - !principal || - typeof principal !== 'object' || - typeof principal.userId !== 'string' || - !/^[A-Za-z0-9_-]{1,128}$/.test(principal.userId) || - typeof principal.accessToken !== 'string' || - !validJwt(principal.accessToken) || - typeof principal.expiresAt !== 'number' || - principal.expiresAt <= Date.now() || - principal.expiresAt > Date.now() + 610_000 - ) { - throw new Error('Orca smoke identity response principal is malformed') - } - return [name, principal] - }) - ) -} - -function parseRelayAsiaLoadPrincipals(value, expectedCount) { - if (!Array.isArray(value) || value.length !== expectedCount) { - throw new Error('Relay Asia load principal response is invalid') - } - const userIds = new Set() - return value.map((principal, principalIndex) => { - if ( - !principal || typeof principal !== 'object' || - principal.principalIndex !== principalIndex || - typeof principal.userId !== 'string' || - !/^usr_relay_asia_load_[A-Za-z0-9_-]{32}$/.test(principal.userId) || - typeof principal.profileId !== 'string' || - principal.profileId !== principal.userId.replace(/^usr_/, 'prof_') || - userIds.has(principal.userId) || - typeof principal.accessToken !== 'string' || !validJwt(principal.accessToken) || - typeof principal.expiresAt !== 'number' || principal.expiresAt <= Date.now() || - principal.expiresAt > Date.now() + 610_000 - ) throw new Error('Relay Asia load principal response is malformed') - userIds.add(principal.userId) - return principal - }) -} - -async function readJson(response, label) { - try { - return await response.json() - } catch { - throw new Error(`${label} did not return JSON`) - } -} diff --git a/cloud/dev/scripts/github-smoke-token.test.mjs b/cloud/dev/scripts/github-smoke-token.test.mjs deleted file mode 100644 index 9ffb0b864d8..00000000000 --- a/cloud/dev/scripts/github-smoke-token.test.mjs +++ /dev/null @@ -1,111 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { requestGitHubSmokeTokens } from './github-smoke-token.mjs' - -const jwt = (value) => `${value}.${value}.${value}` - -test('exchanges the runner OIDC token without returning request credentials', async () => { - const requests = [] - const fetchImpl = async (url, init) => { - requests.push({ url: String(url), init }) - if (requests.length === 1) return Response.json({ value: jwt('github') }) - return Response.json({ - accessTokens: Object.fromEntries( - ['owner', 'recipient', 'outsider'].map((name) => [ - name, - { userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 } - ]) - ) - }) - } - const result = await requestGitHubSmokeTokens( - 'https://auth-staging.onorca.dev', - fetchImpl, - { - ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token?api-version=1', - ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token' - } - ) - assert.equal(result.owner.userId, 'usr_owner') - assert.match(requests[0].url, /audience=https%3A%2F%2Fauth-staging\.onorca\.dev/) - assert.equal(requests[0].init.headers.authorization, 'Bearer runner-request-token') - assert.equal(requests[1].init.headers.authorization, `Bearer ${jwt('github')}`) -}) - -test('fails with bounded errors and never includes credentials', async () => { - await assert.rejects( - requestGitHubSmokeTokens( - 'https://auth-staging.onorca.dev', - async () => new Response('denied', { status: 403 }), - { - ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token', - ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'private-request-token' - } - ), - (error) => { - assert.doesNotMatch(String(error), /private-request-token|denied/) - return true - } - ) -}) - -test('requests and validates an exact Relay Asia principal batch', async () => { - const requests = [] - const principals = Array.from({ length: 32 }, (_, principalIndex) => { - const suffix = String(principalIndex).padStart(32, 'a') - return { - principalIndex, - userId: `usr_relay_asia_load_${suffix}`, - profileId: `prof_relay_asia_load_${suffix}`, - accessToken: jwt(`load${principalIndex}`), - expiresAt: Date.now() + 600_000 - } - }) - const result = await requestGitHubSmokeTokens( - 'https://auth-staging.onorca.dev', - async (url, init) => { - requests.push({ url: String(url), init }) - return requests.length === 1 - ? Response.json({ value: jwt('github') }) - : Response.json({ - accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [ - name, - { userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 } - ])), - relayAsiaLoadPrincipals: principals - }) - }, - { - ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token', - ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token' - }, - { relayAsiaLoad: { shardIndex: 3, principalCount: 32 } } - ) - assert.equal(result.relayAsiaLoadPrincipals.length, 32) - assert.deepEqual(JSON.parse(requests[1].init.body), { - relayAsiaLoad: { v: 1, shardIndex: 3, principalCount: 32 } - }) - assert.equal(requests[1].init.headers['content-type'], 'application/json') -}) - -test('rejects malformed or duplicate Relay Asia principal batches', async () => { - const environment = { - ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token', - ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token' - } - let request = 0 - await assert.rejects(requestGitHubSmokeTokens( - 'https://auth-staging.onorca.dev', - async () => ++request === 1 - ? Response.json({ value: jwt('github') }) - : Response.json({ - accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [ - name, - { userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 } - ])), - relayAsiaLoadPrincipals: [] - }), - environment, - { relayAsiaLoad: { shardIndex: 0, principalCount: 32 } } - ), /principal response is invalid/) -}) diff --git a/cloud/dev/scripts/infra.mjs b/cloud/dev/scripts/infra.mjs deleted file mode 100644 index 6ab85df08b7..00000000000 --- a/cloud/dev/scripts/infra.mjs +++ /dev/null @@ -1,146 +0,0 @@ -import { execFileSync } from 'node:child_process' -import { existsSync } from 'node:fs' -import { homedir } from 'node:os' -import { join } from 'node:path' -import { assertStagingRelayAwake } from './staging-relay-apply-guard.mjs' - -// The relay root keeps its historical path so every existing caller — 9 workflows, the fence -// broker, and the infra:* scripts — is unchanged when --root is omitted. The foundation and apps -// roots stay in the private repository with the services they own. -const ROOT_DIRECTORIES = { - relay: join('infra', 'terraform') -} - -const command = process.argv[2] -const environment = readEnvironment(process.argv.slice(3)) -const root = readRoot(process.argv.slice(3)) -const tool = process.env.IAC_TOOL || findTool() - -if (!command || !['init', 'plan', 'apply'].includes(command)) { - exitWithUsage() -} - -if (!environment) { - exitWithUsage('Missing --env staging|production') -} - -if (!root) { - exitWithUsage(`Unknown --root; expected one of ${Object.keys(ROOT_DIRECTORIES).join('|')}`) -} - -const rootDirectory = ROOT_DIRECTORIES[root] -const terraformDir = join(process.cwd(), rootDirectory) -const backendConfig = join(terraformDir, 'backend', `${environment}.hcl`) -const varFile = join(terraformDir, 'environments', `${environment}.tfvars`) - -if (!existsSync(backendConfig)) { - throw new Error(`Backend config not found: ${backendConfig}`) -} - -if (!existsSync(varFile)) { - throw new Error(`Variable file not found: ${varFile}`) -} - -const chdir = `-chdir=${rootDirectory}` - -if (command === 'init') { - run([chdir, 'init', `-backend-config=backend/${environment}.hcl`]) -} else if (command === 'plan') { - run([ - chdir, - 'plan', - `-var-file=environments/${environment}.tfvars`, - `-out=${environment}.tfplan` - ]) -} else { - // A normal staging apply must not implicitly wake or partially mutate a sleeping data plane. - // Only the relay root can touch that data plane; the guard would refuse app work for no reason. - if (environment === 'staging' && root === 'relay') assertStagingRelayAwake() - run([chdir, 'apply', `${environment}.tfplan`]) -} - -function readEnvironment(args) { - const envIndex = args.indexOf('--env') - if (envIndex >= 0) { - return args[envIndex + 1] - } - - return process.env.ORCA_CLOUD_ENV -} - -function readRoot(args) { - const rootIndex = args.indexOf('--root') - const requested = rootIndex >= 0 ? args[rootIndex + 1] : 'relay' - return requested in ROOT_DIRECTORIES ? requested : undefined -} - -function findTool() { - for (const candidate of ['tofu', 'terraform']) { - try { - execFileSync(candidate, ['version'], { stdio: 'ignore' }) - return candidate - } catch { - // Try the next compatible IaC binary. - } - } - - throw new Error('Install Terraform or OpenTofu, or set IAC_TOOL.') -} - -function run(args) { - execFileSync(tool, args, { env: terraformEnv(), stdio: 'inherit' }) -} - -function terraformEnv() { - if ( - process.env.GOOGLE_APPLICATION_CREDENTIALS || - process.env.GOOGLE_CREDENTIALS || - process.env.GOOGLE_OAUTH_ACCESS_TOKEN - ) { - return process.env - } - - const token = readGcloudAccessToken() - if (!token) { - return process.env - } - - // Local convenience: Terraform uses ADC, while engineers often only have - // gcloud CLI auth. CI should use Workload Identity instead. - return { ...process.env, GOOGLE_OAUTH_ACCESS_TOKEN: token } -} - -function readGcloudAccessToken() { - for (const candidate of gcloudCandidates()) { - try { - return execFileSync(candidate, ['auth', 'print-access-token'], { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'ignore'] - }).trim() - } catch { - // Try the next common gcloud location. - } - } - - return null -} - -function gcloudCandidates() { - return [ - process.env.GCLOUD_PATH, - 'gcloud', - join(homedir(), 'Downloads', 'google-cloud-sdk', 'bin', 'gcloud'), - join(homedir(), 'google-cloud-sdk', 'bin', 'gcloud') - ].filter(Boolean) -} - -function exitWithUsage(message) { - if (message) { - console.error(message) - } - - console.error( - 'Usage: pnpm infra: --env staging|production [--root relay]' - ) - process.exit(1) -} diff --git a/cloud/dev/scripts/infra.test.mjs b/cloud/dev/scripts/infra.test.mjs deleted file mode 100644 index 267bcf540ac..00000000000 --- a/cloud/dev/scripts/infra.test.mjs +++ /dev/null @@ -1,77 +0,0 @@ -import assert from 'node:assert/strict' -import { execFileSync } from 'node:child_process' -import { readFileSync } from 'node:fs' -import { test } from 'node:test' -import { fileURLToPath } from 'node:url' - -const repository = fileURLToPath(new URL('../../', import.meta.url)) -const script = 'dev/scripts/infra.mjs' - -// IAC_TOOL=echo prints the argv the real binary would have received, so the root a flag selects -// is observable without running Terraform. -function invoke(args) { - return execFileSync('node', [script, ...args], { - cwd: repository, - encoding: 'utf8', - env: { ...process.env, IAC_TOOL: 'echo' } - }).trim() -} - -function rejects(args) { - try { - execFileSync('node', [script, ...args], { - cwd: repository, - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'pipe'], - env: { ...process.env, IAC_TOOL: 'echo' } - }) - } catch (error) { - return error.stderr - } - throw new Error(`expected ${args.join(' ')} to exit non-zero`) -} - -// Why: 9 relay workflows, the fence broker, and the three infra:* package scripts all invoke this -// without --root. If the default ever moves off infra/terraform they break silently at the plan. -test('omitting --root keeps every existing caller on the relay root', () => { - for (const environment of ['staging', 'production']) { - assert.equal( - invoke(['init', '--env', environment]), - `-chdir=infra/terraform init -backend-config=backend/${environment}.hcl` - ) - assert.equal( - invoke(['plan', '--env', environment]), - `-chdir=infra/terraform plan -var-file=environments/${environment}.tfvars -out=${environment}.tfplan` - ) - } -}) - -// Only the relay root ships here; the foundation and apps roots stay in the private repository. -test('each root name selects exactly its own directory', () => { - const directories = { relay: 'infra/terraform' } - for (const [root, directory] of Object.entries(directories)) { - for (const environment of ['staging', 'production']) { - assert.equal( - invoke(['init', '--env', environment, '--root', root]), - `-chdir=${directory} init -backend-config=backend/${environment}.hcl` - ) - } - } -}) - -test('an unknown root fails closed rather than falling back to the relay root', () => { - const stderr = rejects(['plan', '--env', 'staging', '--root', 'releay']) - assert.match(stderr, /Unknown --root/) - assert.doesNotMatch(stderr, /infra\/terraform /) -}) - -test('a missing environment still fails before any root is resolved', () => { - assert.match(rejects(['plan']), /Missing --env/) -}) - -// Why: the guard refuses a staging apply while the relay data plane is asleep. Applying it to the -// app or foundation roots would block work that never touches that data plane. -test('the sleeping staging relay guard is scoped to the relay root', () => { - const source = readFileSync(new URL('./infra.mjs', import.meta.url), 'utf8') - assert.match(source, /environment === 'staging' && root === 'relay'/) -}) diff --git a/cloud/dev/scripts/load-relay-controls.mjs b/cloud/dev/scripts/load-relay-controls.mjs deleted file mode 100644 index 8b736e2f25d..00000000000 --- a/cloud/dev/scripts/load-relay-controls.mjs +++ /dev/null @@ -1,695 +0,0 @@ -import { createHash, createPrivateKey, createPublicKey } from 'node:crypto' -import { readFileSync } from 'node:fs' -import { monitorEventLoopDelay } from 'node:perf_hooks' -import { setTimeout as delay } from 'node:timers/promises' -import { RelayLoadControlPeer } from './relay-load-control-peer.mjs' -import { RELAY_ASIA_LOAD_PRINCIPAL_SHARDS, requestGitHubSmokeTokens } from './github-smoke-token.mjs' -import { relayLoadFailureReason } from './relay-load-connection-failure.mjs' -import { - assertRelayLoadDirectorCapacityToken, - waitForRelayLoadDirectorCapacity, - waitForRelayLoadRequestUnits -} from './relay-load-director-capacity-gate.mjs' -import { waitForRelayLoadPhaseBarrier } from './relay-load-phase-barrier.mjs' -import { - proveRelayLoadPlacementBoundary, - proveRelayLoadRegionalFallback -} from './relay-load-placement-boundary.mjs' -import { - proveRelayLoadRebindBoundary, - waitForRelayLoadRebindGate -} from './relay-load-rebind-boundary.mjs' -import { proveRelayLoadRegionBehavior } from './relay-load-region-behavior.mjs' -import { proveRelayLoadDuplicateAssign } from './relay-load-duplicate-assign.mjs' -import { - openRelayLoadInviteOffers, - proveRelayLoadRequestUnitBoundary -} from './relay-load-request-unit-boundary.mjs' -import { - assertRelayLoadRampAccepted, - relayLoadRunHasDisallowedFailures, - runRelayLoadWithShutdown -} from './relay-load-run-lifecycle.mjs' -import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs' -import { holdRelayLoadSplice, relayLoadSpliceReopenSummary } from './relay-load-splice-retry.mjs' -import { - installRelayLoadResumeDevice, - relayLoadResumeSummary, - resumeRelayLoadPhoneOnce -} from './relay-load-resume-phone.mjs' -import { - parseRelayLoadArguments, - relayLoadPrincipalIndex, - relayLoadReaderEvidenceError, - relayLoadResumePhoneIndexes, - relayLoadSpliceIndexes, - relayLoadSpliceProfile, - relayLoadSpliceStartDelayMs -} from './relay-load-profile.mjs' - -function signingKey(path) { - if (!path) return {} - const key = createPrivateKey(readFileSync(path, 'utf8')) - const signingKeyId = createHash('sha256') - .update(createPublicKey(key).export({ type: 'spki', format: 'der' })) - .digest('base64url') - .slice(0, 16) - return { signingKey: key, signingKeyId } -} - -function report(state, final = false) { - const elapsedSeconds = Math.max(1, (Date.now() - state.startedAt) / 1000) - const memory = process.memoryUsage() - const cpu = process.cpuUsage(state.generatorBaselineCpu) - const rssMiB = memory.rss / 1_048_576 - state.generatorPeakRssMiB = Math.max(state.generatorPeakRssMiB, rssMiB) - const readerQueueEvidence = state.readerEvidence?.snapshot() ?? [] - const output = { - event: final ? 'relay_load_complete' : 'relay_load_progress', - controls: state.controls, - shardCount: state.shardCount, - shardIndex: state.shardIndex, - configuredRampSeconds: state.rampMs / 1000, - configuredSteadySeconds: state.durationMs / 1000, - configuredSpliceHoldSeconds: state.spliceHoldMs / 1000, - requiredLeaseHorizons: state.requiredLeaseHorizons, - configuredSplices: state.splices, - configuredSlowReaderSplices: state.slowReaderSplices, - configuredWedgedReaderSplices: state.wedgedReaderSplices, - active: state.active.size, - peakActive: state.peakActive, - steadyMinimumActive: state.steadyMinimumActive, - connected: state.connected, - connectionFailures: state.connectionFailures, - rampConnectionFailures: state.rampConnectionFailures, - steadyConnectionFailures: state.steadyConnectionFailures, - transitionConnectionFailures: state.transitionConnectionFailures, - connectionFailuresByReason: state.connectionFailuresByReason, - closes: state.closes, - unexpectedCloses: state.unexpectedCloses, - unexpectedClosesByCode: state.unexpectedClosesByCode, - drains: state.drains, - pings: state.pings, - pingRate: Number((state.pings / elapsedSeconds).toFixed(2)), - tokens: state.tokens, - tokenRate: Number((state.tokens / elapsedSeconds).toFixed(2)), - refreshes: state.refreshes, - refreshErrors: state.refreshErrors, - protocolErrors: state.protocolErrors, - socketErrors: state.socketErrors, - rebindProbesOpened: state.rebindProbesOpened, - rebindOverflowReason: state.rebindOverflowReason, - placementOverflowReason: state.placementOverflowReason, - regionalFallbacksProved: state.regionalFallbacksProved, - oldClientUsFirstProved: state.oldClientUsFirstProved, - stickyAssignmentProved: state.stickyAssignmentProved, - duplicateAssign: state.duplicateAssign, - requestUnitInvitesOpened: state.requestUnitInvitesOpened, - requestUnitPrincipalCount: state.requestUnitPrincipalCount, - relayAsiaLoadPrincipalCount: state.relayAsiaLoadPrincipalCount, - requestUnitOverflowReason: state.requestUnitOverflowReason, - requestUnitCleanupProved: state.requestUnitCleanupProved, - phaseBarrierPassed: state.phaseBarrierPassed, - activeSplices: state.activeSplices, - peakActiveSplices: state.peakActiveSplices, - completedSplices: state.completedSplices, - failedSplices: state.failedSplices, - spliceRetries: state.spliceRetries, - spliceFailuresByReason: state.spliceFailuresByReason, - spliceReopen: relayLoadSpliceReopenSummary(state.spliceReopenGapsMs, state.spliceFailedAt.size), - slowReaderSplicesCompleted: state.slowReaderSplicesCompleted, - wedgedReaderSplicesClosed: state.wedgedReaderSplicesClosed, - readerQueueEvidence, - readerQueuedBytesPeak: Math.max( - 0, - ...readerQueueEvidence.map(({ increaseBytes }) => increaseBytes) - ), - readerClosesByCode: state.readerClosesByCode, - resumePhones: relayLoadResumeSummary(state.resumeAttempts), - // Every cell a control connected to, so a canary can prove its load reached the target. - assignedCellOrigins: [...state.assignedCellOrigins].sort(), - controlHeadroom: Math.max(0, state.controls - state.active.size), - generatorRssMiB: Number(rssMiB.toFixed(1)), - generatorPeakRssMiB: Number(state.generatorPeakRssMiB.toFixed(1)), - generatorRssGrowthMiB: Number( - Math.max(0, state.generatorPeakRssMiB - state.generatorBaselineRssMiB).toFixed(1) - ), - generatorHeapUsedMiB: Number((memory.heapUsed / 1_048_576).toFixed(1)), - generatorCpuPercent: Number( - (((cpu.user + cpu.system) / 1_000_000 / elapsedSeconds) * 100).toFixed(1) - ), - generatorEventLoopP99Ms: Number((state.eventLoopDelay.percentile(99) / 1_000_000).toFixed(2)), - shutdownEvidence: final ? state.shutdownEvidence : undefined, - elapsedSeconds: Number(elapsedSeconds.toFixed(1)) - } - console.log(JSON.stringify(output)) - return output -} - -const config = parseRelayLoadArguments(process.argv.slice(2)) -let accessToken = process.env.ORCA_RELAY_LOAD_ACCESS_TOKEN -let accessTokenProviderForIndex -const adminToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN -if ( - config.placementOverflowProbes > 0 || config.regionalFallbackProbes > 0 || - config.slowReaderSplices + config.wedgedReaderSplices > 0 || - config.requestUnitOverflowProbes > 0 || config.requestUnitCleanupTimeoutMs > 0 -) { - assertRelayLoadDirectorCapacityToken({ - directorOrigin: config.directorOrigin, - adminToken - }, Date.now, - config.rampMs + config.durationMs + config.wedgedReaderHoldMs + - (config.phaseBarrierDir ? 2 * config.phaseBarrierTimeoutMs : 0) + - config.spliceRampMs + config.requestUnitCleanupTimeoutMs + 120_000) -} -const key = signingKey(config.signingKeyFile) -if (!accessToken && !key.signingKey && process.env.ACTIONS_ID_TOKEN_REQUEST_URL && - process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN) { - let tokens - let refresh - const loadOptions = config.relayAsiaLoadPrincipalCount > 0 - ? { - relayAsiaLoad: { - // Staging auth mints principal sets for shards 0-3 only; shards past that (a second - // hosted runner) reuse a set, and their hosts stay distinct by peer index. - shardIndex: config.shardIndex % RELAY_ASIA_LOAD_PRINCIPAL_SHARDS, - principalCount: config.relayAsiaLoadPrincipalCount - } - } - : undefined - const smokeTokens = async () => { - const expiresAt = config.relayAsiaLoadPrincipalCount > 0 - ? tokens?.relayAsiaLoadPrincipals?.[0]?.expiresAt - : tokens?.owner?.expiresAt - if (expiresAt > Date.now() + 60_000) return tokens - refresh ??= requestGitHubSmokeTokens( - config.authOrigin, - fetch, - process.env, - loadOptions - ) - try { - tokens = await refresh - return tokens - } finally { - refresh = undefined - } - } - accessTokenProviderForIndex = (index) => async () => { - const current = await smokeTokens() - return config.relayAsiaLoadPrincipalCount > 0 - ? current.relayAsiaLoadPrincipals[ - relayLoadPrincipalIndex( - index, - config.shardCount, - current.relayAsiaLoadPrincipals.length - ) - ].accessToken - : current.owner.accessToken - } - await smokeTokens() -} -if (!accessToken && !accessTokenProviderForIndex && !key.signingKey) { - throw new Error('provide GitHub OIDC, ORCA_RELAY_LOAD_ACCESS_TOKEN, or --signing-key-file') -} -const eventLoopDelay = monitorEventLoopDelay({ resolution: 20 }) -eventLoopDelay.enable() -const generatorBaselineRssMiB = process.memoryUsage().rss / 1_048_576 -const generatorBaselineCpu = process.cpuUsage() -const state = { - ...config, - startedAt: Date.now(), - active: new Set(), - peakActive: 0, - steadyMinimumActive: null, - steadyStarted: false, - connected: 0, - connectionFailures: 0, - rampConnectionFailures: 0, - steadyConnectionFailures: 0, - transitionConnectionFailures: 0, - connectionFailuresByReason: {}, - closes: 0, - unexpectedCloses: 0, - unexpectedClosesByCode: {}, - drains: 0, - pings: 0, - tokens: 0, - refreshes: 0, - refreshErrors: 0, - protocolErrors: 0, - socketErrors: 0, - rebindProbesOpened: 0, - rebindOverflowReason: null, - placementOverflowReason: null, - regionalFallbacksProved: 0, - oldClientUsFirstProved: 0, - stickyAssignmentProved: 0, - duplicateAssign: null, - requestUnitInvitesOpened: 0, - requestUnitOverflowReason: null, - requestUnitCleanupProved: 0, - phaseBarrierPassed: false, - activeSplices: 0, - peakActiveSplices: 0, - completedSplices: 0, - failedSplices: 0, - spliceRetries: 0, - spliceFailuresByReason: {}, - spliceFailedAt: new Map(), - resumeAttempts: [], - spliceReopenGapsMs: [], - slowReaderSplicesCompleted: 0, - wedgedReaderSplicesClosed: 0, - readerEvidence: null, - readerClosesByCode: {}, - assignedCellOrigins: new Set(), - generatorBaselineRssMiB, - generatorBaselineCpu, - generatorPeakRssMiB: generatorBaselineRssMiB, - peerShutdowns: 0, - shutdownEvidence: null, - eventLoopDelay, - stopping: false, - transitionWindow: false -} -const peers = new Map() -const reconnectTimers = new Set() -// Duplicate-assign probe hosts: outside `peers`, so they never count toward the run's controls. -const probePeers = [] - -async function readRuntimeQueuedBytes(origin) { - const response = await fetch(`${origin}/v1/admin/runtime-status`, { - method: 'POST', - headers: { authorization: `Bearer ${adminToken}`, 'content-type': 'application/json' }, - body: JSON.stringify({ v: 1 }), - signal: AbortSignal.timeout(5_000) - }) - if (response.status === 401 || response.status === 403) { - throw new Error('reader evidence identity was rejected') - } - if (!response.ok) throw new Error(`reader runtime status returned ${response.status}`) - const status = await response.json() - const queuedBytes = status?.runtime?.queuedBytes - if (!Number.isSafeInteger(queuedBytes) || queuedBytes < 0) { - throw new Error('reader runtime queued bytes are invalid') - } - return queuedBytes -} - -async function observeReaderPressure(input) { - if (!state.readerEvidence) throw new Error('reader evidence baseline is unavailable') - await state.readerEvidence.observe(input) - state.generatorPeakRssMiB = Math.max( - state.generatorPeakRssMiB, - process.memoryUsage().rss / 1_048_576 - ) -} - -function recordSteadyMinimum() { - if (!state.steadyStarted || state.stopping) return - state.steadyMinimumActive = Math.min(state.steadyMinimumActive, state.active.size) -} - -function scheduleReconnect(peer) { - if (state.stopping) return - const timeout = setTimeout(() => { - reconnectTimers.delete(timeout) - void connect(peer) - }, Math.floor(Math.random() * (config.reconnectMaxMs + 1))) - reconnectTimers.add(timeout) -} - -function observe(type, detail) { - if (type === 'connected') { - state.active.add(detail.index) - state.connected++ - const cellOrigin = peers.get(detail.index)?.assignedCellUrl() - if (cellOrigin) state.assignedCellOrigins.add(cellOrigin) - state.peakActive = Math.max(state.peakActive, state.active.size) - recordSteadyMinimum() - } else if (type === 'closed') { - state.active.delete(detail.index) - state.closes++ - if (!detail.stopped && !detail.expectedDrain) { - state.unexpectedCloses++ - const code = String(detail.code) - state.unexpectedClosesByCode[code] = (state.unexpectedClosesByCode[code] ?? 0) + 1 - } - if (!detail.stopped) scheduleReconnect(peers.get(detail.index)) - recordSteadyMinimum() - } else if (type === 'drain') state.drains++ - else if (type === 'ping') state.pings++ - else if (type === 'token') state.tokens++ - else if (type === 'refresh') state.refreshes++ - else if (type === 'refreshError') state.refreshErrors++ - else if (type === 'protocolError') state.protocolErrors++ - else if (type === 'socketError') state.socketErrors++ - else if (type === 'spliceOpened') { - const failedAt = state.spliceFailedAt.get(detail.index) - if (failedAt !== undefined) { - state.spliceReopenGapsMs.push(Date.now() - failedAt) - state.spliceFailedAt.delete(detail.index) - } - state.activeSplices++ - state.peakActiveSplices = Math.max(state.peakActiveSplices, state.activeSplices) - } else if (type === 'spliceCompleted') { - state.completedSplices++ - if (detail.readerMode === 'slow') state.slowReaderSplicesCompleted++ - } else if (type === 'spliceWedged') { - state.wedgedReaderSplicesClosed++ - const code = String(detail.code) - state.readerClosesByCode[code] = (state.readerClosesByCode[code] ?? 0) + 1 - } else if (type === 'spliceClosed') state.activeSplices-- - else if (type === 'spliceFailed') state.failedSplices++ - else if (type === 'shutdown') state.peerShutdowns++ -} - -async function connect(peer) { - try { - await peer.connect() - } catch (error) { - state.connectionFailures++ - if (state.steadyStarted) state.steadyConnectionFailures++ - else if (state.transitionWindow) state.transitionConnectionFailures++ - else state.rampConnectionFailures++ - const reason = relayLoadFailureReason(error) - state.connectionFailuresByReason[reason] = - (state.connectionFailuresByReason[reason] ?? 0) + 1 - scheduleReconnect(peer) - } -} - -const peerOptions = (index, overrides = {}) => ({ - ...config, - ...key, - accessToken, - ...(accessTokenProviderForIndex - ? { accessTokenProvider: accessTokenProviderForIndex(index) } - : {}), - seed: 0x4f524341 ^ config.shardIndex, - ...overrides -}) -if (config.regionBehaviorProbes > 0) { - const proofIndex = config.controls * config.shardCount + 10_000 - const regionProof = await proveRelayLoadRegionBehavior({ - oldClientPeer: new RelayLoadControlPeer( - proofIndex, - peerOptions(proofIndex, { preferredRegion: undefined }), - () => undefined - ), - stickyPeer: new RelayLoadControlPeer( - proofIndex + 1, - peerOptions(proofIndex + 1, { preferredRegion: 'asia-east2' }), - () => undefined - ), - asiaOrigin: config.capacityCellOrigin - }) - state.oldClientUsFirstProved = regionProof.oldClientUsFirst ? 1 : 0 - state.stickyAssignmentProved = regionProof.stickyAssignmentPreserved ? 1 : 0 -} -const initialConnections = [] -for (let localIndex = 0; localIndex < config.controls; localIndex++) { - const globalIndex = localIndex * config.shardCount + config.shardIndex - const peer = new RelayLoadControlPeer(globalIndex, peerOptions(globalIndex), observe) - peers.set(globalIndex, peer) - const rampOffset = - config.controls === 1 ? 0 : Math.floor((localIndex / (config.controls - 1)) * config.rampMs) - const offset = config.rampStartDelayMs + rampOffset - initialConnections.push(delay(offset).then(() => connect(peer))) -} -function recordResumeAttempt(peer, attempt) { - state.resumeAttempts.push(attempt) - console.log(JSON.stringify({ event: 'relay_load_resume_attempt', index: peer.index, ...attempt })) -} -// A paired phone that reconnects every interval until the run stops; every attempt is a line. -async function runResumePhone(peer, localIndex, endsAt) { - const relayDeviceId = `load-resume-device-${peer.index}` - // Spread the phones across one interval. - await delay(Math.floor((localIndex / Math.max(1, config.resumePhones)) * config.resumeIntervalMs)) - let token - while (!state.stopping && Date.now() < endsAt) { - const startedAt = Date.now() - if (!token) { - try { - token = await installRelayLoadResumeDevice(peer, relayDeviceId) - } catch (error) { - recordResumeAttempt(peer, { - at: startedAt, - ok: false, - failure: 'install', - error: String(error.message).slice(0, 160) - }) - } - } else { - recordResumeAttempt(peer, await resumeRelayLoadPhoneOnce(peer, { token, relayDeviceId }, { - holdMs: config.resumeHoldMs, - stopping: () => state.stopping, - delay - })) - } - await delay(Math.min( - Math.max(1_000, config.resumeIntervalMs - (Date.now() - startedAt)), - Math.max(0, endsAt - Date.now()) - )) - } -} -const progressTimer = setInterval(() => report(state), 10_000) -progressTimer.unref() -await runRelayLoadWithShutdown(async () => { - await Promise.all(initialConnections) - assertRelayLoadRampAccepted(state.rampConnectionFailures, config.maxRampConnectionFailures) - if (config.duplicateAssignProbes > 0) { - const reconnects = { closes: 0, closesByCode: {}, reconnectFailures: 0 } - // A probe host closed by a demotion re-assigns through the director, as a desktop would. - const observeProbe = (peer) => (type, detail) => { - if (type !== 'closed' || detail.stopped) return - reconnects.closes++ - const code = String(detail.code) - reconnects.closesByCode[code] = (reconnects.closesByCode[code] ?? 0) + 1 - if (state.stopping) return - const timeout = setTimeout(() => { - reconnectTimers.delete(timeout) - void peer.connect().catch(() => { - reconnects.reconnectFailures++ - }) - }, 1_000 + Math.floor(Math.random() * 5_000)) - reconnectTimers.add(timeout) - } - const base = config.controls * config.shardCount + 20_000 - const pairs = Array.from({ length: config.duplicateAssignProbes }, (_, probe) => { - const index = base + probe * config.shardCount + config.shardIndex - const primary = new RelayLoadControlPeer(index, peerOptions(index), (type, detail) => - observeProbe(primary)(type, detail) - ) - const twin = new RelayLoadControlPeer( - index, - peerOptions(index, { keys: primary.keys }), - () => undefined - ) - probePeers.push(primary, twin) - return { primary, twin } - }) - state.duplicateAssign = { - ...(await proveRelayLoadDuplicateAssign({ - pairs, - holdMs: config.duplicateAssignHoldMs, - delay, - failureReason: relayLoadFailureReason - })), - probeReconnects: reconnects - } - } - if ( - config.rebindProbes > 0 || config.placementOverflowProbes > 0 || - config.regionalFallbackProbes > 0 - ) { - state.transitionWindow = config.rebindDelayMs > 0 - await waitForRelayLoadRebindGate({ - delay, - delayMs: config.rebindDelayMs, - activeCount: () => state.active.size, - requiredCount: config.controls - }) - state.transitionWindow = false - } - if (config.placementOverflowProbes > 0 || config.regionalFallbackProbes > 0) { - const closesBeforeBoundary = state.closes - await waitForRelayLoadDirectorCapacity({ - directorOrigin: config.directorOrigin, - adminToken, - cellId: config.capacityCellId, - hardCap: config.capacityHardCap, - unobservedBound: config.capacityUnobservedBound, - requiredConnections: config.aggregateControls - }) - if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) { - throw new Error('ordinary controls changed during the director capacity gate') - } - const overflowIndex = config.controls * config.shardCount + config.shardIndex - if (config.placementOverflowProbes > 0) { - state.placementOverflowReason = await proveRelayLoadPlacementBoundary({ - peer: new RelayLoadControlPeer(overflowIndex, peerOptions(overflowIndex), observe), - failureReason: relayLoadFailureReason - }) - } - if (config.regionalFallbackProbes > 0) { - await proveRelayLoadRegionalFallback({ - peer: new RelayLoadControlPeer(overflowIndex, peerOptions(overflowIndex), () => undefined), - blockedOrigin: config.capacityCellOrigin - }) - state.regionalFallbacksProved = 1 - } - if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) { - throw new Error('ordinary controls changed during the placement boundary probe') - } - await waitForRelayLoadDirectorCapacity({ - directorOrigin: config.directorOrigin, - adminToken, - cellId: config.capacityCellId, - hardCap: config.capacityHardCap, - unobservedBound: config.capacityUnobservedBound, - requiredConnections: config.aggregateControls, - requiredSamples: 1 - }) - if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) { - throw new Error('ordinary controls changed before post-probe capacity verification') - } - } - const rebindResult = await proveRelayLoadRebindBoundary({ - peers: [...state.active].map((index) => peers.get(index)), - probeCount: config.rebindProbes, - holdMs: config.rebindHoldMs, - delay, - failureReason: relayLoadFailureReason, - requireOverflow: config.requireRebindOverflow - }) - state.rebindProbesOpened = rebindResult.opened - state.rebindOverflowReason = rebindResult.overflowReason - if (config.requestUnitInvites > 0) { - state.requestUnitInvitesOpened = await openRelayLoadInviteOffers({ - peers: [...state.active].sort((left, right) => left - right).map((index) => peers.get(index)), - count: config.requestUnitInvites, - ratePerSecond: config.requestUnitInvitesPerSecond - }) - } - if (config.requestUnitOverflowProbes > 0) { - await waitForRelayLoadRequestUnits({ - directorOrigin: config.directorOrigin, - adminToken, - cellId: config.capacityCellId, - capacityRequests: config.requestUnitCapacity, - expectedRequestUnits: config.requestUnitCapacity, - expectedActivityLeases: config.requestUnitCapacity - }) - state.requestUnitOverflowReason = await proveRelayLoadRequestUnitBoundary( - peers.get([...state.active][0]) - ) - } - if (config.phaseBarrierDir) { - await waitForRelayLoadPhaseBarrier({ - directory: config.phaseBarrierDir, - shardCount: config.shardCount, - shardIndex: config.shardIndex, - timeoutMs: config.phaseBarrierTimeoutMs - }) - state.phaseBarrierPassed = true - } - state.steadyStarted = true - state.steadyMinimumActive = state.active.size - const spliceIndexes = relayLoadSpliceIndexes(config) - const readerOrigins = spliceIndexes.flatMap((index, spliceIndex) => - relayLoadSpliceProfile(config, spliceIndex).readerMode === 'normal' - ? [] - : [peers.get(index).lastAssignment.cellUrl] - ) - state.readerEvidence = await createRelayLoadReaderEvidence(readerOrigins, { - readQueuedBytes: readRuntimeQueuedBytes, - delay - }) - if (config.phaseBarrierDir) { - await waitForRelayLoadPhaseBarrier({ - directory: `${config.phaseBarrierDir}-splices`, - shardCount: config.shardCount, - shardIndex: config.shardIndex, - timeoutMs: config.phaseBarrierTimeoutMs - }) - } - const splicePromises = spliceIndexes.map((index, spliceIndex) => - delay(relayLoadSpliceStartDelayMs(config, spliceIndex)).then(() => - holdRelayLoadSplice({ - open: () => - peers.get(index).openSplice({ - payloadBytes: config.splicePayloadBytes, - ...relayLoadSpliceProfile(config, spliceIndex), - observeReaderPressure, - holdMs: config.spliceHoldMs - }), - retryMaxMs: config.spliceRetryMaxMs, - stopping: () => state.stopping, - onFailure: (reason) => { - state.spliceRetries++ - state.spliceFailuresByReason[reason] = (state.spliceFailuresByReason[reason] ?? 0) + 1 - if (!state.spliceFailedAt.has(index)) state.spliceFailedAt.set(index, Date.now()) - }, - delay - }) - ) - ) - const resumeEndsAt = Date.now() + config.durationMs - const resumePromises = relayLoadResumePhoneIndexes(config).map((index, localIndex) => - runResumePhone(peers.get(index), localIndex, resumeEndsAt) - ) - await Promise.all([...splicePromises, ...resumePromises, delay(config.durationMs)]) -}, async () => { - state.stopping = true - clearInterval(progressTimer) - for (const timeout of reconnectTimers) clearTimeout(timeout) - reconnectTimers.clear() - await Promise.all([...peers.values(), ...probePeers].map((peer) => peer.shutdown())) - eventLoopDelay.disable() - state.shutdownEvidence = { - peerShutdowns: state.peerShutdowns, - activeControls: state.active.size, - activeSplices: state.activeSplices, - reconnectTimers: reconnectTimers.size - } -}) -if (config.requestUnitCleanupTimeoutMs > 0) { - await waitForRelayLoadRequestUnits({ - directorOrigin: config.directorOrigin, - adminToken, - cellId: config.capacityCellId, - capacityRequests: config.requestUnitCapacity, - expectedRequestUnits: 0, - expectedActivityLeases: 0, - timeoutMs: config.requestUnitCleanupTimeoutMs - }) - state.requestUnitCleanupProved = 1 -} -const result = report(state, true) -const minimumPeak = config.allowPartial ? 1 : Math.ceil(config.controls * 0.95) -if (result.peakActive < minimumPeak) { - throw new Error(`peak active controls ${result.peakActive} below required ${minimumPeak}`) -} -if (result.steadyMinimumActive < minimumPeak) { - throw new Error( - `steady minimum active controls ${result.steadyMinimumActive} below required ${minimumPeak}` - ) -} -if (relayLoadRunHasDisallowedFailures(result, config)) { - throw new Error('relay load run observed connection, protocol, refresh, or socket errors') -} -const readerEvidenceError = relayLoadReaderEvidenceError(result, config) -if (readerEvidenceError) throw new Error(readerEvidenceError) -if ( - result.failedSplices > 0 || - result.completedSplices + result.wedgedReaderSplicesClosed !== config.splices || - result.shutdownEvidence.peerShutdowns !== config.controls || - result.shutdownEvidence.activeControls !== 0 || - result.shutdownEvidence.activeSplices !== 0 || - result.shutdownEvidence.reconnectTimers !== 0 -) { - throw new Error('relay load run did not complete splices or shut down cleanly') -} diff --git a/cloud/dev/scripts/measure-relay-drain-disconnect-gap.mjs b/cloud/dev/scripts/measure-relay-drain-disconnect-gap.mjs deleted file mode 100644 index 9487c24b21f..00000000000 --- a/cloud/dev/scripts/measure-relay-drain-disconnect-gap.mjs +++ /dev/null @@ -1,161 +0,0 @@ -import { readFileSync } from 'node:fs' -import { pathToFileURL } from 'node:url' - -// Per-desktop disconnect gap for one drained cell, read from log lines that already ship: -// the source cell's `control closed` line and the director's reconnect `assignment granted` -// line (drains send `reconnect: true`, so every drained desktop's grant carries `hinted=true`). -// Read-only: the operator exports both logs; this file never queries anything. - -const CLOSE = /^\[orca-relay\] control closed host=(\S+) .*?\bsplices=(\d+) .*?\bcode=(\d+) reason=("(?:[^"\\]|\\.)*")/ -const GRANT = /^\[orca-relay\] assignment granted lane=\S+ hinted=true host=(\S+) cell=(\S+)$/ -// The cell's own drain close; a desktop closed this way had not moved yet, idle or not. -const DRAIN_CLOSE_REASON = 'resolve configured director' - -function entryText(entry) { - if (typeof entry.textPayload === 'string') return entry.textPayload - if (typeof entry.jsonPayload?.message === 'string') return entry.jsonPayload.message - return null -} - -function entryTime(entry) { - const at = Date.parse(entry.timestamp) - if (!Number.isFinite(at)) throw new Error('log entry has no timestamp') - return at -} - -export function readDrainCloses(entries) { - const closes = [] - for (const entry of entries) { - const match = CLOSE.exec(entryText(entry) ?? '') - if (!match) continue - closes.push({ - host: match[1], - splices: Number(match[2]), - code: Number(match[3]), - reason: JSON.parse(match[4]), - at: entryTime(entry) - }) - } - // gcloud exports newest first; the measurement needs each host's earliest close. - return closes.sort((left, right) => left.at - right.at) -} - -export function readReconnectGrants(entries) { - const grants = [] - for (const entry of entries) { - const match = GRANT.exec(entryText(entry) ?? '') - if (match) grants.push({ host: match[1], cellId: match[2], at: entryTime(entry) }) - } - return grants.sort((left, right) => left.at - right.at) -} - -function percentile(sorted, fraction) { - if (sorted.length === 0) return null - return sorted[Math.min(sorted.length - 1, Math.ceil(fraction * sorted.length) - 1)] -} - -function summary(values) { - const sorted = [...values].sort((left, right) => left - right) - return { - count: sorted.length, - p50Ms: percentile(sorted, 0.5), - p95Ms: percentile(sorted, 0.95), - maxMs: sorted.at(-1) ?? null - } -} - -// `controlsAtDrainStart` is the denominator: hosts the cell held when the drain began, -// read from its runtime metrics line, so a desktop that never logged a close still counts. -// `drainEndedAt` closes the window: after it the rolled cell's new container serves new sessions, -// and their closes are not part of the drain. Grants after it still count, as the gap's far end. -export function measureDrainDisconnectGap({ - sourceCellId, - drainStartedAt, - drainEndedAt, - controlsAtDrainStart, - closes, - grants, - cellRegions = {} -}) { - if (!Number.isFinite(drainStartedAt)) throw new Error('drain start time is invalid') - if (!Number.isFinite(drainEndedAt) || drainEndedAt <= drainStartedAt) { - throw new Error('drain end time is invalid') - } - const sourceRegion = cellRegions[sourceCellId] - // First close per host after the drain began; later closes are the host's new sessions. - const firstClose = new Map() - for (const close of closes) { - if (close.at < drainStartedAt || close.at > drainEndedAt || firstClose.has(close.host)) continue - firstClose.set(close.host, close) - } - const grantsByHost = new Map() - for (const grant of grants) { - if (grant.at < drainStartedAt || grant.cellId === sourceCellId) continue - grantsByHost.set(grant.host, [...(grantsByHost.get(grant.host) ?? []), grant]) - } - const cutOffGapsMs = [] - const counts = { movedFirst: 0, cutOff: 0, cutOffUnresolved: 0, otherClose: 0 } - const leftRegion = [] - let phoneSessionsDropped = 0 - for (const close of firstClose.values()) { - phoneSessionsDropped += close.splices - const hostGrants = grantsByHost.get(close.host) ?? [] - const first = hostGrants[0] - if (first && first.at <= close.at) counts.movedFirst += 1 - else if (close.reason === DRAIN_CLOSE_REASON) { - if (first) { - counts.cutOff += 1 - cutOffGapsMs.push(first.at - close.at) - } else counts.cutOffUnresolved += 1 - } else counts.otherClose += 1 - if (first && sourceRegion && cellRegions[first.cellId] && cellRegions[first.cellId] !== sourceRegion) { - const back = hostGrants.find( - (grant) => grant.at > first.at && cellRegions[grant.cellId] === sourceRegion - ) - leftRegion.push(back ? back.at - first.at : null) - } - } - const returned = leftRegion.filter((value) => value !== null) - return { - sourceCellId, - controlsAtDrainStart, - closedHosts: firstClose.size, - // Below 1 means some drained desktops left no close line in the export; widen it. - closeCoverage: controlsAtDrainStart > 0 ? firstClose.size / controlsAtDrainStart : null, - ...counts, - // Lower bound until the target cells run an image that logs control activation. - cutOffGap: summary(cutOffGapsMs), - phoneSessionsDropped, - leftSourceRegion: leftRegion.length, - leftSourceRegionStillAway: leftRegion.length - returned.length, - timeUntilBack: summary(returned) - } -} - -function argument(name) { - const index = process.argv.indexOf(`--${name}`) - return index === -1 ? undefined : process.argv[index + 1] -} - -function required(name) { - const value = argument(name) - if (value === undefined) throw new Error(`--${name} is required`) - return value -} - -function main() { - const readJson = (path) => JSON.parse(readFileSync(path, 'utf8')) - const cellRegionsPath = argument('cell-regions') - const result = measureDrainDisconnectGap({ - sourceCellId: required('source-cell'), - drainStartedAt: Date.parse(required('drain-started-at')), - drainEndedAt: Date.parse(required('drain-ended-at')), - controlsAtDrainStart: Number(required('controls')), - closes: readDrainCloses(readJson(required('cell-log'))), - grants: readReconnectGrants(readJson(required('director-log'))), - cellRegions: cellRegionsPath ? readJson(cellRegionsPath) : {} - }) - console.log(JSON.stringify(result, null, 2)) -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) main() diff --git a/cloud/dev/scripts/measure-relay-drain-disconnect-gap.test.mjs b/cloud/dev/scripts/measure-relay-drain-disconnect-gap.test.mjs deleted file mode 100644 index 900cd58d7dc..00000000000 --- a/cloud/dev/scripts/measure-relay-drain-disconnect-gap.test.mjs +++ /dev/null @@ -1,158 +0,0 @@ -import assert from 'node:assert/strict' -import { describe, it } from 'node:test' -import { - measureDrainDisconnectGap, - readDrainCloses, - readReconnectGrants -} from './measure-relay-drain-disconnect-gap.mjs' - -const start = Date.parse('2026-10-06T10:00:00Z') -const at = (seconds) => new Date(start + seconds * 1000).toISOString() - -function close(host, seconds, reason, splices = 0) { - return { - timestamp: at(seconds), - jsonPayload: { - message: - `[orca-relay] control closed host=${host} gen=3 state=closed ageMs=100 app="1.4.0"` + - ` splices=${splices} pending=0 code=4001 reason=${JSON.stringify(reason)}` - } - } -} - -function grant(host, seconds, cellId) { - return { - timestamp: at(seconds), - textPayload: `[orca-relay] assignment granted lane=drain-return hinted=true host=${host} cell=${cellId}` - } -} - -describe('drain disconnect gap', () => { - it('splits moved-first, cut-off and unresolved desktops and sums dropped phone sessions', () => { - const closes = readDrainCloses([ - close('h-moved', 30, 'migration completed', 2), - close('h-cut', 300, 'resolve configured director', 1), - close('h-idle', 310, 'resolve configured director'), - close('h-lost', 320, 'resolve configured director'), - close('h-early', -5, 'resolve configured director'), - // A later close of the same host is its next session, not the drain. - close('h-cut', 900, 'resolve configured director', 7) - ]) - const grants = readReconnectGrants([ - grant('h-moved', 20, 'c2'), - grant('h-cut', 304, 'c2'), - grant('h-idle', 330, 'c3'), - grant('h-other', 5, 'c2'), - grant('h-cut', 290, 'c1') - ]) - const result = measureDrainDisconnectGap({ - sourceCellId: 'c1', - drainStartedAt: start, - drainEndedAt: start + 1_200_000, - controlsAtDrainStart: 5, - closes, - grants - }) - assert.equal(result.closedHosts, 4) - assert.equal(result.closeCoverage, 0.8) - assert.equal(result.movedFirst, 1) - assert.equal(result.cutOff, 2) - assert.equal(result.cutOffUnresolved, 1) - assert.equal(result.otherClose, 0) - assert.deepEqual(result.cutOffGap, { count: 2, p50Ms: 4000, p95Ms: 20000, maxMs: 20000 }) - assert.equal(result.phoneSessionsDropped, 3) - }) - - it('times desktops that left the source region until a grant brings them back', () => { - const result = measureDrainDisconnectGap({ - sourceCellId: 'a1', - drainStartedAt: start, - drainEndedAt: start + 1_200_000, - controlsAtDrainStart: 2, - closes: readDrainCloses([ - close('h-away', 100, 'resolve configured director'), - close('h-stuck', 100, 'resolve configured director') - ]), - grants: readReconnectGrants([ - grant('h-away', 101, 'u1'), - grant('h-away', 3701, 'a2'), - grant('h-stuck', 102, 'u1') - ]), - cellRegions: { a1: 'asia-east2', a2: 'asia-east2', u1: 'us-central1' } - }) - assert.equal(result.leftSourceRegion, 2) - assert.equal(result.leftSourceRegionStillAway, 1) - assert.equal(result.timeUntilBack.maxMs, 3_600_000) - }) - - it('keeps each host earliest close when the export is newest first', () => { - const result = measureDrainDisconnectGap({ - sourceCellId: 'c1', - drainStartedAt: start, - drainEndedAt: start + 1_200_000, - controlsAtDrainStart: 1, - closes: readDrainCloses([ - close('h', 300, 'resolve configured director'), - close('h', 60, 'resolve configured director', 3) - ]), - grants: readReconnectGrants([grant('h', 120, 'c2')]) - }) - assert.equal(result.movedFirst, 0) - assert.equal(result.cutOff, 1) - assert.deepEqual(result.cutOffGap, { count: 1, p50Ms: 60000, p95Ms: 60000, maxMs: 60000 }) - assert.equal(result.phoneSessionsDropped, 3) - }) - - it('refuses an invalid drain start time', () => { - assert.throws( - () => - measureDrainDisconnectGap({ - sourceCellId: 'c1', - drainStartedAt: Date.parse('not a time'), - drainEndedAt: start, - controlsAtDrainStart: 1, - closes: [], - grants: [] - }), - /drain start time is invalid/ - ) - }) - - it('leaves out closes after the drain ended and refuses a bad end time', () => { - const input = { - sourceCellId: 'c1', - drainStartedAt: start, - drainEndedAt: start + 600_000, - controlsAtDrainStart: 1, - closes: readDrainCloses([ - close('h-drained', 100, 'resolve configured director'), - // The new container's session after the roll. - close('h-new', 700, '', 4) - ]), - grants: readReconnectGrants([grant('h-drained', 650, 'c2')]) - } - const result = measureDrainDisconnectGap(input) - assert.equal(result.closedHosts, 1) - assert.equal(result.otherClose, 0) - assert.equal(result.phoneSessionsDropped, 0) - // A grant after the window still ends that host's gap. - assert.equal(result.cutOffGap.maxMs, 550_000) - for (const drainEndedAt of [Number.NaN, start]) { - assert.throws( - () => measureDrainDisconnectGap({ ...input, drainEndedAt }), - /drain end time is invalid/ - ) - } - }) - - it('ignores lines that are not the two it reads', () => { - assert.deepEqual(readDrainCloses([{ timestamp: at(0), textPayload: 'unrelated' }]), []) - assert.deepEqual( - readReconnectGrants([grant('h', 0, 'c1')].map((entry) => ({ - ...entry, - textPayload: entry.textPayload.replace('hinted=true', 'hinted=false') - }))), - [] - ) - }) -}) diff --git a/cloud/dev/scripts/operate-relay-asia-admission.mjs b/cloud/dev/scripts/operate-relay-asia-admission.mjs deleted file mode 100644 index 2a3189944bc..00000000000 --- a/cloud/dev/scripts/operate-relay-asia-admission.mjs +++ /dev/null @@ -1,509 +0,0 @@ -import { createHash } from 'node:crypto' -import { fileURLToPath } from 'node:url' -import { - addExactMigrationCells, - applyExactAdmissionSelector, - inspectAdmissionSelector, - membershipWithStates, - selectorCellState -} from './relay-admission-selector.mjs' - -const SHAPES = { - staging: { - directorOrigin: 'https://relay-staging.onorca.dev', - domain: 'relay-staging.onorca.dev', - allCells: ['staging-gce-c4', 'staging-gce-c5', 'staging-gce-c6', 'staging-gce-c7'], - registrationWaves: [['staging-gce-c4'], ['staging-gce-c5', 'staging-gce-c6', 'staging-gce-c7']], - promotionWaves: [['staging-gce-c4'], ['staging-gce-c5', 'staging-gce-c6', 'staging-gce-c7']] - }, - production: { - directorOrigin: 'https://relay.onorca.dev', - domain: 'relay.onorca.dev', - allCells: [ - 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30', - 'production-gce-c31', 'production-gce-c32', 'production-gce-c33', 'production-gce-c34', - 'production-gce-c35' - ], - // The launch set was registered together; each later cell registers alone beside it. - registrationWaves: [ - ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - ['production-gce-c30'], - ['production-gce-c31'], - ['production-gce-c32'], - ['production-gce-c33'], - ['production-gce-c34'], - ['production-gce-c35'] - ], - promotionWaves: [ - ['production-gce-c27'], - ['production-gce-c28', 'production-gce-c29'], - ['production-gce-c30'], - ['production-gce-c31'], - ['production-gce-c32'], - ['production-gce-c33'], - ['production-gce-c34'], - ['production-gce-c35'] - ] - } -} - -const PRODUCTION_CANARY_CELL = 'production-gce-c27' -const STAGING_PROOF_CELL = 'staging-gce-c4' -const ASIA_REGION = 'asia-east2' -const ASIA_CELL_SHAPE = { region: ASIA_REGION, capacityRequests: 6_000, connectionHardCap: 3_000 } -// The US cells at the Asia shape live in the root region; every other cell here is Asia, except -// the staging dead-cell guard cells, which keep c2's ordinary US shape (staging.tfvars). -const STAGING_US_CELL_SHAPE = { region: 'us-central1', capacityRequests: 4_000, connectionHardCap: 600 } -const CELL_SHAPES = { - 'production-gce-c32': { ...ASIA_CELL_SHAPE, region: 'us-central1' }, - 'production-gce-c33': { ...ASIA_CELL_SHAPE, region: 'us-central1' }, - 'staging-gce-c5': STAGING_US_CELL_SHAPE, - 'staging-gce-c6': STAGING_US_CELL_SHAPE, - 'staging-gce-c7': STAGING_US_CELL_SHAPE -} - -function cellShape(cellId) { - return CELL_SHAPES[cellId] ?? ASIA_CELL_SHAPE -} - -function cellRegion(cellId) { - return cellShape(cellId).region -} - -export function parseRelayAsiaAdmissionArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of ['environment', 'mode', 'cell-ids', 'image-digest']) { - if (!values[key]) throw new Error(`missing --${key}`) - } - if (!/^sha256:[a-f0-9]{64}$/.test(values['image-digest'])) { - throw new Error('--image-digest is invalid') - } - if (![ - 'inspect', 'initialize', 'verify', 'registered', 'register', - 'promote', 'recover-promotion', 'rollback' - ].includes(values.mode)) { - throw new Error('--mode is invalid') - } - const expectedGeneration = values.mode === 'inspect' - ? undefined - : Number(values['expected-generation']) - if ( - values.mode !== 'inspect' && - (!Number.isSafeInteger(expectedGeneration) || expectedGeneration < 0) - ) { - throw new Error('--expected-generation is invalid') - } - const shape = SHAPES[values.environment] - if (!shape) throw new Error('--environment is invalid') - const cells = values['cell-ids'].split(',').map((value) => value.trim()).filter(Boolean) - const distinct = new Set(cells) - if (distinct.size !== cells.length || cells.some((cell) => !shape.allCells.includes(cell))) { - throw new Error('--cell-ids are invalid') - } - const exact = (expected) => JSON.stringify([...cells].sort()) === JSON.stringify([...expected].sort()) - const [launchWave] = shape.registrationWaves - if ( - (['inspect', 'verify'].includes(values.mode) && - !exact(shape.allCells) && !shape.registrationWaves.some(exact)) || - // Generation zero predates every later cell, so the boundary freezes only the launch set. - (values.mode === 'initialize' && !exact(launchWave)) || - (['register', 'registered'].includes(values.mode) && !shape.registrationWaves.some(exact)) || - (['promote', 'recover-promotion'].includes(values.mode) && !shape.promotionWaves.some(exact)) || - // Rollback takes any reviewed wave or the whole set, never a mixed partial set. - (values.mode === 'rollback' && !exact(shape.allCells) && - ![...shape.registrationWaves, ...shape.promotionWaves].some(exact)) - ) throw new Error('--cell-ids do not match the reviewed admission wave') - const attemptId = values['attempt-id'] - if (!['inspect', 'verify', 'registered'].includes(values.mode) && - !/^[A-Za-z0-9_-]{8,128}$/.test(attemptId ?? '')) { - throw new Error('--attempt-id is invalid') - } - return { - environment: values.environment, - mode: values.mode, - cells, - expectedGeneration, - expectedMembershipSha256: values['expected-membership-sha256'], - imageDigest: values['image-digest'], - attemptId, - token: process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '' - } -} - -function hostname(cellId) { - return cellId.split('-').at(-1) -} - -function cellOrigin(shape, cellId) { - return `https://${hostname(cellId)}.${shape.domain}` -} - -async function responseJson(response, label) { - const body = await response.json().catch(() => ({})) - if (!response.ok) throw new Error(`${label} returned ${response.status}`) - return body -} - -function defaultPost(fetchImpl, token) { - return async (url, body) => await responseJson(await fetchImpl(url, { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(30_000) - }), new URL(url).pathname) -} - -async function verifyRuntime(fetchImpl, post, shape, cellId, imageDigest, requireDirector) { - const origin = cellOrigin(shape, cellId) - const [health, ready, runtime] = await Promise.all([ - fetchImpl(`${origin}/health`, { redirect: 'error', signal: AbortSignal.timeout(8_000) }), - fetchImpl(`${origin}/ready`, { redirect: 'error', signal: AbortSignal.timeout(8_000) }), - post(`${origin}/v1/admin/runtime-status`, { v: 1 }) - ]) - if (!health.ok || !ready.ok) throw new Error(`${cellId} is not ready`) - if ( - runtime.cellId !== cellId || - runtime.cellUrl !== origin || - runtime.region !== cellRegion(cellId) || - runtime.imageDigest !== imageDigest || - runtime.draining !== false || - runtime.connectionCapacity?.hardCap !== cellShape(cellId).connectionHardCap || - runtime.connectionCapacity?.unobservedBound !== 60 - ) throw new Error(`${cellId} runtime does not match its reviewed shape`) - if (requireDirector) { - const result = await post(`${shape.directorOrigin}/v1/admin/cell-status`, { v: 1, cellId }) - if ( - result.status?.cellUrl !== origin || - result.status?.runtime?.heartbeatFresh !== true || - result.status?.runtime?.ready !== true - ) throw new Error(`${cellId} has no fresh ready director heartbeat`) - } -} - -function membershipStates(selector, cells) { - return Object.fromEntries(cells.map((cellId) => [cellId, selectorCellState(selector, cellId)])) -} - -function inspectedMembershipStates(selector, cells) { - const known = new Set([ - ...selector.membership.existingOnly, - ...selector.membership.migrationOnly, - ...selector.membership.general - ]) - return Object.fromEntries(cells.map((cellId) => [ - cellId, - known.has(cellId) ? selectorCellState(selector, cellId) : 'absent' - ])) -} - -function sameMembership(left, right) { - return JSON.stringify(left) === JSON.stringify(right) -} - -function membershipSha256(membership) { - return createHash('sha256').update(JSON.stringify(membership)).digest('hex') -} - -async function initializeAdmissionBoundary(post, selectorPost, shape, config, current) { - if (config.expectedGeneration !== 0) { - throw new Error('admission boundary initialization requires generation 0') - } - if ( - !/^[a-f0-9]{64}$/.test(config.expectedMembershipSha256 ?? '') || - membershipSha256(current.selector.membership) !== config.expectedMembershipSha256 - ) { - throw new Error('admission membership changed before boundary initialization') - } - const targetStates = inspectedMembershipStates(current.selector, config.cells) - if (Object.values(targetStates).some((state) => state !== 'absent')) { - throw new Error('Asia cell exists before admission boundary initialization') - } - const intendedMembership = current.intent?.previousMembership ?? current.selector.membership - const exactCommitted = (inspection) => - inspection.intent?.state === 'committed' && - inspection.intent.expectedGeneration === 0 && - inspection.selector.generation === 1 && - inspection.selector.attemptId === config.attemptId && - sameMembership(inspection.intent.previousMembership, intendedMembership) && - sameMembership(inspection.intent.membership, intendedMembership) && - sameMembership(inspection.selector.membership, intendedMembership) - const exactUnchanged = (inspection) => - inspection.intent?.state === 'unchanged' && - inspection.intent.expectedGeneration === 0 && - inspection.selector.generation === 0 && - sameMembership(inspection.intent.previousMembership, intendedMembership) && - sameMembership(inspection.intent.membership, intendedMembership) && - sameMembership(inspection.selector.membership, intendedMembership) - if (exactCommitted(current)) { - return { - mode: config.mode, - generation: current.selector.generation, - states: inspectedMembershipStates(current.selector, config.cells), - recovered: true - } - } - if (current.intent && !exactUnchanged(current)) { - throw new Error('admission boundary initialization attempt diverged') - } - const request = { - v: 1, - attemptId: config.attemptId, - expectedGeneration: 0, - expectedMembershipSha256: config.expectedMembershipSha256, - membership: intendedMembership - } - let applyError - for (let attempt = 0; attempt < 2; attempt++) { - try { - await post(`${shape.directorOrigin}/v1/admin/admission-selector/apply`, request) - } catch (error) { - applyError = error - } - const verified = await inspectAdmissionSelector(selectorPost, config.attemptId) - if (exactCommitted(verified)) { - return { - mode: config.mode, - generation: verified.selector.generation, - states: targetStates, - recovered: current.intent !== null || applyError !== undefined || attempt > 0 - } - } - if (!exactUnchanged(verified)) { - throw new Error('admission boundary initialization did not commit exactly', { - cause: applyError - }) - } - } - throw new Error('admission boundary initialization remained unchanged after retry', { - cause: applyError - }) -} - -export async function operateRelayAsiaAdmission(config, dependencies = {}) { - const shape = SHAPES[config.environment] - const fetchImpl = dependencies.fetch ?? fetch - const post = dependencies.post ?? defaultPost(fetchImpl, config.token) - // Only the C4 proof keeps the server-enforced C4-only route (the Asia proof identity can call - // nothing else); other staging cells go through the general route, as production does. - const stagingProof = config.environment === 'staging' && - config.cells.length === 1 && config.cells[0] === STAGING_PROOF_CELL - const selectorPost = (path, body) => { - if (!stagingProof || path !== '/v1/admin/admission-selector/apply') { - return post(`${shape.directorOrigin}${path}`, body) - } - const state = selectorCellState({ membership: body.membership }, STAGING_PROOF_CELL) - if (!['general', 'migration-only'].includes(state)) { - throw new Error('staging proof can only transition C4 between reviewed states') - } - return post(`${shape.directorOrigin}/v1/admin/admission-selector/apply-staging-asia-proof`, { - v: 1, - attemptId: body.attemptId, - expectedGeneration: body.expectedGeneration, - state - }) - } - const current = await inspectAdmissionSelector( - selectorPost, - ['inspect', 'verify', 'registered'].includes(config.mode) ? undefined : config.attemptId - ) - if (config.mode === 'inspect') { - return { - mode: config.mode, - generation: current.selector.generation, - membership: current.selector.membership, - membershipSha256: membershipSha256(current.selector.membership), - states: inspectedMembershipStates(current.selector, config.cells) - } - } - if ( - !current.intent && - current.selector.generation !== config.expectedGeneration - ) { - throw new Error('admission selector generation changed') - } - if (current.intent && current.intent.expectedGeneration !== config.expectedGeneration) { - throw new Error('admission attempt generation does not match') - } - if (config.mode === 'initialize') { - return await initializeAdmissionBoundary(post, selectorPost, shape, config, current) - } - if (config.mode === 'recover-promotion') { - if (config.cells.every( - (cellId) => selectorCellState(current.selector, cellId) === 'migration-only' - )) { - return { - mode: config.mode, - promoted: false, - generation: current.selector.generation, - states: membershipStates(current.selector, config.cells) - } - } - if (!current.intent) { - if ( - current.selector.generation !== config.expectedGeneration - ) throw new Error('promotion state changed without the reviewed attempt') - return { - mode: config.mode, - promoted: false, - generation: current.selector.generation, - states: membershipStates(current.selector, config.cells) - } - } - const expectedMembership = membershipWithStates( - { membership: current.intent.previousMembership }, - Object.fromEntries(config.cells.map((cellId) => [cellId, 'general'])) - ) - if ( - current.intent.state !== 'committed' || - JSON.stringify(current.intent.membership) !== JSON.stringify(expectedMembership) || - config.cells.some((cellId) => selectorCellState(current.selector, cellId) !== 'general') - ) throw new Error('promotion attempt is not the current general state') - return { - mode: config.mode, - promoted: true, - generation: current.selector.generation, - states: membershipStates(current.selector, config.cells) - } - } - if (config.mode === 'rollback') { - for (const cellId of config.cells) { - if (!['general', 'migration-only'].includes(selectorCellState(current.selector, cellId))) { - throw new Error(`${cellId} cannot roll back to migration-only`) - } - } - } else if (config.mode === 'register') { - const known = new Set([ - ...current.selector.membership.existingOnly, - ...current.selector.membership.migrationOnly, - ...current.selector.membership.general - ]) - if (!current.intent && config.cells.some((cellId) => known.has(cellId))) { - throw new Error('Asia cell is already registered') - } - await Promise.all(config.cells.map((cellId) => - verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, false) - )) - } else if (config.mode !== 'registered') { - await Promise.all(config.cells.map((cellId) => - verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, true) - )) - } - if (config.mode === 'registered') { - if (config.cells.some( - (cellId) => selectorCellState(current.selector, cellId) !== 'migration-only' - )) throw new Error('Asia cells are not registered migration-only') - await Promise.all(config.cells.map((cellId) => - verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, false) - )) - } - if (['verify', 'registered'].includes(config.mode)) { - return { - mode: config.mode, - generation: current.selector.generation, - states: membershipStates(current.selector, config.cells) - } - } - if (config.mode === 'register') { - if (current.intent) { - const expectedCells = new Set(config.cells) - const addedCells = current.intent.membership.migrationOnly.filter( - (cellId) => !current.intent.previousMembership.migrationOnly.includes(cellId) - ) - if ( - current.intent.state !== 'committed' || - addedCells.length !== expectedCells.size || - addedCells.some((cellId) => !expectedCells.has(cellId)) || - current.selector.generation !== config.expectedGeneration + 1 || - JSON.stringify(current.selector.membership) !== JSON.stringify(current.intent.membership) - ) { - throw new Error('admission attempt does not match the requested Asia registration') - } - return { - mode: config.mode, - generation: current.selector.generation, - states: membershipStates(current.selector, config.cells), - recovered: true - } - } - const result = await addExactMigrationCells( - selectorPost, - { - attemptId: config.attemptId, - cells: config.cells.map((cellId) => ({ - cellId, - cellUrl: cellOrigin(shape, cellId), - region: cellRegion(cellId), - capacityRequests: cellShape(cellId).capacityRequests, - connectionHardCap: cellShape(cellId).connectionHardCap, - connectionUnobservedBound: 60 - })) - }, - { expectedCurrentSelector: current.selector } - ) - return { mode: config.mode, generation: result.selector.generation, states: membershipStates(result.selector, config.cells) } - } - const desiredState = config.mode === 'promote' ? 'general' : 'migration-only' - if (current.intent) { - const expectedMembership = membershipWithStates( - { membership: current.intent.previousMembership }, - Object.fromEntries(config.cells.map((cellId) => [cellId, desiredState])) - ) - if ( - current.intent.state !== 'committed' || - JSON.stringify(current.intent.membership) !== JSON.stringify(expectedMembership) || - current.selector.generation !== config.expectedGeneration + 1 || - JSON.stringify(current.selector.membership) !== JSON.stringify(current.intent.membership) - ) { - throw new Error('admission attempt does not match the requested Asia transition') - } - return { - mode: config.mode, - generation: current.selector.generation, - states: membershipStates(current.selector, config.cells), - recovered: true - } - } - if (config.mode === 'promote' && config.cells.some( - (cellId) => selectorCellState(current.selector, cellId) !== 'migration-only' - )) throw new Error('Asia promotion requires migration-only cells') - // The Asia launch order binds Asia cells only; a US cell is proven by its own canary. - const asiaWave = config.cells.every((cellId) => cellRegion(cellId) === ASIA_REGION) - if ( - config.mode === 'promote' && - asiaWave && - config.environment === 'production' && - !config.cells.includes(PRODUCTION_CANARY_CELL) && - selectorCellState(current.selector, PRODUCTION_CANARY_CELL) !== 'general' - ) { - throw new Error('Asia expansion requires the C27 canary to be general') - } - const [launchWave] = shape.registrationWaves - if ( - config.mode === 'promote' && - asiaWave && - !config.cells.some((cellId) => launchWave.includes(cellId)) && - launchWave.some((cellId) => selectorCellState(current.selector, cellId) !== 'general') - ) { - throw new Error('a later Asia cell requires every launch cell to be general') - } - const result = await applyExactAdmissionSelector( - selectorPost, - membershipWithStates(current.selector, Object.fromEntries( - config.cells.map((cellId) => [cellId, desiredState]) - )), - { attemptId: config.attemptId, expectedCurrentSelector: current.selector } - ) - return { mode: config.mode, generation: result.selector.generation, states: membershipStates(result.selector, config.cells) } -} - -if (process.argv[1] === fileURLToPath(import.meta.url)) { - const config = parseRelayAsiaAdmissionArguments(process.argv.slice(2)) - if (!config.token) throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is required') - console.log(JSON.stringify(await operateRelayAsiaAdmission(config))) -} diff --git a/cloud/dev/scripts/operate-relay-asia-admission.test.mjs b/cloud/dev/scripts/operate-relay-asia-admission.test.mjs deleted file mode 100644 index 547c190f80c..00000000000 --- a/cloud/dev/scripts/operate-relay-asia-admission.test.mjs +++ /dev/null @@ -1,967 +0,0 @@ -import assert from 'node:assert/strict' -import { spawnSync } from 'node:child_process' -import { createHash } from 'node:crypto' -import { mkdtempSync, readFileSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { test } from 'node:test' -import { - operateRelayAsiaAdmission, - parseRelayAsiaAdmissionArguments -} from './operate-relay-asia-admission.mjs' -import { readRelayWorkflow } from './relay-repository.mjs' - -const digest = `sha256:${'a'.repeat(64)}` -const membershipDigest = (membership) => - createHash('sha256').update(JSON.stringify(membership)).digest('hex') - -function harness(initialSelector, runtimeDigests = {}, runtimeRegions = {}, runtimeHardCaps = {}) { - const initialMembership = structuredClone(initialSelector.membership) - let selector = structuredClone(initialSelector) - const intents = new Map() - const requests = [] - let fetches = 0 - let failAfterIntent = false - const post = async (url, body) => { - const parsed = new URL(url) - requests.push({ path: parsed.pathname, body }) - if (parsed.pathname === '/v1/admin/runtime-status') { - const [cell, domain] = parsed.hostname.split(/\.(.*)/) - const cellId = `${domain.startsWith('relay-staging.') ? 'staging' : 'production'}-gce-${cell}` - return { - cellId, - cellUrl: parsed.origin, - region: runtimeRegions[cellId] ?? 'asia-east2', - imageDigest: runtimeDigests[cellId] ?? digest, - draining: false, - connectionCapacity: { hardCap: runtimeHardCaps[cellId] ?? 3_000, unobservedBound: 60 } - } - } - if (parsed.pathname === '/v1/admin/cell-status') { - return { - status: { - cellUrl: `https://${body.cellId.split('-').at(-1)}.${parsed.hostname}`, - runtime: { heartbeatFresh: true, ready: true } - } - } - } - if (parsed.pathname.endsWith('/status')) { - return { selector, intent: body.attemptId ? intents.get(body.attemptId) ?? null : null } - } - if (parsed.pathname.endsWith('/add-migration-cells')) { - selector = { - generation: selector.generation + 1, - attemptId: body.attemptId, - membership: { - ...selector.membership, - migrationOnly: [...selector.membership.migrationOnly, ...body.cells.map((cell) => cell.cellId)].sort() - } - } - } else if (parsed.pathname.endsWith('/apply-staging-asia-proof')) { - const membership = structuredClone(selector.membership) - membership.migrationOnly = membership.migrationOnly.filter((cell) => cell !== 'staging-gce-c4') - membership.general = membership.general.filter((cell) => cell !== 'staging-gce-c4') - membership[body.state === 'general' ? 'general' : 'migrationOnly'].push('staging-gce-c4') - selector = { generation: selector.generation + 1, attemptId: body.attemptId, membership } - } else if (parsed.pathname.endsWith('/apply')) { - if ( - body.expectedMembershipSha256 && - body.expectedMembershipSha256 !== membershipDigest(selector.membership) - ) throw new Error('admission_selector_membership_mismatch') - if (failAfterIntent) { - failAfterIntent = false - intents.set(body.attemptId, { - state: 'unchanged', - expectedGeneration: body.expectedGeneration, - previousMembership: structuredClone(selector.membership), - membership: structuredClone(body.membership) - }) - throw new Error('failure after intent persistence') - } - selector = { generation: selector.generation + 1, attemptId: body.attemptId, membership: body.membership } - } else throw new Error(`unexpected ${parsed.pathname}`) - intents.set(body.attemptId, { - state: 'committed', expectedGeneration: body.expectedGeneration, - previousMembership: initialSelector.membership, - membership: selector.membership - }) - return { changed: true, selector } - } - const fetch = async () => { - fetches++ - return new Response(null, { status: 200 }) - } - const commitWithoutResponse = async (path, body) => { - await post(`https://relay.onorca.dev${path}`, body) - throw new Error('response lost after commit') - } - return { - post, fetch, requests, commitWithoutResponse, - failNextApplyAfterIntent: () => (failAfterIntent = true), - apply: async (attemptId, membership) => await post( - 'https://relay.onorca.dev/v1/admin/admission-selector/apply', - { attemptId, expectedGeneration: selector.generation, membership } - ), - fetchCount: () => fetches, selector: () => selector - } -} - -const baseSelector = { - generation: 7, - membership: { existingOnly: [], migrationOnly: [], general: ['production-gce-c26'] } -} - -test('inspects generation zero without requiring target registration or making a mutation', async () => { - const subject = harness({ - generation: 0, - membership: { - existingOnly: ['staging-gce-c3'], - migrationOnly: [], - general: ['staging-gce-c1', 'staging-gce-c2'] - } - }) - const result = await operateRelayAsiaAdmission({ - environment: 'staging', mode: 'inspect', cells: ['staging-gce-c4'], - imageDigest: digest, token: 'not-logged' - }, subject) - assert.equal(result.generation, 0) - assert.equal(result.states['staging-gce-c4'], 'absent') - assert.deepEqual(result.membership, subject.selector().membership) - assert.equal(result.membershipSha256, membershipDigest(subject.selector().membership)) - assert.deepEqual(subject.requests.map(({ path }) => path), [ - '/v1/admin/admission-selector/status' - ]) -}) - -test('initializes generation zero without changing membership', async () => { - const membership = { - existingOnly: ['staging-gce-c3'], - migrationOnly: [], - general: ['staging-gce-c1', 'staging-gce-c2'] - } - const subject = harness({ generation: 0, membership }) - const result = await operateRelayAsiaAdmission({ - environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'], - expectedGeneration: 0, imageDigest: digest, - expectedMembershipSha256: membershipDigest(membership), - attemptId: 'asia_boundary_0', token: 'not-logged' - }, subject) - const request = subject.requests.find(({ path }) => path.endsWith('/apply')) - assert.deepEqual(request.body, { - v: 1, - attemptId: 'asia_boundary_0', - expectedGeneration: 0, - expectedMembershipSha256: membershipDigest(membership), - membership - }) - assert.equal(result.generation, 1) - assert.equal(result.states['staging-gce-c4'], 'absent') - assert.deepEqual(subject.selector().membership, membership) -}) - -test('retries the same fingerprint-bound initialization after intent persistence', async () => { - const membership = { - existingOnly: ['staging-gce-c3'], - migrationOnly: [], - general: ['staging-gce-c1', 'staging-gce-c2'] - } - const subject = harness({ generation: 0, membership }) - subject.failNextApplyAfterIntent() - const result = await operateRelayAsiaAdmission({ - environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'], - expectedGeneration: 0, imageDigest: digest, - expectedMembershipSha256: membershipDigest(membership), - attemptId: 'asia_boundary_intent_retry', token: 'not-logged' - }, subject) - const applies = subject.requests.filter(({ path }) => path.endsWith('/apply')) - assert.equal(applies.length, 2) - assert.deepEqual(applies[1].body, applies[0].body) - assert.equal(result.recovered, true) - assert.equal(result.generation, 1) - assert.deepEqual(subject.selector().membership, membership) -}) - -test('recovers a committed generation-zero initialization', async () => { - const membership = { - existingOnly: ['staging-gce-c3'], - migrationOnly: [], - general: ['staging-gce-c1', 'staging-gce-c2'] - } - const subject = harness({ generation: 0, membership }) - const config = { - environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'], - expectedGeneration: 0, imageDigest: digest, - expectedMembershipSha256: membershipDigest(membership), - attemptId: 'asia_boundary_retry', token: 'not-logged' - } - await assert.rejects(subject.commitWithoutResponse( - '/v1/admin/admission-selector/apply', - { - v: 1, - attemptId: config.attemptId, - expectedGeneration: 0, - expectedMembershipSha256: membershipDigest(membership), - membership - } - ), /response lost after commit/) - const recovered = await operateRelayAsiaAdmission(config, subject) - assert.equal(recovered.recovered, true) - assert.equal(recovered.generation, 1) - assert.deepEqual(subject.selector().membership, membership) -}) - -test('rejects generation-zero membership drift after inspect', async () => { - const inspected = { - existingOnly: ['staging-gce-c3'], - migrationOnly: [], - general: ['staging-gce-c1', 'staging-gce-c2'] - } - const changed = { - existingOnly: ['staging-gce-c2', 'staging-gce-c3'], - migrationOnly: [], - general: ['staging-gce-c1'] - } - const subject = harness({ generation: 0, membership: changed }) - await assert.rejects(operateRelayAsiaAdmission({ - environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'], - expectedGeneration: 0, imageDigest: digest, - expectedMembershipSha256: membershipDigest(inspected), - attemptId: 'asia_boundary_drift', token: 'not-logged' - }, subject), /membership changed/) - assert.equal(subject.requests.some(({ path }) => path.endsWith('/apply')), false) -}) - -test('registers all three Asia cells atomically with region and exact limits', async () => { - const subject = harness(baseSelector) - const result = await operateRelayAsiaAdmission({ - environment: 'production', mode: 'register', - cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - expectedGeneration: 7, imageDigest: digest, attemptId: 'asia_register_7', token: 'not-logged' - }, subject) - const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells')) - assert.equal(request.body.cells.length, 3) - assert.ok(request.body.cells.every((cell) => - cell.region === 'asia-east2' && cell.capacityRequests === 6_000 && - cell.connectionHardCap === 3_000 && cell.connectionUnobservedBound === 60 - )) - assert.equal(result.generation, 8) - assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only'])) -}) - -test('promotes the canary only after runtime and director-heartbeat checks', async () => { - const subject = harness({ - generation: 8, - membership: { existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] } - }) - const result = await operateRelayAsiaAdmission({ - environment: 'production', mode: 'promote', cells: ['production-gce-c27'], - expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_promote_8', token: 'not-logged' - }, subject) - assert.equal(subject.fetchCount(), 2) - assert.equal(result.states['production-gce-c27'], 'general') -}) - -test('checks registered migration-only cells before director configuration without requiring heartbeat', async () => { - const subject = harness({ - generation: 8, - membership: { - existingOnly: [], - migrationOnly: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - general: ['production-gce-c26'] - } - }) - const result = await operateRelayAsiaAdmission({ - environment: 'production', mode: 'registered', - cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - expectedGeneration: 8, imageDigest: digest, token: 'not-logged' - }, subject) - assert.equal(subject.fetchCount(), 6) - assert.equal(subject.requests.filter(({ path }) => path === '/v1/admin/cell-status').length, 0) - assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only'])) -}) - -test('rolls back admission without requiring an unhealthy runtime to answer', async () => { - const subject = harness({ - generation: 9, - membership: { existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] } - }) - const result = await operateRelayAsiaAdmission({ - environment: 'production', mode: 'rollback', cells: ['production-gce-c27'], - expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_rollback_9', token: 'not-logged' - }, subject) - assert.equal(subject.fetchCount(), 0) - assert.equal(result.states['production-gce-c27'], 'migration-only') -}) - -test('uses the server-enforced C4-only route for staging proof transitions', async () => { - const subject = harness({ - generation: 3, - membership: { - existingOnly: ['staging-gce-c1'], - migrationOnly: [], - general: ['staging-gce-c2', 'staging-gce-c4'] - } - }) - const result = await operateRelayAsiaAdmission({ - environment: 'staging', mode: 'rollback', cells: ['staging-gce-c4'], - expectedGeneration: 3, imageDigest: digest, attemptId: 'asia_staging_rollback', - token: 'not-logged' - }, subject) - const request = subject.requests.find( - ({ path }) => path.endsWith('/apply-staging-asia-proof') - ) - assert.deepEqual(request.body, { - v: 1, - attemptId: 'asia_staging_rollback', - expectedGeneration: 3, - state: 'migration-only' - }) - assert.deepEqual(subject.selector().membership, { - existingOnly: ['staging-gce-c1'], - migrationOnly: ['staging-gce-c4'], - general: ['staging-gce-c2'] - }) - assert.equal(result.states['staging-gce-c4'], 'migration-only') -}) - -test('fails closed when the exact selector generation moved', async () => { - const subject = harness(baseSelector) - await assert.rejects(operateRelayAsiaAdmission({ - environment: 'production', mode: 'register', - cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - expectedGeneration: 6, imageDigest: digest, attemptId: 'asia_register_6', token: 'not-logged' - }, subject), /generation changed/) - assert.equal(subject.fetchCount(), 0) -}) - -test('recovers a committed registration when the workflow retries the original generation', async () => { - const subject = harness(baseSelector) - const config = { - environment: 'production', mode: 'register', - cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - expectedGeneration: 7, imageDigest: digest, attemptId: 'asia_register_retry', - token: 'not-logged' - } - await assert.rejects(subject.commitWithoutResponse( - '/v1/admin/admission-selector/add-migration-cells', - { - v: 1, - attemptId: config.attemptId, - expectedGeneration: config.expectedGeneration, - cells: config.cells.map((cellId) => ({ cellId })) - } - ), /response lost after commit/) - const recovered = await operateRelayAsiaAdmission(config, subject) - assert.equal(recovered.recovered, true) - assert.equal(recovered.generation, 8) -}) - -test('recovers a committed promotion when the workflow retries the original generation', async () => { - const subject = harness({ - generation: 8, - membership: { - existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] - } - }) - const config = { - environment: 'production', mode: 'promote', cells: ['production-gce-c27'], - expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_promote_retry', - token: 'not-logged' - } - await assert.rejects(subject.commitWithoutResponse( - '/v1/admin/admission-selector/apply', - { - v: 1, - attemptId: config.attemptId, - expectedGeneration: config.expectedGeneration, - membership: { - existingOnly: [], migrationOnly: [], - general: ['production-gce-c26', 'production-gce-c27'] - } - } - ), /response lost after commit/) - const recovered = await operateRelayAsiaAdmission(config, subject) - assert.equal(recovered.recovered, true) - assert.equal(recovered.generation, 9) - assert.equal(recovered.states['production-gce-c27'], 'general') -}) - -test('inspects an ambiguous promotion without creating a new transition', async () => { - const untouched = harness({ - generation: 8, - membership: { - existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] - } - }) - const config = { - environment: 'production', mode: 'recover-promotion', cells: ['production-gce-c27'], - expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_recover_promote', - token: 'not-logged' - } - const absent = await operateRelayAsiaAdmission(config, untouched) - assert.equal(absent.promoted, false) - assert.equal(untouched.requests.some(({ path }) => path.endsWith('/apply')), false) - - const committed = harness({ - generation: 8, - membership: { - existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] - } - }) - await assert.rejects(committed.commitWithoutResponse('/v1/admin/admission-selector/apply', { - v: 1, - attemptId: config.attemptId, - expectedGeneration: 8, - membership: { - existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] - } - }), /response lost after commit/) - const recovered = await operateRelayAsiaAdmission(config, committed) - assert.equal(recovered.promoted, true) - assert.equal(recovered.generation, 9) -}) - -test('treats an already rolled-back promotion as recovered', async () => { - const subject = harness({ - generation: 8, - membership: { - existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] - } - }) - const config = { - environment: 'production', mode: 'recover-promotion', cells: ['production-gce-c27'], - expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_recover_after_rollback', - token: 'not-logged' - } - await assert.rejects(subject.commitWithoutResponse('/v1/admin/admission-selector/apply', { - v: 1, attemptId: config.attemptId, expectedGeneration: 8, - membership: { - existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] - } - }), /response lost after commit/) - await subject.apply('later_rollback', { - existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] - }) - const recovered = await operateRelayAsiaAdmission(config, subject) - assert.equal(recovered.promoted, false) - assert.equal(recovered.generation, 10) -}) - -test('recovers a committed rollback when the workflow retries the original generation', async () => { - const subject = harness({ - generation: 9, - membership: { - existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] - } - }) - const config = { - environment: 'production', mode: 'rollback', cells: ['production-gce-c27'], - expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_rollback_retry', - token: 'not-logged' - } - await assert.rejects(subject.commitWithoutResponse( - '/v1/admin/admission-selector/apply', - { - v: 1, - attemptId: config.attemptId, - expectedGeneration: config.expectedGeneration, - membership: { - existingOnly: [], migrationOnly: ['production-gce-c27'], - general: ['production-gce-c26'] - } - } - ), /response lost after commit/) - const recovered = await operateRelayAsiaAdmission(config, subject) - assert.equal(recovered.recovered, true) - assert.equal(recovered.generation, 10) - assert.equal(recovered.states['production-gce-c27'], 'migration-only') -}) - -test('rejects a committed transition retry after a later selector change', async () => { - const subject = harness({ - generation: 8, - membership: { - existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] - } - }) - const config = { - environment: 'production', mode: 'promote', cells: ['production-gce-c27'], - expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_stale_promote', - token: 'not-logged' - } - await assert.rejects(subject.commitWithoutResponse('/v1/admin/admission-selector/apply', { - v: 1, attemptId: config.attemptId, expectedGeneration: 8, - membership: { - existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] - } - }), /response lost after commit/) - await subject.apply('later_rollback', { - existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] - }) - await assert.rejects( - operateRelayAsiaAdmission(config, subject), - /does not match the requested Asia transition/ - ) -}) - -test('requires the C27 canary before promoting C28 and C29', async () => { - const subject = harness({ - generation: 8, - membership: { - existingOnly: [], - migrationOnly: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - general: ['production-gce-c26'] - } - }) - await assert.rejects(operateRelayAsiaAdmission({ - environment: 'production', mode: 'promote', - cells: ['production-gce-c28', 'production-gce-c29'], expectedGeneration: 8, - imageDigest: digest, attemptId: 'asia_wave_before_canary', token: 'not-logged' - }, subject), /C27 canary/) -}) - -const launchCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'] - -function admissionArguments(environment, mode, cellIds) { - return [ - '--environment', environment, '--mode', mode, '--cell-ids', cellIds, - '--image-digest', digest, '--expected-generation', '9', '--attempt-id', 'asia_wave_9' - ] -} - -test('accepts only reviewed Asia admission waves', () => { - const accepted = [ - ['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29'], - ['inspect', 'production-gce-c30'], - ['inspect', 'production-gce-c31'], - ['verify', 'production-gce-c27,production-gce-c28,production-gce-c29'], - ['verify', 'production-gce-c30'], - ['verify', 'production-gce-c31'], - ['initialize', 'production-gce-c27,production-gce-c28,production-gce-c29'], - ['register', 'production-gce-c27,production-gce-c28,production-gce-c29'], - ['register', 'production-gce-c30'], - ['registered', 'production-gce-c30'], - ['register', 'production-gce-c31'], - ['registered', 'production-gce-c31'], - ['promote', 'production-gce-c27'], - ['promote', 'production-gce-c28,production-gce-c29'], - ['promote', 'production-gce-c30'], - ['recover-promotion', 'production-gce-c30'], - ['promote', 'production-gce-c31'], - ['recover-promotion', 'production-gce-c31'], - ['rollback', 'production-gce-c27'], - ['rollback', 'production-gce-c28,production-gce-c29'], - ['rollback', 'production-gce-c30'], - ['rollback', 'production-gce-c31'], - ['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29'], - ['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34,production-gce-c35'], - ...['production-gce-c32', 'production-gce-c33'].flatMap((cellId) => [ - 'inspect', 'verify', 'register', 'registered', 'promote', 'recover-promotion', 'rollback' - ].map((mode) => [mode, cellId])), - ...['inspect', 'verify', 'register', 'registered', 'promote', 'recover-promotion', 'rollback'] - .map((mode) => [mode, 'production-gce-c34']), - ...['inspect', 'verify', 'register', 'registered', 'promote', 'recover-promotion', 'rollback'] - .map((mode) => [mode, 'production-gce-c35']), - ['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34,production-gce-c35'] - ] - for (const [mode, cellIds] of accepted) { - assert.deepEqual( - parseRelayAsiaAdmissionArguments(admissionArguments('production', mode, cellIds)).cells, - cellIds.split(','), - `${mode} ${cellIds}` - ) - } - const rejected = [ - ['promote', 'production-gce-c34,production-gce-c35'], - ['promote', 'production-gce-c31,production-gce-c35'], - ['register', 'production-gce-c34,production-gce-c35'], - ['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34'], - ['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34'], - ['initialize', 'production-gce-c30'], - ['initialize', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'], - ['register', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'], - ['register', 'production-gce-c30,production-gce-c31'], - ['register', 'production-gce-c29,production-gce-c30'], - ['registered', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'], - ['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'], - ['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'], - ['initialize', 'production-gce-c32'], - ['register', 'production-gce-c32,production-gce-c33'], - ['register', 'production-gce-c31,production-gce-c32'], - ['promote', 'production-gce-c32,production-gce-c33'], - ['rollback', 'production-gce-c32,production-gce-c33'], - ['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'], - ['verify', 'production-gce-c30,production-gce-c31'], - ['verify', 'production-gce-c27,production-gce-c30'], - ['promote', 'production-gce-c27,production-gce-c30'], - ['promote', 'production-gce-c28,production-gce-c29,production-gce-c30'], - ['promote', 'production-gce-c30,production-gce-c31'], - ['promote', 'production-gce-c33,production-gce-c34'], - ['promote', 'production-gce-c31,production-gce-c34'], - ['register', 'production-gce-c33,production-gce-c34'], - ['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33'], - ['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33'], - ['rollback', 'production-gce-c27,production-gce-c30'], - ['rollback', 'production-gce-c30,production-gce-c31'], - ['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'], - ['rollback', 'production-gce-c28,production-gce-c29,production-gce-c30'], - ['rollback', 'production-gce-c29'], - ['register', 'staging-gce-c4'] - ] - for (const [mode, cellIds] of rejected) { - assert.throws( - () => parseRelayAsiaAdmissionArguments(admissionArguments('production', mode, cellIds)), - /--cell-ids/, - `${mode} ${cellIds}` - ) - } - assert.deepEqual( - parseRelayAsiaAdmissionArguments(admissionArguments('staging', 'promote', 'staging-gce-c4')).cells, - ['staging-gce-c4'] - ) - assert.throws( - () => parseRelayAsiaAdmissionArguments(admissionArguments('staging', 'promote', 'production-gce-c30')), - /--cell-ids are invalid/ - ) -}) - -test('registers C30 alone beside the general launch cells', async () => { - const subject = harness({ - generation: 9, - membership: { existingOnly: [], migrationOnly: [], general: [...launchCells] } - }) - const result = await operateRelayAsiaAdmission({ - environment: 'production', mode: 'register', cells: ['production-gce-c30'], - expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_register_c30', token: 'not-logged' - }, subject) - const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells')) - assert.deepEqual(request.body.cells, [{ - cellId: 'production-gce-c30', cellUrl: 'https://c30.relay.onorca.dev', region: 'asia-east2', - capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60 - }]) - assert.deepEqual(result.states, { 'production-gce-c30': 'migration-only' }) - assert.deepEqual(subject.selector().membership.general, launchCells) -}) - -test('registers C31 alone beside the general C27-C30', async () => { - const general = [...launchCells, 'production-gce-c30'] - const subject = harness({ - generation: 11, - membership: { existingOnly: [], migrationOnly: [], general: [...general] } - }) - const result = await operateRelayAsiaAdmission({ - environment: 'production', mode: 'register', cells: ['production-gce-c31'], - expectedGeneration: 11, imageDigest: digest, attemptId: 'asia_register_c31', token: 'not-logged' - }, subject) - const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells')) - assert.deepEqual(request.body.cells, [{ - cellId: 'production-gce-c31', cellUrl: 'https://c31.relay.onorca.dev', region: 'asia-east2', - capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60 - }]) - assert.deepEqual(result.states, { 'production-gce-c31': 'migration-only' }) - assert.deepEqual(subject.selector().membership.general, general) -}) - -test('registers the C34 spare alone as migration-only in asia-east2', async () => { - const general = [...launchCells, 'production-gce-c30', 'production-gce-c31'] - const subject = harness({ - generation: 17, - membership: { - existingOnly: [], - migrationOnly: [], - general: [...general, 'production-gce-c32', 'production-gce-c33'] - } - }) - const result = await operateRelayAsiaAdmission({ - environment: 'production', mode: 'register', cells: ['production-gce-c34'], - expectedGeneration: 17, imageDigest: digest, attemptId: 'asia_register_c34', token: 'not-logged' - }, subject) - const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells')) - assert.deepEqual(request.body.cells, [{ - cellId: 'production-gce-c34', cellUrl: 'https://c34.relay.onorca.dev', region: 'asia-east2', - capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60 - }]) - assert.deepEqual(result.states, { 'production-gce-c34': 'migration-only' }) - assert.equal(subject.selector().membership.general.includes('production-gce-c34'), false) -}) - -test('registers the C35 spare alone as migration-only in asia-east2', async () => { - const general = [...launchCells, 'production-gce-c30', 'production-gce-c31'] - const subject = harness({ - generation: 17, - membership: { - existingOnly: [], - migrationOnly: [], - general: [...general, 'production-gce-c32', 'production-gce-c33', 'production-gce-c34'] - } - }) - const result = await operateRelayAsiaAdmission({ - environment: 'production', mode: 'register', cells: ['production-gce-c35'], - expectedGeneration: 17, imageDigest: digest, attemptId: 'asia_register_c35', token: 'not-logged' - }, subject) - const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells')) - assert.deepEqual(request.body.cells, [{ - cellId: 'production-gce-c35', cellUrl: 'https://c35.relay.onorca.dev', region: 'asia-east2', - capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60 - }]) - assert.deepEqual(result.states, { 'production-gce-c35': 'migration-only' }) - assert.equal(subject.selector().membership.general.includes('production-gce-c35'), false) -}) - -test('promotes the registered C34 spare to general and leaves every other cell alone', async () => { - const general = [ - ...launchCells, 'production-gce-c30', 'production-gce-c31', 'production-gce-c32', 'production-gce-c33' - ] - const membership = { - existingOnly: [], - migrationOnly: ['production-gce-c17', 'production-gce-c34'], - general: [...general] - } - const subject = harness({ generation: 345, membership }) - const result = await operateRelayAsiaAdmission({ - environment: 'production', mode: 'promote', cells: ['production-gce-c34'], - expectedGeneration: 345, imageDigest: digest, attemptId: 'asia_promote_c34', token: 'not-logged' - }, subject) - assert.deepEqual(result.states, { 'production-gce-c34': 'general' }) - assert.deepEqual(subject.selector().membership.migrationOnly, ['production-gce-c17']) - assert.deepEqual(subject.selector().membership.general, [...general, 'production-gce-c34'].sort()) -}) - -test('promotes the registered C35 spare to general and leaves every other cell alone', async () => { - const general = [ - ...launchCells, 'production-gce-c30', 'production-gce-c31', 'production-gce-c32', 'production-gce-c33', - 'production-gce-c34' - ] - const membership = { - existingOnly: [], - migrationOnly: ['production-gce-c17', 'production-gce-c35'], - general: [...general] - } - const subject = harness({ generation: 345, membership }) - const result = await operateRelayAsiaAdmission({ - environment: 'production', mode: 'promote', cells: ['production-gce-c35'], - expectedGeneration: 345, imageDigest: digest, attemptId: 'asia_promote_c35', token: 'not-logged' - }, subject) - assert.deepEqual(result.states, { 'production-gce-c35': 'general' }) - assert.deepEqual(subject.selector().membership.migrationOnly, ['production-gce-c17']) - assert.deepEqual(subject.selector().membership.general, [...general, 'production-gce-c35'].sort()) -}) - -const usRegions = { 'production-gce-c32': 'us-central1', 'production-gce-c33': 'us-central1' } - -test('registers C32 and C33 one at a time in us-central1 at the Asia shape', async () => { - const general = [...launchCells, 'production-gce-c30', 'production-gce-c31'] - for (const [cellId, generation] of [['production-gce-c32', 13], ['production-gce-c33', 15]]) { - const hostname = cellId.split('-').at(-1) - const subject = harness({ - generation, membership: { existingOnly: [], migrationOnly: [], general: [...general] } - }, {}, usRegions) - const result = await operateRelayAsiaAdmission({ - environment: 'production', mode: 'register', cells: [cellId], - expectedGeneration: generation, imageDigest: digest, attemptId: `us_register_${hostname}`, - token: 'not-logged' - }, subject) - const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells')) - assert.deepEqual(request.body.cells, [{ - cellId, cellUrl: `https://${hostname}.relay.onorca.dev`, region: 'us-central1', - capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60 - }]) - assert.deepEqual(result.states, { [cellId]: 'migration-only' }) - // A US cell whose runtime reports Asia is the wrong cell, not a US one. - await assert.rejects(operateRelayAsiaAdmission({ - environment: 'production', mode: 'register', cells: [cellId], - expectedGeneration: generation, imageDigest: digest, attemptId: `us_register_${hostname}`, - token: 'not-logged' - }, harness({ - generation, membership: { existingOnly: [], migrationOnly: [], general: [...general] } - })), new RegExp(`${cellId} runtime does not match`)) - } -}) - -test('promotes a US cell without the Asia launch order, which still binds Asia cells', async () => { - const selector = (cellId) => ({ - generation: 14, - membership: { - existingOnly: [], - migrationOnly: [cellId, 'production-gce-c27', 'production-gce-c28'].sort(), - general: ['production-gce-c29'] - } - }) - const config = (cellId) => ({ - environment: 'production', mode: 'promote', cells: [cellId], - expectedGeneration: 14, imageDigest: digest, attemptId: 'us_promote_wave', token: 'not-logged' - }) - const result = await operateRelayAsiaAdmission( - config('production-gce-c32'), harness(selector('production-gce-c32'), {}, usRegions) - ) - assert.deepEqual(result.states, { 'production-gce-c32': 'general' }) - await assert.rejects( - operateRelayAsiaAdmission(config('production-gce-c31'), harness(selector('production-gce-c31'))), - /C27 canary/ - ) -}) - -test('requires the C27 canary to be general before promoting C30', async () => { - const selector = (general) => ({ - generation: 10, - membership: { - existingOnly: [], - migrationOnly: ['production-gce-c30', ...launchCells.filter((cell) => !general.includes(cell))].sort(), - general - } - }) - const config = { - environment: 'production', mode: 'promote', cells: ['production-gce-c30'], - expectedGeneration: 10, imageDigest: digest, attemptId: 'asia_promote_c30', token: 'not-logged' - } - await assert.rejects( - operateRelayAsiaAdmission(config, harness(selector(['production-gce-c28', 'production-gce-c29']))), - /C27 canary/ - ) - await assert.rejects( - operateRelayAsiaAdmission(config, harness(selector(['production-gce-c27', 'production-gce-c29']))), - /every launch cell to be general/ - ) - const subject = harness(selector([...launchCells])) - const result = await operateRelayAsiaAdmission(config, subject) - assert.deepEqual(result.states, { 'production-gce-c30': 'general' }) - assert.equal(subject.requests.filter(({ path }) => path === '/v1/admin/cell-status').length, 1) -}) - -test('promotes C30 on its own digest while the launch cells serve another', async () => { - const c30Digest = `sha256:${'b'.repeat(64)}` - const selector = { - generation: 10, - membership: { existingOnly: [], migrationOnly: ['production-gce-c30'], general: [...launchCells] } - } - const config = { - environment: 'production', mode: 'promote', cells: ['production-gce-c30'], - expectedGeneration: 10, imageDigest: c30Digest, attemptId: 'asia_promote_c30', token: 'not-logged' - } - const digests = { 'production-gce-c30': c30Digest } - const result = await operateRelayAsiaAdmission(config, harness(selector, digests)) - assert.deepEqual(result.states, { 'production-gce-c30': 'general' }) - await assert.rejects( - operateRelayAsiaAdmission({ ...config, imageDigest: digest }, harness(selector, digests)), - /production-gce-c30 runtime does not match/ - ) -}) - -const admissionWorkflow = readRelayWorkflow('operate-relay-asia-admission.yml') - -function workflowBlock(first, last) { - const start = admissionWorkflow.indexOf(first) - const end = admissionWorkflow.indexOf(last, start) - assert.ok(start !== -1 && end !== -1, first) - return admissionWorkflow.slice(start, end + last.length).replace(/^ {10}/gm, '') -} - -// Runs the workflow's own promotion input block, so each wave's canary routing is what ships. -function promotionOutputs(cellIds) { - const temp = mkdtempSync(join(tmpdir(), 'relay-admission-inputs-')) - try { - const output = join(temp, 'output') - const result = spawnSync('bash', ['-c', workflowBlock( - 'set -euo pipefail\n test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"', - '} >> "${GITHUB_OUTPUT}"' - )], { - env: { - ...process.env, GITHUB_OUTPUT: output, DEPLOY_WORKLOAD_IDENTITY_PROVIDER: 'provider', - DEPLOY_SERVICE_ACCOUNT: 'account', IMAGE_DIGEST: digest, OPERATION_MODE: 'promote', - EXPECTED_SELECTOR_GENERATION: '9', SELECTOR_ATTEMPT_ID: 'promote_wave_9', - OPERATION_CONFIRMATION: 'PROMOTE_ASIA_GENERAL', TARGET_ENVIRONMENT: 'production', - TARGET_CELL_IDS: cellIds, EXPECTED_SELECTOR_MEMBERSHIP_SHA256: '', DIRECTOR_IMAGE_DIGEST: '', - EVIDENCE_RUN_ID: '', EVIDENCE_RUN_ATTEMPT: '' - }, - encoding: 'utf8' - }) - if (result.status !== 0) return null - return Object.fromEntries(readFileSync(output, 'utf8').trim().split('\n') - .map((line) => line.split('='))) - } finally { - rmSync(temp, { recursive: true, force: true }) - } -} - -test('runs each later cell\'s own canary with load aimed at that cell\'s region', () => { - for (const [cellId, region] of [ - ['production-gce-c30', 'asia-east2'], ['production-gce-c31', 'asia-east2'], - ['production-gce-c32', 'us-central1'], ['production-gce-c33', 'us-central1'], - ['production-gce-c34', 'asia-east2'], ['production-gce-c35', 'asia-east2'] - ]) { - const outputs = promotionOutputs(cellId) - assert.equal(outputs?.canary, 'true', cellId) - assert.equal(outputs.canary_cell, cellId) - assert.equal(outputs.canary_region, region, cellId) - assert.equal(outputs.evidence_kind, 'none') - } - assert.equal(promotionOutputs('production-gce-c32,production-gce-c33'), null) - assert.equal(promotionOutputs('production-gce-c33,production-gce-c34'), null) - assert.equal(promotionOutputs('production-gce-c34,production-gce-c35'), null) - const canaryStart = workflowBlock('case "${CANARY_CELL}" in', '\n esac') - for (const cellId of ['production-gce-c32', 'production-gce-c33', 'production-gce-c34', 'production-gce-c35']) { - const result = spawnSync('bash', ['-euo', 'pipefail', '-c', - `${canaryStart}\necho "\${verify_cells} \${expected_states}"`], { - env: { ...process.env, CANARY_CELL: cellId }, encoding: 'utf8' - }) - assert.equal(result.stdout.trim(), `${cellId} {"${cellId}":"general"}`) - } - assert.match(admissionWorkflow, /--preferred-region "\$\{CANARY_REGION\}"/) - assert.match(admissionWorkflow, /CANARY_REGION: \$\{\{ steps\.inputs\.outputs\.canary_region \}\}/) -}) - -const stagingDeadCellWave = ['staging-gce-c5', 'staging-gce-c6', 'staging-gce-c7'] -const stagingUsShape = Object.fromEntries(stagingDeadCellWave.map((cellId) => [cellId, 'us-central1'])) -const stagingUsCaps = Object.fromEntries(stagingDeadCellWave.map((cellId) => [cellId, 600])) - -test('registers the staging dead-cell wave at its own US shape', async () => { - const subject = harness({ - generation: 43, - membership: { existingOnly: ['staging-gce-c3'], migrationOnly: [], general: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c4'] } - }, {}, stagingUsShape, stagingUsCaps) - const result = await operateRelayAsiaAdmission({ - environment: 'staging', mode: 'register', cells: stagingDeadCellWave, - expectedGeneration: 43, imageDigest: digest, attemptId: 'staging_c5_c7_register', token: 'not-logged' - }, subject) - const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells')) - assert.deepEqual(request.body.cells.map((cell) => cell.cellUrl), [ - 'https://c5.relay-staging.onorca.dev', 'https://c6.relay-staging.onorca.dev', 'https://c7.relay-staging.onorca.dev' - ]) - assert.ok(request.body.cells.every((cell) => - cell.region === 'us-central1' && cell.capacityRequests === 4_000 && - cell.connectionHardCap === 600 && cell.connectionUnobservedBound === 60 - )) - assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only'])) -}) - -test('promotes the staging dead-cell wave through the general route, not the C4 proof route', async () => { - const subject = harness({ - generation: 44, - membership: { - existingOnly: ['staging-gce-c3'], - migrationOnly: [...stagingDeadCellWave], - general: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c4'] - } - }, {}, stagingUsShape, stagingUsCaps) - const result = await operateRelayAsiaAdmission({ - environment: 'staging', mode: 'promote', cells: stagingDeadCellWave, - expectedGeneration: 44, imageDigest: digest, attemptId: 'staging_c5_c7_promote', token: 'not-logged' - }, subject) - assert.ok(subject.requests.some(({ path }) => path === '/v1/admin/admission-selector/apply')) - assert.ok(!subject.requests.some(({ path }) => path.endsWith('/apply-staging-asia-proof'))) - assert.deepEqual(new Set(Object.values(result.states)), new Set(['general'])) - assert.deepEqual(subject.selector().membership.general, [ - 'staging-gce-c1', 'staging-gce-c2', 'staging-gce-c4', ...stagingDeadCellWave - ]) -}) - -test('refuses a staging wave that was not reviewed', () => { - assert.throws(() => parseRelayAsiaAdmissionArguments([ - '--environment', 'staging', '--mode', 'promote', '--cell-ids', 'staging-gce-c4,staging-gce-c5', - '--expected-generation', '44', '--attempt-id', 'staging_mixed_wave', '--image-digest', digest - ]), /reviewed admission wave/) - assert.throws(() => parseRelayAsiaAdmissionArguments([ - '--environment', 'staging', '--mode', 'register', '--cell-ids', 'staging-gce-c8', - '--expected-generation', '44', '--attempt-id', 'staging_unknown', '--image-digest', digest - ]), /cell-ids are invalid/) -}) diff --git a/cloud/dev/scripts/operate-relay-cell-flags.mjs b/cloud/dev/scripts/operate-relay-cell-flags.mjs deleted file mode 100644 index 5456cc59fca..00000000000 --- a/cloud/dev/scripts/operate-relay-cell-flags.mjs +++ /dev/null @@ -1,619 +0,0 @@ -#!/usr/bin/env node -// Writes one cell's runtime switch object (cells/.json in -relay-control) and -// reads back what the cell applied. One cell per run by construction: the object is per cell. -// -// dry-run: checks the cell and the current generation, prints the change, writes nothing. -// write: the same checks, then a compare-and-swap write (ifGenerationMatch), then waits for -// the cell's runtime status to report the new generation applied. -import { parseArgs } from 'node:util' -import { pathToFileURL } from 'node:url' -import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' - -export const CELL_ID_PATTERN = /^(production|staging)-gce-c[0-9]{1,3}$/ -export const WRITE_CONFIRMATION = 'WRITE_RELAY_CELL_FLAGS' -// The cell polls every 5 s (+20% jitter); 15 s leaves two polls of slack. -export const READ_BACK_TIMEOUT_MS = 15_000 -const READ_BACK_INTERVAL_MS = 1_000 -const STORAGE_ROOT = 'https://storage.googleapis.com' - -// Every switch a cell image may know, with what an absent key means on the cell. A cell -// resets a key missing from its object to this default, so a write must carry every key it -// does not mean to change: the tool reads the current object and changes only what is named. -const enumOf = (...options) => (value) => (options.includes(value) ? value : undefined) -const booleanValue = (value) => (value === 'true' ? true : value === 'false' ? false : undefined) -const numberIn = (min, max, integer) => (value) => { - const number = Number(value) - if (value === '' || !Number.isFinite(number) || number < min || number > max) return undefined - if (integer && !Number.isInteger(number)) return undefined - return number -} -export const CELL_FLAG_SPECS = { - readinessLocal: { parse: booleanValue, default: false }, - ticketCheck: { parse: enumOf('off', 'shadow', 'enforce'), default: 'off' }, - admitMode: { parse: enumOf('db', 'reserve'), default: 'db' }, - // Absent means the cell's region default. - intakePerSec: { - parse: (value) => { - const rate = numberIn(0, 1_000, false)(value) - return rate === 0 ? undefined : rate - } - }, - reserveDryRun: { parse: booleanValue, default: false }, - rejectionFence: { parse: booleanValue, default: true }, - // Absent keeps the cell's boot value. - readTimeoutMarginMs: { parse: numberIn(1_000, 60_000, true) }, - // Flip-back leases in flight; absent is a third of the cell's pool. - reregisterInFlight: { parse: numberIn(1, 16, true) }, - // Step 6: acts only while the cell is reserve. - phonePath: { parse: enumOf('db', 'memory'), default: 'db' } -} - -export const RESERVE_CONFIRMATION = 'RESERVE' -export const BREAK_GLASS_CONFIRMATION = 'BREAK GLASS' -// Longer than a director's cached read of the admit-mode table (5 s). -export const RESERVE_MODE_CACHE_WAIT_MS = 10_000 -// How long a flip back may take to lease every control the cell admitted from memory: its -// control count at a per-lane pace below the measured 0.38/s (170 ms trips), within the job. -export const REREGISTER_WAIT_MIN_MS = 8 * 60_000 -export const REREGISTER_WAIT_MAX_MS = 50 * 60_000 -const REREGISTER_CONTROLS_PER_LANE_PER_SEC = 0.3 -// The cell takes a third of its database pool, and caps the switch at the pool less two -// (host-session-registry.ts reregisterInFlightLimit). An image from before runtime-status -// reported the pool falls back to the deployed sizes (US 10, Asia 16). -const CELL_DATABASE_POOL_MAX = { 'asia-east2': 16 } -const CELL_DATABASE_POOL_MAX_DEFAULT = 10 - -export function reregisterLanes(runtime) { - const reported = runtime?.databasePoolMax - const pool = - Number.isInteger(reported) && reported > 0 - ? reported - : (CELL_DATABASE_POOL_MAX[runtime?.region] ?? CELL_DATABASE_POOL_MAX_DEFAULT) - const flag = runtime?.flagsApplied?.flags?.reregisterInFlight - if (flag === undefined) return Math.max(1, Math.floor(pool / 3)) - return Math.max(1, Math.min(Number(flag), pool - 2)) -} - -export function reregisterWaitMs(runtime) { - const controls = Number(runtime?.runtime?.controls ?? 0) - const lanes = reregisterLanes(runtime) - const estimate = (controls / (lanes * REREGISTER_CONTROLS_PER_LANE_PER_SEC)) * 1_000 - return Math.min(REREGISTER_WAIT_MAX_MS, Math.max(REREGISTER_WAIT_MIN_MS, Math.ceil(estimate))) -} - -// An image from before `supportedFlags` accepts the keys its applied flags list, and only -// ticketCheck off/shadow: anything else is dropped there, or voids the whole object. -function legacySupportedFlags(appliedFlags) { - return Object.fromEntries( - Object.entries(appliedFlags ?? {}).map(([key, value]) => [ - key, - key === 'ticketCheck' - ? { type: 'enum', values: ['off', 'shadow'] } - : { type: typeof value === 'boolean' ? 'boolean' : typeof value === 'number' ? 'number' : 'enum', values: [value] } - ]) - ) -} - -function accepts(spec, value) { - if (spec?.type === 'boolean') return typeof value === 'boolean' - if (spec?.type === 'enum') return spec.values.includes(value) - if (spec?.type === 'number') { - return ( - typeof value === 'number' && - (spec.min === undefined || value >= spec.min) && - (spec.max === undefined || value <= spec.max) && - (!spec.integer || Number.isInteger(value)) - ) - } - return false -} - -// The whole object as it will be written, not only the change: a bad key or value already in -// the file would void or be dropped on every later write too. -export function assertSupportedObject(flags, runtime) { - const supported = runtime.supportedFlags ?? legacySupportedFlags(runtime.flagsApplied?.flags) - for (const [key, value] of Object.entries(flags)) { - if (!accepts(supported[key], value)) { - throw new Error( - `the cell's image does not support ${key}=${JSON.stringify(value)}; roll the image first` - ) - } - } -} - -// A reserve cell's dead-man is fed only by director polls through the load balancer. With SQL -// readiness the balancer drops the cell in a database outage, the polls fail, and the cell trips -// (staging drill 2026-10-11), so reserve requires readinessLocal=true in the object it writes. -export function assertReserveKeepsLocalReadiness(flags, changes) { - if ((flags.admitMode ?? 'db') !== 'reserve' || flags.readinessLocal === true) return - if ('readinessLocal' in changes && !('admitMode' in changes)) { - throw new Error( - 'readinessLocal must stay true while the cell is reserve; flip it back with admitMode=db first, or in the same write' - ) - } - throw new Error( - 'admitMode=reserve requires readinessLocal=true on the cell, already written or in the same write ' + - '(e.g. --set readinessLocal=true,admitMode=reserve): with SQL readiness the load balancer drops the cell ' + - "in a database outage and the directors' polls can no longer feed its dead-man" - ) -} - -// The cell ignores phonePath=memory outside reserve, so writing it there only arms a later -// admitMode=reserve flip to turn phones over unnamed. Rolling back with admitMode=db stays allowed. -export function assertPhonePathNeedsReserve(flags, changes) { - if ((flags.phonePath ?? 'db') !== 'memory') return - if ((flags.admitMode ?? 'db') === 'reserve') { - if (changes.admitMode === 'reserve' && !('phonePath' in changes)) { - throw new Error( - 'the object still says phonePath=memory: name phonePath in the same write ' + - '(e.g. --set admitMode=reserve,phonePath=default), so a reserve flip never turns phones over unnamed' - ) - } - return - } - if ('phonePath' in changes) { - throw new Error("phonePath=memory requires admitMode=reserve on the cell, already written or in the same write") - } -} - -// `key=value,key=value`; `key=default` removes the key so the cell uses its default. -export function parseFlagChanges(text) { - const changes = {} - for (const part of (text ?? '').split(',').map((entry) => entry.trim()).filter(Boolean)) { - const separator = part.indexOf('=') - const key = separator < 0 ? part : part.slice(0, separator) - const raw = separator < 0 ? '' : part.slice(separator + 1).trim() - const spec = CELL_FLAG_SPECS[key] - if (!spec) throw new Error(`unknown switch ${key}`) - if (key in changes) throw new Error(`switch ${key} named twice`) - if (raw === 'default') { - changes[key] = null - continue - } - const value = spec.parse(raw) - if (value === undefined) throw new Error(`switch ${key} cannot be ${JSON.stringify(raw)}`) - // phonePath=db is the default, stored as no key: an image without the switch refuses the key. - changes[key] = key === 'phonePath' && value === 'db' ? null : value - } - if (Object.keys(changes).length === 0) throw new Error('set names no switch to change') - return changes -} - -// The switches as the cell will apply them: every known key, absent ones at their default. -function effectiveFlags(flags) { - return Object.fromEntries( - Object.entries(CELL_FLAG_SPECS).map(([key, spec]) => [key, flags?.[key] ?? spec.default]) - ) -} - -export function parseCellFlagsRequest(values) { - const mode = values.mode - if (mode !== 'dry-run' && mode !== 'write') throw new Error('mode must be dry-run or write') - const cellId = values['cell-id'] ?? '' - if (!CELL_ID_PATTERN.test(cellId)) throw new Error('cell id is not a relay GCE cell id') - const changes = parseFlagChanges(values.set) - // 0 means "no object yet": the write then creates it and refuses if one appeared. - const expectedGeneration = values['expected-generation'] ?? '' - if (!/^(0|[1-9][0-9]{0,18})$/.test(expectedGeneration)) { - throw new Error('expected-generation must be 0 or the exact current object generation') - } - if (mode === 'write' && values.confirmation !== `${WRITE_CONFIRMATION} ${cellId}`) { - throw new Error(`write requires the confirmation "${WRITE_CONFIRMATION} ${cellId}"`) - } - // Reserve moves this cell's hosts off the database: it is typed for this cell, every time. - if ( - mode === 'write' && - changes.admitMode === 'reserve' && - values['confirm-reserve'] !== `${RESERVE_CONFIRMATION} ${cellId}` - ) { - throw new Error(`admitMode=reserve requires --confirm-reserve "${RESERVE_CONFIRMATION} ${cellId}"`) - } - // Break glass: a dead or unreachable reserve cell cannot report db, so nothing would ever - // let the sweeps re-place its hosts. Only admitMode=db, typed for this cell. - const breakGlass = values['break-glass'] ?? '' - if (breakGlass !== '') { - if (breakGlass !== `${BREAK_GLASS_CONFIRMATION} ${cellId}`) { - throw new Error(`break glass requires --break-glass "${BREAK_GLASS_CONFIRMATION} ${cellId}"`) - } - if (mode !== 'write' || changes.admitMode !== null || Object.keys(changes).length !== 1) { - throw new Error('break glass only writes admitMode=default (db), on its own') - } - } - const directorOrigin = values['director-origin'] ?? '' - if (changes.admitMode !== undefined && !/^https:\/\/relay(-staging)?\.onorca\.dev$/.test(directorOrigin)) { - throw new Error('an admitMode change needs --director-origin, where Postgres records it') - } - const projectId = values['project-id'] ?? '' - if (!/^[a-z][a-z0-9-]{4,62}$/.test(projectId)) throw new Error('project id is invalid') - const cellOrigin = values['cell-origin'] ?? '' - if (!/^https:\/\/c[0-9]{1,3}\.relay(-staging)?\.onorca\.dev$/.test(cellOrigin)) { - throw new Error('cell origin is not a relay cell origin') - } - if (new URL(cellOrigin).hostname.split('.')[0] !== cellId.replace(/^[a-z]+-gce-/, '')) { - throw new Error('cell origin names another cell') - } - return { - mode, - cellId, - cellOrigin, - directorOrigin, - breakGlass: breakGlass !== '', - projectId, - expectedGeneration, - changes - } -} - -// Read-modify-write: the current object's switches with only the named ones changed. -export function desiredObject(request, current) { - if (current !== null && (current?.v !== 1 || current?.cellId !== request.cellId)) { - throw new Error('the current object is not this cell\'s v1 switch object; fix it by hand') - } - const flags = { ...(current?.flags ?? {}) } - for (const [key, value] of Object.entries(request.changes)) { - if (value === null) delete flags[key] - else flags[key] = value - } - return { v: 1, cellId: request.cellId, flags } -} - -function objectUrl(request, suffix = '') { - const bucket = `${request.projectId}-relay-control` - return `${STORAGE_ROOT}/storage/v1/b/${bucket}/o/${encodeURIComponent( - `cells/${request.cellId}.json` - )}${suffix}` -} - -// One transient 5xx or network failure is retried once, as every relay admin step does. -export async function readRuntime(fetchImpl, request, idToken, wait) { - const response = await fetchAdminOnceMore( - fetchImpl, - `${request.cellOrigin}/v1/admin/runtime-status`, - { - method: 'POST', - headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' }, - body: JSON.stringify({ v: 1 }) - }, - { wait, timeoutMs: 10_000, retryDelayMs: 1_000 } - ) - if (!response.ok) { - await response.body?.cancel().catch(() => undefined) - throw new Error(`cell runtime status returned ${response.status}`) - } - return await response.json() -} - -export async function readCurrentObject(fetchImpl, request, accessToken) { - const headers = { authorization: `Bearer ${accessToken}` } - const metadata = await fetchImpl(objectUrl(request), { headers }) - if (metadata.status === 404) { - await metadata.body?.cancel().catch(() => undefined) - return { generation: '0', object: null } - } - if (!metadata.ok) throw new Error(`object metadata read returned ${metadata.status}`) - const { generation } = await metadata.json() - const media = await fetchImpl(objectUrl(request, `?alt=media&ifGenerationMatch=${generation}`), { - headers - }) - if (!media.ok) throw new Error(`object read returned ${media.status}`) - return { generation: String(generation), object: await media.json().catch(() => null) } -} - -function generationConflict(current, expected) { - return Object.assign(new Error(`current generation is ${current}, not expected ${expected}`), { - generationConflict: true - }) -} - -async function writeObject(fetchImpl, request, object, accessToken, audit) { - const boundary = `relay-cell-flags-${Date.now()}` - const metadata = { - name: `cells/${request.cellId}.json`, - contentType: 'application/json', - cacheControl: 'no-store', - // Who and which run, beside the generation the bucket's versioning already keeps. - metadata: audit - } - const body = [ - `--${boundary}`, - 'Content-Type: application/json; charset=UTF-8', - '', - JSON.stringify(metadata), - `--${boundary}`, - 'Content-Type: application/json', - '', - JSON.stringify(object), - `--${boundary}--`, - '' - ].join('\r\n') - const bucket = `${request.projectId}-relay-control` - const response = await fetchImpl( - `${STORAGE_ROOT}/upload/storage/v1/b/${bucket}/o?uploadType=multipart&ifGenerationMatch=${request.expectedGeneration}`, - { - method: 'POST', - headers: { - authorization: `Bearer ${accessToken}`, - 'content-type': `multipart/related; boundary=${boundary}` - }, - body - } - ) - // A 4xx means nothing was written; a 5xx or a lost reply may have been. - if (response.status === 412) { - await response.body?.cancel().catch(() => undefined) - throw Object.assign(new Error('the object changed since expected-generation; re-read and retry'), { - notWritten: true, - generationConflict: true - }) - } - if (!response.ok) { - throw Object.assign(new Error(`object write returned ${response.status}`), { - notWritten: response.status < 500 - }) - } - return String((await response.json()).generation) -} - -// The census's record (relay_cell_admit_modes, through a director). Without `admitMode` it only -// reads. Read back on every write: sweeps act on this, not on the cell's switch file. -async function directorAdmitMode(fetchImpl, request, idToken, wait, admitMode) { - const response = await fetchAdminOnceMore( - fetchImpl, - `${request.directorOrigin}/v1/admin/cell-admit-mode`, - { - method: 'POST', - headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' }, - body: JSON.stringify({ v: 1, cellId: request.cellId, ...(admitMode ? { admitMode } : {}) }) - }, - { wait, timeoutMs: 10_000, retryDelayMs: 1_000 } - ) - const body = await response.json().catch(() => null) - if (!response.ok) throw new Error(`director refused admitMode: ${response.status} ${body?.error ?? ''}`) - if (admitMode && body?.admitMode !== admitMode) { - throw new Error(`director recorded admitMode ${body?.admitMode}, not ${admitMode}`) - } - return body -} - -// Every known key, not just the ones this run changed: a cell that kept an old value of any -// switch has not applied this object. A key with no fixed default (the cell reports its own -// value) is compared only when the object names it. -export function sameFlags(applied, desired) { - const expected = effectiveFlags(desired) - const actual = effectiveFlags(applied) - return Object.entries(CELL_FLAG_SPECS).every( - ([key, spec]) => - (!('default' in spec) && desired?.[key] === undefined) || actual[key] === expected[key] - ) -} - -// Sweeps resume only once every control the cell admitted from memory holds a lease again. -async function waitForLeasedControls(fetchImpl, request, idToken, { now, sleep, log }) { - let waitMs - let deadline = now() + REREGISTER_WAIT_MIN_MS - while (now() < deadline) { - const runtime = await readRuntime(fetchImpl, request, idToken, sleep).catch(() => null) - // An image without the field never admitted from memory. - if (runtime && (runtime.admitModeEffective ?? 'db') === 'db') return - if (runtime && waitMs === undefined) { - waitMs = reregisterWaitMs(runtime) - deadline = now() + waitMs - log(JSON.stringify({ event: 'orca_relay_cell_reregistration_wait', cellId: request.cellId, waitMs })) - } - await sleep(5_000) - } - log(JSON.stringify({ event: 'orca_relay_cell_reregistration_still_running', cellId: request.cellId })) - throw new Error( - `cell still re-registering after ${waitMs ?? REREGISTER_WAIT_MIN_MS} ms. Its connections are safe and Postgres ` + - 'keeps reserve, so sweeps stay off the cell. Run again with --set admitMode=db, which records db once ' + - 'runtime-status reports admitModeEffective=db; to speed it up, name both in one write: ' + - '--set admitMode=db,reregisterInFlight=N' - ) -} - -// No cell read-back and no wait for its controls: the cell is not answering. The file says db -// so a cell that comes back boots in db, and Postgres says db so the sweeps re-place its hosts. -async function breakGlassToDatabase(request, dependencies, { fetchImpl, log, sleep }) { - const { accessToken, idToken, audit } = dependencies - // Only for a dead cell: one that answers, or still heartbeats, takes the ordinary flip back. - const answered = await fetchImpl(`${request.cellOrigin}/v1/admin/runtime-status`, { - method: 'POST', - headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' }, - body: JSON.stringify({ v: 1 }), - signal: AbortSignal.timeout(10_000) - }) - .then(async (response) => { - await response.body?.cancel().catch(() => undefined) - return response.ok - }) - .catch(() => false) - if (answered) throw new Error('the cell answers runtime-status: flip it back with --set admitMode=db, not break glass') - const recorded = await directorAdmitMode(fetchImpl, request, idToken, sleep) - if (recorded?.heartbeatFresh !== false) { - throw new Error( - `the director ${recorded?.heartbeatFresh ? 'still sees heartbeats from' : 'cannot say whether it hears'} ` + - 'the cell: break glass is only for a cell that has stopped' - ) - } - const current = await readCurrentObject(fetchImpl, request, accessToken) - if (current.generation !== request.expectedGeneration) { - throw generationConflict(current.generation, request.expectedGeneration) - } - // phonePath goes too: a dead cell comes back with phones on the database path as well. - const object = desiredObject({ ...request, changes: { ...request.changes, phonePath: null } }, current.object) - const generation = await writeObject(fetchImpl, request, object, accessToken, audit) - await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db') - const result = { event: 'orca_relay_cell_admit_mode_break_glass', cellId: request.cellId, generation, audit } - log(JSON.stringify(result)) - return { ...result, written: true } -} - -export async function operateCellFlags(request, dependencies) { - const { fetchImpl = fetch, accessToken, idToken, audit, log = console.log } = dependencies - const now = dependencies.now ?? Date.now - const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) - if (request.breakGlass) return await breakGlassToDatabase(request, dependencies, { fetchImpl, log, sleep }) - const runtime = await readRuntime(fetchImpl, request, idToken, sleep) - // Proves the origin is this cell, and that its image reads switch files at all. - if (runtime.role !== 'cell' || runtime.cellId !== request.cellId) { - throw new Error(`origin answers as ${runtime.role}/${runtime.cellId}, not ${request.cellId}`) - } - if (!runtime.flagsApplied) throw new Error('cell image has no flag channel (no flagsApplied)') - const current = await readCurrentObject(fetchImpl, request, accessToken) - const object = desiredObject(request, current.object) - assertSupportedObject(object.flags, runtime) - // A tripped dead-man: any new generation that keeps reserve re-arms it, so the cell trips - // again and its re-registration restarts. Only a write that also says db is safe. - if ( - runtime.flagsApplied.flags?.admitMode === 'reserve' && - runtime.admitModeEffective === 'db' && - (object.flags.admitMode ?? 'db') === 'reserve' - ) { - throw new Error( - 'the cell tripped its dead-man (reserve configured, db effective); a write that keeps reserve re-arms it. ' + - 'Name admitMode=db in the same write, e.g. --set admitMode=db,reregisterInFlight=N' - ) - } - assertReserveKeepsLocalReadiness(object.flags, request.changes) - assertPhonePathNeedsReserve(object.flags, request.changes) - const plan = { - event: 'orca_relay_cell_flags_plan', - mode: request.mode, - cellId: request.cellId, - currentGeneration: current.generation, - currentObject: current.object, - appliedBefore: runtime.flagsApplied, - changes: request.changes, - desired: object - } - // Before the generation check, so a dry run always shows the generation to pass next. - log(JSON.stringify(plan)) - if (current.generation !== request.expectedGeneration) { - throw generationConflict(current.generation, request.expectedGeneration) - } - // The cell applies every readable write; a generation it did not apply was voided there. A - // write of an object this image accepts whole (checked above) repairs it. - if (current.generation !== '0' && String(runtime.flagsApplied.generation) !== current.generation) { - log( - JSON.stringify({ - event: 'orca_relay_cell_flags_repairing_unapplied_object', - cellId: request.cellId, - appliedGeneration: runtime.flagsApplied.generation, - currentGeneration: current.generation - }) - ) - } - const admitModeChange = - request.changes.admitMode === undefined ? undefined : (request.changes.admitMode ?? 'db') - if (admitModeChange !== undefined) { - const recorded = await directorAdmitMode(fetchImpl, request, idToken, sleep) - log(JSON.stringify({ event: 'orca_relay_cell_admit_mode_recorded_before', ...recorded })) - } - if (request.mode === 'dry-run') return { ...plan, written: false } - // Sweeps skip the cell before it starts admitting from memory, never after. - let generation - if (admitModeChange === 'reserve') { - const before = await directorAdmitMode(fetchImpl, request, idToken, sleep) - await directorAdmitMode(fetchImpl, request, idToken, sleep, 'reserve') - // Every director's cached set expires (5 s) before the cell starts admitting from memory; - // the second write re-runs the open-flow check over anything started meanwhile. - await sleep(RESERVE_MODE_CACHE_WAIT_MS) - try { - await directorAdmitMode(fetchImpl, request, idToken, sleep, 'reserve') - } catch (error) { - // A drain, migration or rehome opened during the wait: Postgres goes back as it was. - if (before?.admitMode === 'db') await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db') - throw error - } - try { - generation = await writeObject(fetchImpl, request, object, accessToken, audit) - } catch (error) { - // The file was refused outright (412, 4xx): put Postgres back as it was. After a 5xx the - // object may be written, so Postgres keeps reserve (sweeps stay off: the safe side). - if (error?.notWritten && before?.admitMode === 'db') { - await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db') - } - throw error - } - } else { - generation = await writeObject(fetchImpl, request, object, accessToken, audit) - } - log(JSON.stringify({ event: 'orca_relay_cell_flags_written', cellId: request.cellId, generation })) - const deadline = now() + READ_BACK_TIMEOUT_MS - let last = runtime.flagsApplied - while (now() < deadline) { - await sleep(READ_BACK_INTERVAL_MS) - try { - last = (await readRuntime(fetchImpl, request, idToken, sleep)).flagsApplied - } catch { - // The object is written; a failed poll only delays the read-back until the deadline. - continue - } - if (String(last?.generation) !== generation) continue - // The image predates a switch this object names: it applied the rest and dropped that one. - if (Array.isArray(last.ignoredKeys) && last.ignoredKeys.length > 0) { - throw new Error( - `cell applied generation ${generation} but ignores ${last.ignoredKeys.join(', ')}; ` + - 'its image predates them, so flip the object back' - ) - } - if (sameFlags(last.flags, object.flags)) { - const result = { ...plan, written: true, generation, applied: last } - log(JSON.stringify({ ...result, event: 'orca_relay_cell_flags_applied_read_back' })) - if (admitModeChange === 'db') { - await waitForLeasedControls(fetchImpl, request, idToken, { now, sleep, log }) - await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db') - log(JSON.stringify({ event: 'orca_relay_cell_admit_mode_recorded', cellId: request.cellId, admitMode: 'db' })) - } - return result - } - } - throw new Error( - `cell did not apply generation ${generation} within ${READ_BACK_TIMEOUT_MS} ms ` + - `(last applied ${JSON.stringify(last)}); the object is written` - ) -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { - const { values } = parseArgs({ - options: Object.fromEntries( - [ - 'mode', - 'cell-id', - 'cell-origin', - 'project-id', - 'set', - 'expected-generation', - 'confirmation', - 'confirm-reserve', - 'break-glass', - 'director-origin' - ].map((name) => [name, { type: 'string' }]) - ) - }) - try { - const request = parseCellFlagsRequest(values) - const accessToken = process.env.ORCA_RELAY_CONTROL_ACCESS_TOKEN - const idToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN - if (!accessToken || !idToken) throw new Error('both tokens are required') - await operateCellFlags(request, { - accessToken, - idToken, - audit: { - // The person who started this attempt, which differs from GITHUB_ACTOR on a re-run. - writtenBy: process.env.GITHUB_TRIGGERING_ACTOR ?? process.env.GITHUB_ACTOR ?? 'unknown', - runId: process.env.GITHUB_RUN_ID ?? 'local', - runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? '0' - } - }) - } catch (error) { - // On stdout, so the job summary's copy of the run carries the reason. - console.log( - JSON.stringify({ - event: 'orca_relay_cell_flags_failed', - message: error instanceof Error ? error.message : String(error) - }) - ) - process.exitCode = 1 - } -} diff --git a/cloud/dev/scripts/operate-relay-cell-flags.test.mjs b/cloud/dev/scripts/operate-relay-cell-flags.test.mjs deleted file mode 100644 index 5d30e7e9483..00000000000 --- a/cloud/dev/scripts/operate-relay-cell-flags.test.mjs +++ /dev/null @@ -1,611 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { - assertSupportedObject, - desiredObject, - operateCellFlags, - parseCellFlagsRequest, - parseFlagChanges, - READ_BACK_TIMEOUT_MS, - REREGISTER_WAIT_MAX_MS, - REREGISTER_WAIT_MIN_MS, - reregisterLanes, - reregisterWaitMs, - WRITE_CONFIRMATION -} from './operate-relay-cell-flags.mjs' - -const CELL = 'production-gce-c26' -const RESERVE = { 'director-origin': 'https://relay.onorca.dev', 'confirm-reserve': `RESERVE ${CELL}` } -const values = (overrides = {}) => ({ - mode: 'write', - 'cell-id': CELL, - 'cell-origin': 'https://c26.relay.onorca.dev', - 'project-id': 'onorca-cloud', - set: 'readinessLocal=true', - 'expected-generation': '0', - confirmation: `${WRITE_CONFIRMATION} ${CELL}`, - ...overrides -}) - -// A cell whose runtime status applies whatever was written, after `applyAfterPolls` polls. -const SUPPORTED = { - readinessLocal: { type: 'boolean' }, - ticketCheck: { type: 'enum', values: ['off', 'shadow', 'enforce'] }, - admitMode: { type: 'enum', values: ['db', 'reserve'] }, - intakePerSec: { type: 'number', min: 0, max: 1_000 }, - reserveDryRun: { type: 'boolean' }, - rejectionFence: { type: 'boolean' }, - readTimeoutMarginMs: { type: 'number', min: 1_000, max: 60_000, integer: true }, - reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true }, - phonePath: { type: 'enum', values: ['db', 'memory'] } -} - -function fakeGoogleAndCell({ - stored = null, - applyAfterPolls = 2, - role = 'cell', - cellId = CELL, - supportedFlags = SUPPORTED, - // Polls that still report admitModeEffective=reserve after a flip back. - reregisteringPolls = 0, - controls, - // Forces admitModeEffective, as a tripped dead-man reports it. - effectiveOverride, - heartbeatFresh = false, - // The director refuses this numbered reserve post (1-based) with this error. - refuseReservePost -} = {}) { - const state = { - stored, - writes: [], - polls: 0, - // A cell that has applied whatever object is already there. - applied: stored - ? { generation: Number(stored.generation), flags: stored.object.flags } - : { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } }, - directorAdmitMode: 'db', - reservePosts: 0, - order: [], - reregisteringPolls - } - const fetchImpl = async (url, init = {}) => { - const target = new URL(url) - if (target.pathname === '/v1/admin/cell-admit-mode') { - const body = JSON.parse(String(init.body)) - if (body.admitMode === 'reserve') { - state.reservePosts += 1 - if (refuseReservePost?.post === state.reservePosts) { - return Response.json({ error: refuseReservePost.error }, { status: 409 }) - } - } - if (body.admitMode) { - state.directorAdmitMode = body.admitMode - state.order.push(`director:${body.admitMode}`) - } - return Response.json({ v: 1, cellId: body.cellId, admitMode: state.directorAdmitMode, updatedAt: 1, heartbeatFresh }) - } - if (target.pathname === '/v1/admin/runtime-status') { - state.polls += 1 - if (state.stored && state.polls > applyAfterPolls) { - state.applied = { generation: Number(state.stored.generation), flags: state.stored.object.flags } - } - const reserveApplied = state.applied.flags.admitMode === 'reserve' - const effective = reserveApplied || (state.writes.length > 0 && state.reregisteringPolls-- > 0) ? 'reserve' : 'db' - return Response.json({ - v: 1, - role, - cellId, - flagsApplied: state.applied, - admitModeEffective: effectiveOverride ?? effective, - ...(controls === undefined ? {} : { runtime: { controls } }), - ...(supportedFlags ? { supportedFlags } : {}) - }) - } - if (target.pathname.startsWith('/upload/')) { - const expected = target.searchParams.get('ifGenerationMatch') - if (expected !== (state.stored?.generation ?? '0')) return new Response(null, { status: 412 }) - const parts = String(init.body).split('\r\n') - const object = JSON.parse(parts[parts.indexOf('Content-Type: application/json') + 2]) - const metadata = JSON.parse(parts[3]) - const generation = String(Number(state.stored?.generation ?? 1000) + 1) - state.stored = { generation, object } - state.writes.push({ generation, object, metadata }) - state.order.push(`file:${object.flags.admitMode ?? 'db'}`) - return Response.json({ generation }) - } - if (!state.stored) return new Response(null, { status: 404 }) - if (target.searchParams.get('alt') === 'media') return Response.json(state.stored.object) - return Response.json({ generation: state.stored.generation }) - } - return { state, fetchImpl } -} - -const run = (request, fake, extra = {}) => { - let clock = 0 - const lines = [] - return { - lines, - result: operateCellFlags(request, { - fetchImpl: fake.fetchImpl, - accessToken: 'access', - idToken: 'id', - audit: { writtenBy: 'octocat', runId: '42', runAttempt: '1' }, - log: (line) => lines.push(JSON.parse(line)), - now: () => clock, - sleep: async (ms) => { - clock += ms - }, - ...extra - }) - } -} - -test('refuses malformed requests before touching anything', () => { - assert.throws(() => parseCellFlagsRequest(values({ mode: 'apply' })), /mode/) - assert.throws(() => parseCellFlagsRequest(values({ 'cell-id': 'all' })), /cell id/) - assert.throws(() => parseCellFlagsRequest(values({ set: 'ticketCheck=always' })), /ticketCheck/) - assert.throws(() => parseCellFlagsRequest(values({ set: 'placerMode=on' })), /unknown switch/) - assert.throws(() => parseCellFlagsRequest(values({ set: '' })), /names no switch/) - assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db,admitMode=reserve' })), /twice/) - assert.throws(() => parseCellFlagsRequest(values({ set: 'readTimeoutMarginMs=500' })), /cannot be/) - assert.throws(() => parseCellFlagsRequest(values({ 'expected-generation': '' })), /expected/) - assert.throws(() => parseCellFlagsRequest(values({ confirmation: WRITE_CONFIRMATION })), /confirmation/) - assert.throws( - () => parseCellFlagsRequest(values({ 'cell-origin': 'https://c27.relay.onorca.dev' })), - /another cell/ - ) - assert.throws(() => parseCellFlagsRequest(values({ 'cell-origin': 'https://evil.example' })), /origin/) - // A dry run needs no confirmation. - assert.equal(parseCellFlagsRequest(values({ mode: 'dry-run', confirmation: '' })).mode, 'dry-run') -}) - -test('a dry run reports the change and writes nothing', async () => { - const fake = fakeGoogleAndCell() - const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake) - const outcome = await result - assert.equal(outcome.written, false) - assert.equal(fake.state.writes.length, 0) - assert.equal(lines[0].event, 'orca_relay_cell_flags_plan') - assert.deepEqual(lines[0].desired.flags, { readinessLocal: true }) -}) - -test('writes with compare-and-swap, records who wrote it, and waits for the cell to apply it', async () => { - const fake = fakeGoogleAndCell() - const { result, lines } = run(parseCellFlagsRequest(values()), fake) - const outcome = await result - assert.equal(outcome.written, true) - assert.equal(outcome.generation, '1001') - assert.deepEqual(outcome.applied, { generation: 1001, flags: { readinessLocal: true } }) - assert.deepEqual(fake.state.writes[0].metadata.metadata, { writtenBy: 'octocat', runId: '42', runAttempt: '1' }) - assert.deepEqual(fake.state.writes[0].object, { - v: 1, - cellId: CELL, - flags: { readinessLocal: true } - }) - assert.deepEqual(lines.map((line) => line.event), [ - 'orca_relay_cell_flags_plan', - 'orca_relay_cell_flags_written', - 'orca_relay_cell_flags_applied_read_back' - ]) - // Flip back: the next write must name the generation just written. - const back = run( - parseCellFlagsRequest(values({ set: 'readinessLocal=false', 'expected-generation': '1001' })), - fake - ) - assert.equal((await back.result).generation, '1002') -}) - -test('refuses a stale expected generation, and a concurrent writer between read and write', async () => { - const fake = fakeGoogleAndCell({ - stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } } - }) - await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /current generation is 7/) - assert.equal(fake.state.writes.length, 0) - const racing = fakeGoogleAndCell() - const write = racing.fetchImpl - racing.fetchImpl = async (url, init) => { - if (new URL(url).pathname.startsWith('/upload/')) racing.state.stored = { generation: '9', object: {} } - return await write(url, init) - } - await assert.rejects(run(parseCellFlagsRequest(values()), racing).result, /changed since/) -}) - -test('a dry run against a stale generation still shows the plan, then refuses', async () => { - const fake = fakeGoogleAndCell({ - stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } } - }) - const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake) - await assert.rejects(result, /current generation is 7/) - assert.equal(lines[0].event, 'orca_relay_cell_flags_plan') - assert.equal(lines[0].currentGeneration, '7') -}) - -test('rides out a transient 5xx on the first read and during the read-back', async () => { - const fake = fakeGoogleAndCell() - const inner = fake.fetchImpl - let runtimeCalls = 0 - fake.fetchImpl = async (url, init) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') { - runtimeCalls += 1 - // The first call, and the first read-back poll after the write. - if (runtimeCalls === 1 || runtimeCalls === 3) return new Response(null, { status: 503 }) - } - return await inner(url, init) - } - const outcome = await run(parseCellFlagsRequest(values()), fake).result - assert.equal(outcome.written, true) - assert.equal(outcome.generation, '1001') -}) - -test('refuses an origin that is not the named cell, or an image without the flag channel', async () => { - await assert.rejects( - run(parseCellFlagsRequest(values()), fakeGoogleAndCell({ cellId: 'production-gce-c25' })).result, - /not production-gce-c26/ - ) - const old = fakeGoogleAndCell() - const fetchImpl = old.fetchImpl - old.fetchImpl = async (url, init) => - new URL(url).pathname === '/v1/admin/runtime-status' - ? Response.json({ v: 1, role: 'cell', cellId: CELL }) - : await fetchImpl(url, init) - await assert.rejects(run(parseCellFlagsRequest(values()), old).result, /no flag channel/) -}) - -test('fails loudly when the cell does not apply the write within the read-back window', async () => { - const fake = fakeGoogleAndCell({ applyAfterPolls: READ_BACK_TIMEOUT_MS }) - await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /did not apply generation 1001/) - assert.equal(fake.state.writes.length, 1) -}) - -test('changes only the named switches and keeps every other key the object already has', () => { - const current = { - v: 1, - cellId: CELL, - flags: { readinessLocal: true, ticketCheck: 'shadow', readTimeoutMarginMs: 5_000 } - } - const request = parseCellFlagsRequest( - values({ set: 'admitMode=reserve, intakePerSec=40, readTimeoutMarginMs=default', ...RESERVE }) - ) - assert.deepEqual(desiredObject(request, current), { - v: 1, - cellId: CELL, - flags: { readinessLocal: true, ticketCheck: 'shadow', admitMode: 'reserve', intakePerSec: 40 } - }) - assert.deepEqual(desiredObject(request, null).flags, { admitMode: 'reserve', intakePerSec: 40 }) - assert.throws(() => desiredObject(request, { v: 1, cellId: 'production-gce-c27', flags: {} }), /by hand/) - assert.deepEqual(parseFlagChanges('rejectionFence=false'), { rejectionFence: false }) - assert.deepEqual(parseFlagChanges('reregisterInFlight=8'), { reregisterInFlight: 8 }) - assert.throws(() => parseFlagChanges('reregisterInFlight=32'), /cannot be/) -}) - -test('a write keeps the switches it did not name, end to end', async () => { - const fake = fakeGoogleAndCell({ - stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } } - }) - const outcome = await run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result - assert.equal(outcome.written, true) - assert.deepEqual(fake.state.writes[0].object.flags, { ticketCheck: 'shadow', readinessLocal: true }) -}) - -test('the read-back compares every switch, not only the one this run changed', async () => { - const fake = fakeGoogleAndCell({ - stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } } - }) - const inner = fake.fetchImpl - fake.fetchImpl = async (url, init) => { - const response = await inner(url, init) - if (new URL(url).pathname !== '/v1/admin/runtime-status') return response - const body = await response.json() - // The cell reports the new generation but still runs the old ticketCheck. - if (body.flagsApplied.generation === 8) body.flagsApplied.flags = { readinessLocal: true } - return Response.json(body) - } - await assert.rejects( - run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result, - /did not apply generation 8/ - ) -}) - -test('a switch the cell reports on its own is compared only when the object names it', async () => { - const fake = fakeGoogleAndCell() - const inner = fake.fetchImpl - fake.fetchImpl = async (url, init) => { - const response = await inner(url, init) - if (new URL(url).pathname !== '/v1/admin/runtime-status') return response - const body = await response.json() - body.flagsApplied.flags = { ...body.flagsApplied.flags, intakePerSec: 50, readTimeoutMarginMs: 10_000 } - return Response.json(body) - } - assert.equal((await run(parseCellFlagsRequest(values()), fake).result).written, true) -}) - -test('fails the read-back when the cell ignores a switch the object names', async () => { - const fake = fakeGoogleAndCell() - const inner = fake.fetchImpl - fake.fetchImpl = async (url, init) => { - const response = await inner(url, init) - if (new URL(url).pathname !== '/v1/admin/runtime-status') return response - const body = await response.json() - if (body.flagsApplied.generation === 1001) body.flagsApplied.ignoredKeys = ['admitMode'] - return Response.json(body) - } - await assert.rejects( - run(parseCellFlagsRequest(values({ set: 'readinessLocal=true,admitMode=reserve', ...RESERVE })), fake).result, - /ignores admitMode/ - ) -}) - -test('refuses a switch the cell image does not support, before writing anything', async () => { - const old = fakeGoogleAndCell({ supportedFlags: null }) - await assert.rejects( - run(parseCellFlagsRequest(values({ set: 'ticketCheck=enforce' })), old).result, - /does not support ticketCheck="enforce"/ - ) - await assert.rejects( - run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), old).result, - /does not support admitMode/ - ) - assert.equal(old.state.writes.length, 0) - // An image from before supportedFlags still takes the two switches it always had. - assert.equal((await run(parseCellFlagsRequest(values()), old).result).written, true) - assert.throws( - () => assertSupportedObject({ readTimeoutMarginMs: 1_500.5 }, { supportedFlags: SUPPORTED }), - /readTimeoutMarginMs/ - ) - assertSupportedObject({ ticketCheck: 'enforce' }, { supportedFlags: SUPPORTED }) -}) - -test('refuses to carry a bad value already in the file', async () => { - const bad = fakeGoogleAndCell({ - supportedFlags: null, - stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'enforce' } } } - }) - // The old image voided generation 7 and still runs generation 0. - bad.state.applied = { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } } - await assert.rejects( - run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), bad).result, - /does not support ticketCheck="enforce"/ - ) - assert.equal(bad.state.writes.length, 0) -}) - -const DIRECTOR = { 'director-origin': RESERVE['director-origin'] } - -test('admitMode=reserve needs a typed confirmation, and Postgres records reserve before the file', async () => { - assert.throws( - () => parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=reserve' })), - /--confirm-reserve "RESERVE production-gce-c26"/ - ) - assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db' })), /--director-origin/) - const fake = fakeGoogleAndCell() - const request = parseCellFlagsRequest( - values({ ...DIRECTOR, set: 'readinessLocal=true,admitMode=reserve', 'confirm-reserve': `RESERVE ${CELL}` }) - ) - assert.equal((await run(request, fake).result).written, true) - // Recorded, re-checked after every director's cache has expired, then the file. - assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'file:reserve']) -}) - -test('a refused file write after Postgres says reserve puts Postgres back to db', async () => { - const fake = fakeGoogleAndCell() - const inner = fake.fetchImpl - fake.fetchImpl = async (url, init) => { - if (new URL(url).pathname.startsWith('/upload/')) return new Response(null, { status: 403 }) - return await inner(url, init) - } - const request = parseCellFlagsRequest(values({ set: 'readinessLocal=true,admitMode=reserve', ...RESERVE })) - await assert.rejects(run(request, fake).result, /returned 403/) - assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'director:db']) -}) - -test('repairs an object the cell did not apply when the whole new object is valid', async () => { - const voided = fakeGoogleAndCell({ - stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } } - }) - voided.state.applied = { generation: 3, flags: { readinessLocal: false } } - const { result, lines } = run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), voided) - assert.equal((await result).written, true) - assert.ok(lines.some((line) => line.event === 'orca_relay_cell_flags_repairing_unapplied_object')) -}) - -test('a flip back records db in Postgres only after the cell has leased every control', async () => { - const fake = fakeGoogleAndCell({ - stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }, - reregisteringPolls: 3 - }) - fake.state.directorAdmitMode = 'reserve' - const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' })) - assert.equal((await run(request, fake).result).written, true) - assert.deepEqual(fake.state.order, ['file:db', 'director:db']) - assert.equal(fake.state.reregisteringPolls, -1) -}) - -test('the flip-back wait scales with the control count and the switch-file pace', () => { - assert.equal(reregisterWaitMs({ runtime: { controls: 100 } }), REREGISTER_WAIT_MIN_MS) - // 1,500 controls at the default 3 lanes: 1,667 s. - assert.equal(reregisterWaitMs({ runtime: { controls: 1_500 } }), 1_666_667) - assert.equal( - reregisterWaitMs({ runtime: { controls: 1_500 }, flagsApplied: { flags: { reregisterInFlight: 10 } } }), - // The cell caps 10 at its US pool of 10 less two. - 625_000 - ) - assert.equal(reregisterWaitMs({ runtime: { controls: 3_000 } }), REREGISTER_WAIT_MAX_MS) -}) - -test('the wait counts the lanes the cell actually uses', () => { - assert.equal(reregisterLanes({ region: 'us-central1' }), 3) - assert.equal(reregisterLanes({ region: 'asia-east2' }), 5) - assert.equal(reregisterLanes({ region: 'us-central1', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 8) - assert.equal(reregisterLanes({ region: 'asia-east2', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 14) - // The pool the cell reports wins over the deployed-size table. - assert.equal(reregisterLanes({ region: 'us-central1', databasePoolMax: 22 }), 7) - assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 12, flagsApplied: { flags: { reregisterInFlight: 16 } } }), 10) - assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 'x' }), 5) -}) - -test('a tripped cell takes no write that keeps reserve, only one that also says db', async () => { - const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } } - const tripped = () => fakeGoogleAndCell({ stored, effectiveOverride: 'db' }) - for (const set of ['readinessLocal=true', 'reregisterInFlight=4']) { - const fake = tripped() - const request = parseCellFlagsRequest(values({ set, 'expected-generation': '7' })) - await assert.rejects(run(request, fake).result, /tripped its dead-man[\s\S]*admitMode=db,reregisterInFlight=N/) - assert.deepEqual(fake.state.writes, []) - } - const fake = tripped() - fake.state.directorAdmitMode = 'reserve' - const request = parseCellFlagsRequest( - values({ ...DIRECTOR, set: 'admitMode=db,reregisterInFlight=4', 'expected-generation': '7' }) - ) - assert.equal((await run(request, fake).result).written, true) - assert.deepEqual(fake.state.writes[0].object.flags, { admitMode: 'db', reregisterInFlight: 4 }) -}) - -test('reserve requires readinessLocal=true, already written or in the same write', async () => { - const refused = fakeGoogleAndCell() - await assert.rejects( - run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), refused).result, - /admitMode=reserve requires readinessLocal=true/ - ) - assert.deepEqual(refused.state.order, []) - assert.deepEqual(refused.state.writes, []) - - // Already written by an earlier run: reserve alone is enough. - const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } } - const ready = fakeGoogleAndCell({ stored }) - const request = parseCellFlagsRequest( - values({ set: 'admitMode=reserve', 'expected-generation': '7', ...RESERVE }) - ) - assert.equal((await run(request, ready).result).written, true) - assert.deepEqual(ready.state.writes[0].object.flags, { readinessLocal: true, admitMode: 'reserve' }) -}) - -test('a reserve cell refuses readinessLocal=false, but takes it together with the flip back', async () => { - const stored = { - generation: '7', - object: { v: 1, cellId: CELL, flags: { readinessLocal: true, admitMode: 'reserve' } } - } - for (const set of ['readinessLocal=false', 'readinessLocal=default']) { - const fake = fakeGoogleAndCell({ stored }) - const request = parseCellFlagsRequest(values({ set, 'expected-generation': '7' })) - await assert.rejects(run(request, fake).result, /readinessLocal must stay true while the cell is reserve/) - assert.deepEqual(fake.state.writes, []) - } - const fake = fakeGoogleAndCell({ stored }) - fake.state.directorAdmitMode = 'reserve' - const request = parseCellFlagsRequest( - values({ ...DIRECTOR, set: 'admitMode=db,readinessLocal=false', 'expected-generation': '7' }) - ) - assert.equal((await run(request, fake).result).written, true) - assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: false, admitMode: 'db' }) -}) - -test('phonePath=memory needs reserve, and a reserve flip never turns phones over unnamed', async () => { - const onDatabase = { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } } - const refused = fakeGoogleAndCell({ stored: onDatabase }) - await assert.rejects( - run(parseCellFlagsRequest(values({ set: 'phonePath=memory', 'expected-generation': '7' })), refused).result, - /phonePath=memory requires admitMode=reserve/ - ) - assert.deepEqual(refused.state.writes, []) - - const together = fakeGoogleAndCell({ stored: onDatabase }) - const both = parseCellFlagsRequest( - values({ set: 'admitMode=reserve,phonePath=memory', 'expected-generation': '7', ...RESERVE }) - ) - assert.equal((await run(both, together).result).written, true) - assert.deepEqual(together.state.writes[0].object.flags, { - readinessLocal: true, - admitMode: 'reserve', - phonePath: 'memory' - }) - - const reserve = { readinessLocal: true, admitMode: 'reserve' } - const onReserve = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: reserve } } }) - const alone = parseCellFlagsRequest(values({ set: 'phonePath=memory', 'expected-generation': '7' })) - assert.equal((await run(alone, onReserve).result).written, true) - - // Rolling back with admitMode=db leaves phonePath=memory in the object, which the cell ignores. - const memory = { ...reserve, phonePath: 'memory' } - const rollback = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: memory } } }) - rollback.state.directorAdmitMode = 'reserve' - const toDatabase = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=db', 'expected-generation': '7' })) - assert.equal((await run(toDatabase, rollback).result).written, true) - assert.deepEqual(rollback.state.writes[0].object.flags, { ...memory, admitMode: 'db' }) - - // phonePath=db removes the key, so an image without the switch still takes later writes. - const off = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: memory } } }) - assert.equal((await run(parseCellFlagsRequest(values({ set: 'phonePath=db', 'expected-generation': '7' })), off).result).written, true) - assert.deepEqual(off.state.writes[0].object.flags, reserve) - - const lingering = { readinessLocal: true, admitMode: 'db', phonePath: 'memory' } - const unnamed = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: lingering } } }) - await assert.rejects( - run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', 'expected-generation': '7', ...RESERVE })), unnamed) - .result, - /still says phonePath=memory/ - ) - assert.deepEqual(unnamed.state.writes, []) -}) - -test('a migration opened during the cache wait puts Postgres back to db', async () => { - const fake = fakeGoogleAndCell({ refuseReservePost: { post: 2, error: 'cell_has_open_migration' } }) - const request = parseCellFlagsRequest(values({ ...RESERVE, set: 'readinessLocal=true,admitMode=reserve' })) - await assert.rejects(run(request, fake).result, /cell_has_open_migration/) - assert.deepEqual(fake.state.order, ['director:reserve', 'director:db']) - assert.deepEqual(fake.state.writes, []) -}) - -test('break glass refuses a cell that answers or that the director still hears', async () => { - const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7', 'break-glass': `BREAK GLASS ${CELL}` } - const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } } - const answering = fakeGoogleAndCell({ stored }) - await assert.rejects(run(parseCellFlagsRequest(values(glass)), answering).result, /answers runtime-status/) - const heard = fakeGoogleAndCell({ stored, heartbeatFresh: true }) - const inner = heard.fetchImpl - heard.fetchImpl = async (url, init) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused') - return await inner(url, init) - } - await assert.rejects(run(parseCellFlagsRequest(values(glass)), heard).result, /still sees heartbeats/) - assert.deepEqual(heard.state.writes, []) - assert.deepEqual(answering.state.writes, []) -}) - -test('a flip back that outlasts its wait keeps Postgres on reserve and says how to finish', async () => { - const fake = fakeGoogleAndCell({ - stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }, - reregisteringPolls: Number.POSITIVE_INFINITY, - controls: 1_500 - }) - fake.state.directorAdmitMode = 'reserve' - const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' })) - const { lines, result } = run(request, fake) - await assert.rejects(result, /after 1666667 ms[\s\S]*Run again with --set admitMode=db[\s\S]*--set admitMode=db,reregisterInFlight=N/) - assert.deepEqual(fake.state.order, ['file:db']) - assert.equal(lines.find((line) => line.event === 'orca_relay_cell_reregistration_wait')?.waitMs, 1_666_667) -}) - -test('break glass records db (and drops phonePath) for a cell that cannot answer, typed for that cell only', async () => { - const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7' } - assert.throws(() => parseCellFlagsRequest(values({ ...glass, 'break-glass': 'BREAK GLASS production-gce-c27' })), /break glass requires/) - assert.throws( - () => parseCellFlagsRequest(values({ ...glass, set: 'admitMode=default,readinessLocal=true', 'break-glass': `BREAK GLASS ${CELL}` })), - /on its own/ - ) - const fake = fakeGoogleAndCell({ - stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } } } - }) - fake.state.directorAdmitMode = 'reserve' - const inner = fake.fetchImpl - fake.fetchImpl = async (url, init) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused') - return await inner(url, init) - } - const { result, lines } = run(parseCellFlagsRequest(values({ ...glass, 'break-glass': `BREAK GLASS ${CELL}` })), fake) - assert.equal((await result).written, true) - assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: true }) - assert.deepEqual(fake.state.order, ['file:db', 'director:db']) - assert.equal(lines.at(-1).event, 'orca_relay_cell_admit_mode_break_glass') -}) - diff --git a/cloud/dev/scripts/operate-relay-regional-rehome.mjs b/cloud/dev/scripts/operate-relay-regional-rehome.mjs deleted file mode 100644 index 816107ae15f..00000000000 --- a/cloud/dev/scripts/operate-relay-regional-rehome.mjs +++ /dev/null @@ -1,344 +0,0 @@ -import { pathToFileURL } from 'node:url' -import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' -import { inspectAdmissionSelector } from './relay-admission-selector.mjs' - -const DIRECTOR_ORIGIN = 'https://relay.onorca.dev' -const MODES = new Set(['inspect', 'enable', 'pause', 'disable', 'recover-enable']) -// c1-c99 with no leading zero; a fixed ceiling here once refused a newly general cell. -const PRODUCTION_CELL_ID = /^production-gce-c[1-9][0-9]?$/ - -function canonicalCells(value) { - if (value === 'none') return [] - const cells = value.split(',').map((cell) => cell.trim()).filter(Boolean).sort() - if ( - cells.length === 0 || - new Set(cells).size !== cells.length || - cells.some((cell) => !PRODUCTION_CELL_ID.test(cell)) - ) throw new Error('selector membership is invalid') - return cells -} - -function integer(value, name, { minimum = 0, maximum = Number.MAX_SAFE_INTEGER } = {}) { - const parsed = Number(value) - if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { - throw new Error(`${name} is invalid`) - } - return parsed -} - -export function parseRegionalRehomeArguments(argv, environment = process.env) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of ['mode', 'director-origin', 'expected-control-generation']) { - if (!values[key]) throw new Error(`missing --${key}`) - } - if (!MODES.has(values.mode)) throw new Error('--mode is invalid') - if (values['director-origin'] !== DIRECTOR_ORIGIN) { - throw new Error('--director-origin must be the production Relay origin') - } - const recovery = values.mode === 'recover-enable' - const mutation = values.mode !== 'inspect' && !recovery - const mutationKeys = [ - 'not-before', - 'rate-per-minute', - 'preference-max-age-ms', - 'host-cooldown-ms', - 'drain-grace-ms', - 'confirmation' - ] - if (mutation && mutationKeys.some((key) => values[key] === undefined)) { - throw new Error('mutations require the complete durable control shape') - } - if (!mutation && !recovery && mutationKeys.some((key) => values[key] !== undefined)) { - throw new Error('inspect cannot carry mutation arguments') - } - const selectorKeys = [ - 'expected-selector-generation', - 'expected-existing-only-cells', - 'expected-migration-only-cells', - 'expected-general-cells' - ] - if (!recovery && selectorKeys.some((key) => values[key] === undefined)) { - throw new Error('operation requires exact selector state') - } - if (recovery && selectorKeys.some((key) => values[key] !== undefined)) { - throw new Error('enable recovery cannot depend on selector diagnostics') - } - const expectedConfirmation = { - enable: 'ENABLE_REGIONAL_REHOMING', - pause: 'PAUSE_REGIONAL_REHOMING', - disable: 'DISABLE_REGIONAL_REHOMING' - }[values.mode] - if (mutation && values.confirmation !== expectedConfirmation) { - throw new Error('confirmation does not match the requested control action') - } - if (recovery && values.confirmation !== 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE') { - throw new Error('confirmation does not authorize failed-enable recovery') - } - if ( - recovery && - mutationKeys - .filter((key) => key !== 'confirmation') - .some((key) => values[key] !== undefined) - ) throw new Error('enable recovery cannot carry durable control shape arguments') - const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN - if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') - return { - mode: values.mode, - directorOrigin: DIRECTOR_ORIGIN, - ...(!recovery - ? { - expectedSelectorGeneration: integer( - values['expected-selector-generation'], - '--expected-selector-generation' - ), - expectedMembership: { - existingOnly: canonicalCells(values['expected-existing-only-cells']), - migrationOnly: canonicalCells(values['expected-migration-only-cells']), - general: canonicalCells(values['expected-general-cells']) - } - } - : {}), - expectedControlGeneration: integer( - values['expected-control-generation'], - '--expected-control-generation' - ), - ...(mutation - ? { - notBefore: integer(values['not-before'], '--not-before'), - ratePerMinute: integer(values['rate-per-minute'], '--rate-per-minute', { - minimum: 1, - maximum: 120 - }), - preferenceMaxAgeMs: integer( - values['preference-max-age-ms'], - '--preference-max-age-ms', - { minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 } - ), - hostCooldownMs: integer( - values['host-cooldown-ms'], - '--host-cooldown-ms', - { minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 } - ), - drainGraceMs: integer(values['drain-grace-ms'], '--drain-grace-ms', { - minimum: 60_000, - maximum: 60 * 60_000 - }) - } - : {}), - token - } -} - -async function responseJson(response, label) { - const body = await response.json().catch(() => ({})) - if (!response.ok) throw new Error(`${label} returned ${response.status}: ${body.error ?? 'unknown'}`) - return body -} - -function exactMembership(actual, expected) { - return ['existingOnly', 'migrationOnly', 'general'].every( - (key) => JSON.stringify(actual[key]) === JSON.stringify(expected[key]) - ) -} - -function assertControl(control, expected) { - if ( - (expected.generation !== undefined && control?.generation !== expected.generation) || - typeof control.enabled !== 'boolean' || - !Number.isSafeInteger(control.observationStartedAt) || - !Number.isSafeInteger(control.notBefore) || - !Number.isSafeInteger(control.ratePerMinute) || - !Number.isSafeInteger(control.preferenceMaxAgeMs) || - // A director predating the per-host cooldown does not report it. Reading - // the control and both emergency brakes must keep working against that - // image; only enable requires the field. - (control.hostCooldownMs !== undefined && - !Number.isSafeInteger(control.hostCooldownMs)) || - !Number.isSafeInteger(control.drainGraceMs) - ) throw new Error('director returned an invalid regional rehome control') - if (expected.enabled !== undefined && control.enabled !== expected.enabled) { - throw new Error('regional rehome enabled state does not match') - } - return control -} - -// Echo the cooldown only when the director already reports it: a legacy -// director rejects the unknown key outright and would refuse every brake. -function cooldownField(before, value) { - return before.hostCooldownMs === undefined ? {} : { hostCooldownMs: value } -} - -async function verifiedDisabledControl(post, generation) { - return assertControl((await post('/v1/admin/regional-rehome-control', { - v: 1, - action: 'inspect' - })).control, { generation, enabled: false }) -} - -async function applyDisabledControl(post, before) { - return assertControl((await post('/v1/admin/regional-rehome-control', { - v: 1, - action: 'apply', - expectedGeneration: before.generation, - enabled: false, - notBefore: before.notBefore, - ratePerMinute: before.ratePerMinute, - preferenceMaxAgeMs: before.preferenceMaxAgeMs, - ...cooldownField(before, before.hostCooldownMs), - drainGraceMs: before.drainGraceMs, - confirmation: 'DISABLE_REGIONAL_REHOMING' - })).control, { generation: before.generation + 1, enabled: false }) -} - -async function resolveAmbiguousDisable(post, before, firstError) { - const observed = assertControl((await post('/v1/admin/regional-rehome-control', { - v: 1, - action: 'inspect' - })).control, {}) - if (observed.generation === before.generation + 1 && !observed.enabled) { - return observed - } - if (observed.generation !== before.generation || !observed.enabled) { - throw new AggregateError( - [firstError], - 'failed-enable recovery reached an unexpected control generation' - ) - } - try { - return await applyDisabledControl(post, before) - } catch (retryError) { - try { - return await verifiedDisabledControl(post, before.generation + 1) - } catch (readbackError) { - throw new AggregateError( - [firstError, retryError, readbackError], - 'failed-enable recovery exhausted two bounded CAS attempts' - ) - } - } -} - -export async function recoverRegionalRehomeEnable(config, post) { - const before = assertControl((await post('/v1/admin/regional-rehome-control', { - v: 1, - action: 'inspect' - })).control, {}) - if ( - before.generation < config.expectedControlGeneration || - (before.generation === config.expectedControlGeneration && before.enabled) - ) throw new Error('durable control cannot belong to the failed enable attempt') - if (!before.enabled) { - const verified = await verifiedDisabledControl(post, before.generation) - return { mode: config.mode, recovered: false, control: verified } - } - let applied - try { - applied = await applyDisabledControl(post, before) - } catch (error) { - applied = await resolveAmbiguousDisable(post, before, error) - } - const verified = await verifiedDisabledControl(post, applied.generation) - return { mode: config.mode, recovered: true, control: verified } -} - -export async function operateRegionalRehome(config, dependencies = {}) { - const fetchImpl = dependencies.fetch ?? fetch - const post = dependencies.post ?? (async (path, body) => await responseJson( - // Generation-guarded writes make a retry a no-op or an explicit mismatch, never a double apply. - await fetchAdminOnceMore( - fetchImpl, - `${config.directorOrigin}${path}`, - { - method: 'POST', - headers: { - authorization: `Bearer ${config.token}`, - 'content-type': 'application/json' - }, - body: JSON.stringify(body) - }, - { wait: dependencies.wait } - ), - path - )) - if (config.mode === 'recover-enable') { - return await recoverRegionalRehomeEnable(config, post) - } - const selector = (await inspectAdmissionSelector(post)).selector - if ( - selector.generation !== config.expectedSelectorGeneration || - !exactMembership(selector.membership, config.expectedMembership) - ) throw new Error('admission selector does not match the reviewed generation and membership') - - const inspected = await post('/v1/admin/regional-rehome-control', { - v: 1, - action: 'inspect' - }) - const before = assertControl(inspected.control, { - generation: config.expectedControlGeneration - }) - if (config.mode === 'inspect') return { mode: config.mode, selector, control: before } - if (config.mode === 'enable' && before.enabled) { - throw new Error('regional rehome is already enabled; inspect before changing its rate') - } - if (config.mode === 'pause' && !before.enabled) { - throw new Error('regional rehome is already paused') - } - const enabled = config.mode === 'enable' - if (enabled && before.hostCooldownMs === undefined) { - throw new Error( - 'director does not report a per-host rehome cooldown; deploy a director that supports it before enabling' - ) - } - const applied = await post('/v1/admin/regional-rehome-control', { - v: 1, - action: 'apply', - expectedGeneration: config.expectedControlGeneration, - enabled, - notBefore: config.notBefore, - ratePerMinute: config.ratePerMinute, - preferenceMaxAgeMs: config.preferenceMaxAgeMs, - ...cooldownField(before, config.hostCooldownMs), - drainGraceMs: config.drainGraceMs, - confirmation: enabled - ? 'ENABLE_REGIONAL_REHOMING' - : 'DISABLE_REGIONAL_REHOMING' - }) - const after = assertControl(applied.control, { - generation: config.expectedControlGeneration + 1, - enabled - }) - const verified = assertControl((await post('/v1/admin/regional-rehome-control', { - v: 1, - action: 'inspect' - })).control, { - generation: after.generation, - enabled - }) - return { mode: config.mode, selector, control: verified } -} - -export async function main( - argv = process.argv.slice(2), - environment = process.env, - dependencies = {}, - write = (value) => process.stdout.write(value) -) { - const result = await operateRegionalRehome( - parseRegionalRehomeArguments(argv, environment), - dependencies - ) - write(`${JSON.stringify({ event: 'relay_regional_rehome_control', ...result })}\n`) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/operate-relay-regional-rehome.test.mjs b/cloud/dev/scripts/operate-relay-regional-rehome.test.mjs deleted file mode 100644 index 8e676833ec7..00000000000 --- a/cloud/dev/scripts/operate-relay-regional-rehome.test.mjs +++ /dev/null @@ -1,446 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { - main, - operateRegionalRehome, - parseRegionalRehomeArguments, - recoverRegionalRehomeEnable -} from './operate-relay-regional-rehome.mjs' - -const membership = { - existingOnly: ['production-gce-c1'], - migrationOnly: ['production-gce-c2'], - general: ['production-gce-c7', 'production-gce-c27'] -} - -function argumentsFor(mode, confirmation) { - return [ - '--mode', mode, - '--director-origin', 'https://relay.onorca.dev', - '--expected-selector-generation', '11', - '--expected-existing-only-cells', membership.existingOnly.join(','), - '--expected-migration-only-cells', membership.migrationOnly.join(','), - '--expected-general-cells', membership.general.join(','), - '--expected-control-generation', '4', - ...(mode === 'inspect' ? [] : [ - '--not-before', '2000000000000', - '--rate-per-minute', '10', - '--preference-max-age-ms', '86400000', - '--host-cooldown-ms', '604800000', - '--drain-grace-ms', '60000', - '--confirmation', confirmation - ]) - ] -} - -function control(generation, enabled) { - return { - generation, - enabled, - observationStartedAt: 1, - notBefore: 2_000_000_000_000, - ratePerMinute: 10, - preferenceMaxAgeMs: 86_400_000, - hostCooldownMs: 604_800_000, - drainGraceMs: 60_000 - } -} - -// The control a director predating the per-host cooldown reports. -function legacyControl(generation, enabled) { - const { hostCooldownMs: _absent, ...rest } = control(generation, enabled) - return rest -} - -function legacyDirector(controls) { - const requests = [] - const post = async (path, body) => { - requests.push({ path, body }) - if (path === '/v1/admin/admission-selector/status') { - return { selector: { generation: 11, membership } } - } - return { v: 1, control: controls.shift() } - } - return { requests, post } -} - -test('parses exact selector and typed control confirmation', () => { - const parsed = parseRegionalRehomeArguments( - argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ) - assert.equal(parsed.expectedSelectorGeneration, 11) - assert.equal(parsed.expectedControlGeneration, 4) - assert.equal(parsed.ratePerMinute, 10) - assert.equal(parsed.hostCooldownMs, 604_800_000) - assert.throws( - () => parseRegionalRehomeArguments( - argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING').filter( - (value, index, all) => - value !== '--host-cooldown-ms' && all[index - 1] !== '--host-cooldown-ms' - ), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ), - /complete durable control shape/ - ) - assert.throws( - () => parseRegionalRehomeArguments( - argumentsFor('pause', 'DISABLE_REGIONAL_REHOMING'), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ), - /confirmation/ - ) - assert.throws( - () => parseRegionalRehomeArguments( - argumentsFor('inspect').concat('--rate-per-minute', '10'), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ), - /inspect cannot/ - ) -}) - -test('accepts every one- and two-digit production cell and rejects malformed membership', () => { - const parseGeneral = (general) => parseRegionalRehomeArguments( - argumentsFor('inspect').map((value, index, all) => - all[index - 1] === '--expected-general-cells' ? general : value - ), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ).expectedMembership.general - assert.deepEqual( - parseGeneral('production-gce-c30,production-gce-c9,production-gce-c29'), - ['production-gce-c29', 'production-gce-c30', 'production-gce-c9'] - ) - assert.deepEqual(parseGeneral('production-gce-c99'), ['production-gce-c99']) - for (const general of [ - 'production-gce-c0', - 'production-gce-c100', - 'production-gce-c09', - 'production-gce-c30,production-gce-c30', - 'staging-gce-c30', - ',' - ]) { - assert.throws(() => parseGeneral(general), /selector membership is invalid/, general) - } -}) - -test('binds enable to exact selector and durable control generations', async () => { - const requests = [] - const controls = [ - { generation: 4, enabled: false }, - { generation: 5, enabled: true }, - { generation: 5, enabled: true } - ].map((control) => ({ - observationStartedAt: 1, - notBefore: 0, - ratePerMinute: 10, - preferenceMaxAgeMs: 86_400_000, - hostCooldownMs: 604_800_000, - drainGraceMs: 60_000, - ...control - })) - const config = parseRegionalRehomeArguments( - argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ) - const result = await operateRegionalRehome(config, { - post: async (path, body) => { - requests.push({ path, body }) - if (path === '/v1/admin/admission-selector/status') { - return { selector: { generation: 11, membership } } - } - return { v: 1, control: controls.shift() } - } - }) - assert.equal(result.control.generation, 5) - assert.equal(result.control.hostCooldownMs, 604_800_000) - assert.deepEqual(requests[2].body, { - v: 1, - action: 'apply', - expectedGeneration: 4, - enabled: true, - notBefore: 2_000_000_000_000, - ratePerMinute: 10, - preferenceMaxAgeMs: 86_400_000, - hostCooldownMs: 604_800_000, - drainGraceMs: 60_000, - confirmation: 'ENABLE_REGIONAL_REHOMING' - }) -}) - -test('inspects a director that predates the per-host cooldown', async () => { - const director = legacyDirector([legacyControl(4, true)]) - const config = parseRegionalRehomeArguments( - argumentsFor('inspect'), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ) - - const result = await operateRegionalRehome(config, { post: director.post }) - - assert.equal(result.control.generation, 4) - assert.equal(result.control.hostCooldownMs, undefined) -}) - -for (const [mode, confirmation, enabledBefore] of [ - ['pause', 'PAUSE_REGIONAL_REHOMING', true], - ['disable', 'DISABLE_REGIONAL_REHOMING', false] -]) { - test(`${mode} still brakes a director that predates the cooldown`, async () => { - const director = legacyDirector([ - legacyControl(4, enabledBefore), - legacyControl(5, false), - legacyControl(5, false) - ]) - const config = parseRegionalRehomeArguments( - argumentsFor(mode, confirmation), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ) - - const result = await operateRegionalRehome(config, { post: director.post }) - - assert.equal(result.control.generation, 5) - // The unknown key would be refused by that director's strict schema. - assert.equal('hostCooldownMs' in director.requests[2].body, false) - assert.equal(director.requests[2].body.confirmation, 'DISABLE_REGIONAL_REHOMING') - }) -} - -test('failed-enable recovery brakes a director that predates the cooldown', async () => { - const requests = [] - let current = legacyControl(7, true) - const result = await recoverRegionalRehomeEnable({ - mode: 'recover-enable', - expectedControlGeneration: 4 - }, async (_path, body) => { - requests.push(body) - if (body.action === 'inspect') return { control: current } - current = legacyControl(8, false) - return { control: current } - }) - - assert.equal(result.control.generation, 8) - assert.equal('hostCooldownMs' in requests[1], false) -}) - -test('refuses to enable a director that does not report the cooldown', async () => { - const director = legacyDirector([legacyControl(4, false)]) - const config = parseRegionalRehomeArguments( - argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ) - - await assert.rejects( - operateRegionalRehome(config, { post: director.post }), - /per-host rehome cooldown/ - ) - // Read-only: selector status and the control inspect, and nothing else. - assert.equal(director.requests.length, 2) - assert.equal(director.requests.every(({ body }) => body.action !== 'apply'), true) -}) - -test('fails closed on selector drift before reading or mutating control', async () => { - let calls = 0 - const config = parseRegionalRehomeArguments( - argumentsFor('disable', 'DISABLE_REGIONAL_REHOMING'), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ) - await assert.rejects( - operateRegionalRehome(config, { - post: async () => { - calls += 1 - return { selector: { generation: 12, membership } } - } - }), - /selector/ - ) - assert.equal(calls, 1) -}) - -test('failed-enable recovery CAS-disables an advanced enabled generation', async () => { - const requests = [] - let current = control(7, true) - const result = await recoverRegionalRehomeEnable({ - mode: 'recover-enable', - expectedControlGeneration: 4 - }, async (_path, body) => { - requests.push(body) - if (body.action === 'inspect') return { control: current } - assert.equal(body.expectedGeneration, 7) - current = control(8, false) - throw new Error('enable recovery response was lost') - }) - assert.equal(result.recovered, true) - assert.deepEqual(result.control, control(8, false)) - assert.deepEqual(requests[1], { - v: 1, - action: 'apply', - expectedGeneration: 7, - enabled: false, - notBefore: 2_000_000_000_000, - ratePerMinute: 10, - preferenceMaxAgeMs: 86_400_000, - hostCooldownMs: 604_800_000, - drainGraceMs: 60_000, - confirmation: 'DISABLE_REGIONAL_REHOMING' - }) -}) - -test('failed-enable recovery retries once when the first CAS never commits', async () => { - let current = control(7, true) - const applyRequests = [] - const result = await recoverRegionalRehomeEnable({ - mode: 'recover-enable', - expectedControlGeneration: 4 - }, async (_path, body) => { - if (body.action === 'inspect') return { control: current } - applyRequests.push(body) - if (applyRequests.length === 1) { - throw new Error('disable request was lost before commit') - } - current = control(8, false) - throw new Error('retry response was lost after commit') - }) - assert.equal(applyRequests.length, 2) - assert.deepEqual(applyRequests[1], applyRequests[0]) - assert.equal(result.recovered, true) - assert.deepEqual(result.control, control(8, false)) -}) - -test('failed-enable recovery stops after two uncommitted CAS attempts', async () => { - let applyCalls = 0 - await assert.rejects( - recoverRegionalRehomeEnable({ - mode: 'recover-enable', - expectedControlGeneration: 4 - }, async (_path, body) => { - if (body.action === 'inspect') return { control: control(7, true) } - applyCalls += 1 - throw new Error(`disable attempt ${applyCalls} was lost before commit`) - }), - /exhausted two bounded CAS attempts/ - ) - assert.equal(applyCalls, 2) -}) - -test('failed-enable recovery is a verified no-op before enable and after cleanup', async () => { - for (const current of [control(4, false), control(8, false)]) { - const requests = [] - const result = await recoverRegionalRehomeEnable({ - mode: 'recover-enable', - expectedControlGeneration: 4 - }, async (_path, body) => { - requests.push(body) - return { control: current } - }) - assert.equal(result.recovered, false) - assert.equal(result.control.enabled, false) - assert.deepEqual(requests.map(({ action }) => action), ['inspect', 'inspect']) - } -}) - -test('failed-enable recovery rejects an unchanged pre-existing enabled state', async () => { - await assert.rejects( - recoverRegionalRehomeEnable({ - mode: 'recover-enable', - expectedControlGeneration: 4 - }, async () => ({ control: control(4, true) })), - /cannot belong to the failed enable attempt/ - ) -}) - -test('parses recovery without depending on selector diagnostics', () => { - const recoveryArguments = [ - '--mode', 'recover-enable', - '--director-origin', 'https://relay.onorca.dev', - '--expected-control-generation', '4', - '--confirmation', 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE' - ] - const parsed = parseRegionalRehomeArguments( - recoveryArguments, - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ) - assert.equal(parsed.expectedControlGeneration, 4) - assert.equal(parsed.expectedMembership, undefined) - assert.throws( - () => parseRegionalRehomeArguments( - recoveryArguments.concat('--not-before', '2000000000000'), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ), - /cannot carry durable control shape/ - ) -}) - -test('main executes recovery mode and emits verified disabled control', async () => { - let current = control(5, true) - let output = '' - await main([ - '--mode', 'recover-enable', - '--director-origin', 'https://relay.onorca.dev', - '--expected-control-generation', '4', - '--confirmation', 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE' - ], { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }, { - post: async (_path, body) => { - if (body.action === 'apply') current = control(6, false) - return { control: current } - } - }, (value) => { - output += value - }) - assert.deepEqual(JSON.parse(output), { - event: 'relay_regional_rehome_control', - mode: 'recover-enable', - recovered: true, - control: control(6, false) - }) -}) - -test('retries a transient 503 on the director control endpoint', async () => { - const config = parseRegionalRehomeArguments( - argumentsFor('inspect'), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ) - const paths = [] - let selectorCalls = 0 - const result = await operateRegionalRehome(config, { - wait: async () => {}, - fetch: async (url) => { - const path = new URL(url).pathname - paths.push(path) - if (path === '/v1/admin/admission-selector/status') { - selectorCalls += 1 - // The first read of each admin path 503s the way a warming instance does. - if (selectorCalls === 1) return new Response('warming up', { status: 503 }) - return Response.json({ selector: { generation: 11, membership } }) - } - if (paths.filter((value) => value === path).length === 1) { - return new Response('warming up', { status: 503 }) - } - return Response.json({ v: 1, control: control(4, false) }) - } - }) - assert.equal(result.control.generation, 4) - assert.deepEqual(paths, [ - '/v1/admin/admission-selector/status', - '/v1/admin/admission-selector/status', - '/v1/admin/regional-rehome-control', - '/v1/admin/regional-rehome-control' - ]) -}) - -test('fails when both attempts at the director control endpoint return 503', async () => { - const config = parseRegionalRehomeArguments( - argumentsFor('inspect'), - { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } - ) - let calls = 0 - await assert.rejects( - operateRegionalRehome(config, { - wait: async () => {}, - fetch: async () => { - calls += 1 - return new Response('warming up', { status: 503 }) - } - }), - /returned 503/ - ) - assert.equal(calls, 2) -}) diff --git a/cloud/dev/scripts/power-staging-relay.mjs b/cloud/dev/scripts/power-staging-relay.mjs deleted file mode 100644 index e68c01eadb7..00000000000 --- a/cloud/dev/scripts/power-staging-relay.mjs +++ /dev/null @@ -1,537 +0,0 @@ -import { readFileSync } from 'node:fs' -import { spawnSync } from 'node:child_process' -import { pathToFileURL } from 'node:url' -import { - inspectAdmissionSelector, - selectorCellState -} from './relay-admission-selector.mjs' - -const PROJECT = 'onorca-cloud-staging' -const REGION = 'us-central1' -const DIRECTOR_ORIGIN = 'https://relay-staging.onorca.dev' -const ADMIN_AUDIENCE = `${DIRECTOR_ORIGIN}/v1/admin/drain` -const SQL_INSTANCE = 'orca-cloud-staging-auth-db' -const CLOUD_RUN_SERVICES = [ - { name: 'orca-cloud-relay-staging', healthOrigin: DIRECTOR_ORIGIN }, - { name: 'orca-cloud-auth-staging', healthOrigin: 'https://auth-staging.onorca.dev' } -] -const POLL_INTERVAL_MS = 5_000 -const WAKE_TIMEOUT_MS = 12 * 60 * 1_000 - -export function parseArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`) - const name = key.slice(2) - if (!['mode', 'wake-cells', 'topology-file'].includes(name)) { - throw new Error(`unsupported argument --${name}`) - } - values[name] = value - } - if (!['status', 'sleep', 'wake'].includes(values.mode)) { - throw new Error('--mode must be status, sleep, or wake') - } - if (!['configured', 'all'].includes(values['wake-cells'])) { - throw new Error('--wake-cells must be configured or all') - } - if (!values['topology-file']) throw new Error('missing --topology-file') - return { - mode: values.mode, - wakeCells: values['wake-cells'], - topologyFile: values['topology-file'] - } -} - -function canonicalStagingCell(cellId, value) { - if (!/^staging-gce-[a-z0-9-]+$/.test(cellId)) throw new Error(`unsafe staging cell ID ${cellId}`) - if (!value || typeof value !== 'object') throw new Error(`missing topology for ${cellId}`) - const cell = { - cellId, - migName: String(value.mig_name ?? ''), - zone: String(value.zone ?? ''), - origin: String(value.origin ?? ''), - initiallyEnabled: value.initially_enabled - } - if (!/^orca-cloud-staging-relay-gce-[a-z0-9-]+$/.test(cell.migName)) { - throw new Error(`${cellId} has an unsafe MIG name`) - } - if (!/^(?:us-central1|asia-east2)-[a-z]$/.test(cell.zone)) { - throw new Error(`${cellId} has an unsafe zone`) - } - const origin = new URL(cell.origin) - if ( - origin.protocol !== 'https:' || - origin.origin !== cell.origin || - !origin.hostname.endsWith('.relay-staging.onorca.dev') - ) { - throw new Error(`${cellId} has an unsafe origin`) - } - if (typeof cell.initiallyEnabled !== 'boolean') { - throw new Error(`${cellId} has no initial admission state`) - } - return cell -} - -export function readStagingTopology(file) { - const parsed = JSON.parse(readFileSync(file, 'utf8')) - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - throw new Error('staging topology must be an object') - } - const cells = Object.entries(parsed) - .map(([cellId, value]) => canonicalStagingCell(cellId, value)) - .sort((left, right) => left.cellId.localeCompare(right.cellId)) - if (cells.length < 2 || cells.length > 10) { - throw new Error('staging topology must contain 2..10 cells') - } - if (cells.filter((cell) => cell.initiallyEnabled).length < 2) { - throw new Error('staging topology must retain two configured admission cells') - } - return cells -} - -function defaultCommand(args, json) { - const result = spawnSync('gcloud', args, { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'pipe'] - }) - if (result.status !== 0) { - throw new Error(`gcloud ${args.slice(0, 5).join(' ')} failed: ${result.stderr.trim()}`) - } - return json ? JSON.parse(result.stdout) : result.stdout.trim() -} - -function suppliedAdminToken(environment = process.env) { - const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN - if (!token || token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) { - throw new Error('workflow did not supply a valid masked staging admin token') - } - return token -} - -async function responseJson(response, label) { - const body = await response.json().catch(() => ({ error: `http_${response.status}` })) - if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`) - return body -} - -function createAdminPost(deps) { - const token = deps.adminToken() - return async (origin, path, body) => - await responseJson( - await deps.fetch(`${origin}${path}`, { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(30_000) - }), - path - ) -} - -async function waitUntil(deps, label, operation, timeoutMs = WAKE_TIMEOUT_MS) { - const deadline = deps.now() + timeoutMs - let lastError - while (deps.now() < deadline) { - try { - const result = await operation() - if (result) return result - } catch (error) { - lastError = error - } - await deps.wait(POLL_INTERVAL_MS) - } - const detail = lastError instanceof Error ? `: ${lastError.message}` : '' - throw new Error(`timed out waiting for ${label}${detail}`) -} - -async function checkHealth(deps, origin, path) { - const response = await deps.fetch(`${origin}${path}`, { signal: AbortSignal.timeout(15_000) }) - const body = await response.json().catch(() => ({})) - return response.ok && body.ok === true -} - -function sqlActivationPolicy(instance) { - return String(instance.settings?.activationPolicy ?? '') -} - -function describeSql(deps) { - return deps.commandJson([ - 'sql', - 'instances', - 'describe', - SQL_INSTANCE, - '--project', - PROJECT, - '--format=json' - ]) -} - -async function ensureSqlPolicy(deps, policy) { - if (sqlActivationPolicy(describeSql(deps)) === policy) return false - deps.command([ - 'sql', - 'instances', - 'patch', - SQL_INSTANCE, - '--project', - PROJECT, - `--activation-policy=${policy}`, - '--quiet' - ]) - await waitUntil(deps, `Cloud SQL activation policy ${policy}`, () => - sqlActivationPolicy(describeSql(deps)) === policy - ) - return true -} - -function describeMig(deps, cell) { - return deps.commandJson([ - 'compute', - 'instance-groups', - 'managed', - 'describe', - cell.migName, - '--project', - PROJECT, - '--zone', - cell.zone, - '--format=json' - ]) -} - -async function setMigSize(deps, cell, size) { - const before = describeMig(deps, cell) - if (Number(before.targetSize) !== size) { - deps.command([ - 'compute', - 'instance-groups', - 'managed', - 'resize', - cell.migName, - '--project', - PROJECT, - '--zone', - cell.zone, - `--size=${size}`, - '--quiet' - ]) - } - await waitUntil(deps, `${cell.cellId} size ${size}`, () => { - const current = describeMig(deps, cell) - return Number(current.targetSize) === size && current.status?.isStable === true - }) -} - -function activeRevisionName(service) { - const active = (service.status?.traffic ?? []).filter((entry) => Number(entry.percent ?? 0) > 0) - if (active.length !== 1 || Number(active[0].percent) !== 100 || !active[0].revisionName) { - throw new Error('Cloud Run service must have exactly one active revision') - } - return active[0].revisionName -} - -function describeRunService(deps, name) { - return deps.commandJson([ - 'run', - 'services', - 'describe', - name, - '--project', - PROJECT, - '--region', - REGION, - '--format=json' - ]) -} - -function describeRunRevision(deps, name) { - return deps.commandJson([ - 'run', - 'revisions', - 'describe', - name, - '--project', - PROJECT, - '--region', - REGION, - '--format=json' - ]) -} - -function revisionMinimum(revision) { - return Number(revision.metadata?.annotations?.['autoscaling.knative.dev/minScale'] ?? 0) -} - -async function ensureCloudRunScaleToZero(deps, service) { - const before = describeRunService(deps, service.name) - const activeRevision = describeRunRevision(deps, activeRevisionName(before)) - if (revisionMinimum(activeRevision) === 0) return false - - const latestName = before.status?.latestReadyRevisionName - const latest = latestName ? describeRunRevision(deps, latestName) : null - if (!latest || revisionMinimum(latest) !== 0) { - deps.command([ - 'run', - 'services', - 'update', - service.name, - '--project', - PROJECT, - '--region', - REGION, - '--min-instances=0', - '--quiet' - ]) - } - deps.command([ - 'run', - 'services', - 'update-traffic', - service.name, - '--project', - PROJECT, - '--region', - REGION, - '--to-latest', - '--quiet' - ]) - await waitUntil(deps, `${service.name} scale-to-zero revision`, async () => { - const current = describeRunService(deps, service.name) - const revision = describeRunRevision(deps, activeRevisionName(current)) - return revisionMinimum(revision) === 0 && (await checkHealth(deps, service.healthOrigin, '/health')) - }) - return true -} - -async function cellStatus(adminPost, cell) { - const response = await adminPost(DIRECTOR_ORIGIN, '/v1/admin/cell-status', { - v: 1, - cellId: cell.cellId - }) - if (!response.status || response.status.cellId !== cell.cellId) { - throw new Error(`${cell.cellId} returned an invalid status`) - } - return response.status -} - -function assertQuiescent(status) { - const active = { - activityLeases: status.activityLeases, - activityRequestUnits: status.activityRequestUnits, - outgoingMigrations: status.outgoingMigrations, - incomingMigrations: status.incomingMigrations, - observedRequests: status.runtime?.observedRequests ?? 0 - } - if (Object.values(active).some((value) => Number(value) !== 0)) { - throw new Error(`${status.cellId} still has active Relay work: ${JSON.stringify(active)}`) - } -} - -// `undefined` when the director predates the step-5 reserve census (its router 404s the route). -async function reserveCells(deps, cells) { - const token = deps.adminToken() - const reserve = [] - for (const cell of cells) { - const response = await deps.fetch(`${DIRECTOR_ORIGIN}/v1/admin/cell-admit-mode`, { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify({ v: 1, cellId: cell.cellId }), - signal: AbortSignal.timeout(30_000) - }) - const result = await response.json().catch(() => null) - if (response.status === 404 && result === null) return undefined - if (!response.ok || result?.v !== 1 || !['db', 'reserve'].includes(result.admitMode)) { - throw new Error(`${cell.cellId} admit mode read failed: ${result?.error ?? response.status}`) - } - if (result.admitMode === 'reserve') reserve.push(cell.cellId) - } - return reserve -} - -// Under the monotonic selector, admission belongs to the selector, and the selector-era wake -// restores it; so this sleep proves the fleet empty and never writes a cell's admission. -async function sleepUnderSelector(deps, cells, adminPost, selector) { - const reserve = await reserveCells(deps, cells) - if (reserve?.length) { - throw new Error(`staging sleep refuses reserve-mode cells: ${reserve.join(', ')}`) - } - for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service) - const final = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell))) - for (const status of final) assertQuiescent(status) - const after = await inspectAdmissionSelector( - async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body) - ) - if (after.selector.generation !== selector.generation) { - throw new Error('admission selector changed during staging sleep') - } - // The flag workflow runs outside the staging lock, so a reserve flip can land mid-sleep. - const lateReserve = await reserveCells(deps, cells) - if (lateReserve?.length) { - throw new Error(`staging sleep refuses reserve-mode cells: ${lateReserve.join(', ')}`) - } - await Promise.all(cells.map((cell) => setMigSize(deps, cell, 0))) - await ensureSqlPolicy(deps, 'NEVER') - deps.emit({ - event: 'staging_relay_slept', - stoppedCells: cells.map((cell) => cell.cellId), - selectorGeneration: selector.generation - }) -} - -async function setCellState(adminPost, cell, enabled) { - await adminPost(DIRECTOR_ORIGIN, '/v1/admin/cell-state', { - v: 1, - cellId: cell.cellId, - enabled - }) -} - -async function stagingStatus(deps, cells) { - return { - event: 'staging_relay_power_status', - project: PROJECT, - sqlActivationPolicy: sqlActivationPolicy(describeSql(deps)), - cells: cells.map((cell) => ({ - cellId: cell.cellId, - initiallyEnabled: cell.initiallyEnabled, - targetSize: Number(describeMig(deps, cell).targetSize) - })), - cloudRun: CLOUD_RUN_SERVICES.map((service) => { - const described = describeRunService(deps, service.name) - const revision = describeRunRevision(deps, activeRevisionName(described)) - return { service: service.name, activeRevisionMinimum: revisionMinimum(revision) } - }) - } -} - -async function sleepStaging(deps, cells) { - const sqlPolicy = sqlActivationPolicy(describeSql(deps)) - if (sqlPolicy === 'NEVER') { - const runningCells = cells.filter((cell) => Number(describeMig(deps, cell).targetSize) !== 0) - if (runningCells.length > 0) { - // SQL-off plus running workers is an unknown partial state; never kill those workers blindly. - throw new Error( - `staging is partially asleep with running cells: ${runningCells.map((cell) => cell.cellId).join(', ')}` - ) - } - deps.emit({ event: 'staging_relay_sleep_reconciled', alreadyAsleep: true }) - return - } - - const adminPost = createAdminPost(deps) - const initial = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell))) - for (const status of initial) assertQuiescent(status) - const selector = await inspectAdmissionSelector( - async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body) - ) - if (selector.selector.generation > 0) { - await sleepUnderSelector(deps, cells, adminPost, selector.selector) - return - } - const previouslyEnabled = new Set(initial.filter((status) => status.enabled).map((status) => status.cellId)) - - for (const cell of cells) await setCellState(adminPost, cell, false) - await deps.wait(15_000) - try { - const disabled = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell))) - for (const status of disabled) { - if (status.enabled) throw new Error(`${status.cellId} admission did not disable`) - assertQuiescent(status) - } - for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service) - const final = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell))) - for (const status of final) assertQuiescent(status) - } catch (error) { - for (const cell of cells.filter((candidate) => previouslyEnabled.has(candidate.cellId))) { - await setCellState(adminPost, cell, true).catch(() => undefined) - } - throw error - } - - await Promise.all(cells.map((cell) => setMigSize(deps, cell, 0))) - await ensureSqlPolicy(deps, 'NEVER') - deps.emit({ event: 'staging_relay_slept', stoppedCells: cells.map((cell) => cell.cellId) }) -} - -async function wakeStaging(deps, cells, wakeCells) { - await ensureSqlPolicy(deps, 'ALWAYS') - await waitUntil(deps, 'staging director health', () => checkHealth(deps, DIRECTOR_ORIGIN, '/health')) - for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service) - - const adminPost = createAdminPost(deps) - const selector = await inspectAdmissionSelector( - async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body) - ) - const selectorActive = selector.selector.generation > 0 - // Existing-only cells may still own live or dormant assignments, so a - // selector-era wake restores the complete retained topology. - const selected = selectorActive - ? cells - : cells.filter((cell) => wakeCells === 'all' || cell.initiallyEnabled) - await Promise.all( - cells.map((cell) => setMigSize(deps, cell, selected.includes(cell) ? 1 : 0)) - ) - for (const cell of selected) { - await waitUntil(deps, `${cell.cellId} health`, () => checkHealth(deps, cell.origin, '/health')) - await waitUntil(deps, `${cell.cellId} readiness`, () => checkHealth(deps, cell.origin, '/ready')) - } - - for (const cell of cells) { - if (selectorActive) { - const status = await waitUntil(deps, `${cell.cellId} authenticated heartbeat`, async () => { - const current = await cellStatus(adminPost, cell) - return current.runtime?.heartbeatFresh && current.runtime.ready ? current : null - }) - if (status.admissionState !== selectorCellState(selector.selector, cell.cellId)) { - throw new Error(`${cell.cellId} admission does not match selector`) - } - continue - } - if (!selected.includes(cell)) { - await setCellState(adminPost, cell, false) - continue - } - const status = await waitUntil(deps, `${cell.cellId} authenticated heartbeat`, async () => { - const current = await cellStatus(adminPost, cell) - return current.runtime?.heartbeatFresh && current.runtime.ready ? current : null - }) - if (cell.initiallyEnabled && !status.enabled) await setCellState(adminPost, cell, true) - if (!cell.initiallyEnabled && status.enabled) await setCellState(adminPost, cell, false) - } - deps.emit({ - event: 'staging_relay_woke', - runningCells: selected.map((cell) => cell.cellId), - admissionCells: selectorActive - ? selector.selector.membership.general - : selected.filter((cell) => cell.initiallyEnabled).map((cell) => cell.cellId) - }) -} - -export async function runStagingRelayPower(config, overrides = {}) { - const deps = { - command: overrides.command ?? ((args) => defaultCommand(args, false)), - commandJson: overrides.commandJson ?? ((args) => defaultCommand(args, true)), - fetch: overrides.fetch ?? fetch, - adminToken: overrides.adminToken ?? suppliedAdminToken, - emit: overrides.emit ?? ((event) => process.stdout.write(`${JSON.stringify(event)}\n`)), - now: overrides.now ?? Date.now, - wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) - } - const cells = readStagingTopology(config.topologyFile) - if (config.mode === 'status') deps.emit(await stagingStatus(deps, cells)) - else if (config.mode === 'sleep') await sleepStaging(deps, cells) - else await wakeStaging(deps, cells, config.wakeCells) -} - -export async function main(argv = process.argv.slice(2)) { - await runStagingRelayPower(parseArguments(argv)) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/power-staging-relay.test.mjs b/cloud/dev/scripts/power-staging-relay.test.mjs deleted file mode 100644 index 847e1de8744..00000000000 --- a/cloud/dev/scripts/power-staging-relay.test.mjs +++ /dev/null @@ -1,427 +0,0 @@ -import assert from 'node:assert/strict' -import { mkdtempSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { test } from 'node:test' -import { - parseArguments, - readStagingTopology, - runStagingRelayPower -} from './power-staging-relay.mjs' - -function topologyFile(overrides = {}) { - const directory = mkdtempSync(join(tmpdir(), 'staging-relay-power-')) - const topology = { - 'staging-gce-c1': { - mig_name: 'orca-cloud-staging-relay-gce-c1', - zone: 'us-central1-b', - origin: 'https://c1.relay-staging.onorca.dev', - initially_enabled: true - }, - 'staging-gce-c2': { - mig_name: 'orca-cloud-staging-relay-gce-c2', - zone: 'us-central1-c', - origin: 'https://c2.relay-staging.onorca.dev', - initially_enabled: true - }, - 'staging-gce-c3': { - mig_name: 'orca-cloud-staging-relay-gce-c3', - zone: 'us-central1-a', - origin: 'https://c3.relay-staging.onorca.dev', - initially_enabled: false - }, - 'staging-gce-c4': { - mig_name: 'orca-cloud-staging-relay-gce-c4', - zone: 'asia-east2-a', - origin: 'https://c4.relay-staging.onorca.dev', - initially_enabled: false - }, - ...overrides - } - const file = join(directory, 'topology.json') - writeFileSync(file, JSON.stringify(topology)) - return file -} - -function argumentConfig(file, mode, wakeCells = 'configured') { - return parseArguments([ - '--mode', - mode, - '--wake-cells', - wakeCells, - '--topology-file', - file - ]) -} - -function response(body, status = 200) { - return new Response(JSON.stringify(body), { - status, - headers: { 'content-type': 'application/json' } - }) -} - -function harness({ - sqlPolicy = 'ALWAYS', - migSize = 1, - observedRequests = 0, - selectorGeneration = 0, - admitModes -} = {}) { - const cells = new Map( - ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'].map((cellId, index) => [ - cellId, - { - enabled: index < 2, - targetSize: migSize, - observedRequests, - initiallyEnabled: index < 2 - } - ]) - ) - const revisions = new Map([ - ['orca-cloud-relay-staging', { active: 'relay-00001', latest: 'relay-00001', min: 1 }], - ['orca-cloud-auth-staging', { active: 'auth-00001', latest: 'auth-00001', min: 1 }] - ]) - const revisionMinimums = new Map([ - ['relay-00001', 1], - ['auth-00001', 1] - ]) - const commands = [] - const events = [] - const posts = [] - let activationPolicy = sqlPolicy - let clock = 0 - - function cellForMig(name) { - return [...cells.entries()].find(([, value], index) => { - const suffix = `c${index + 1}` - return name.endsWith(suffix) && value - }) - } - - function commandJson(args) { - if (args[0] === 'sql') return { settings: { activationPolicy } } - if (args[0] === 'compute') { - const entry = cellForMig(args[4]) - return { targetSize: entry[1].targetSize, status: { isStable: true } } - } - if (args[0] === 'run' && args[1] === 'services') { - const state = revisions.get(args[3]) - return { - status: { - latestReadyRevisionName: state.latest, - traffic: [{ percent: 100, revisionName: state.active }] - } - } - } - if (args[0] === 'run' && args[1] === 'revisions') { - return { - metadata: { - annotations: { - 'autoscaling.knative.dev/minScale': String(revisionMinimums.get(args[3]) ?? 0) - } - } - } - } - throw new Error(`unexpected JSON command ${args.join(' ')}`) - } - - function command(args) { - commands.push(args) - if (args[0] === 'sql') { - activationPolicy = args.find((arg) => arg.startsWith('--activation-policy='))?.split('=')[1] - return - } - if (args[0] === 'compute') { - const entry = cellForMig(args[4]) - entry[1].targetSize = Number(args.find((arg) => arg.startsWith('--size='))?.split('=')[1]) - return - } - if (args[0] === 'run' && args[1] === 'services' && args[2] === 'update') { - const state = revisions.get(args[3]) - state.latest = `${args[3]}-power` - revisionMinimums.set(state.latest, 0) - return - } - if (args[0] === 'run' && args[1] === 'services' && args[2] === 'update-traffic') { - const state = revisions.get(args[3]) - state.active = state.latest - return - } - throw new Error(`unexpected command ${args.join(' ')}`) - } - - async function fetchImpl(url, options = {}) { - const parsed = new URL(url) - if (!options.method) return response({ ok: true }) - const body = JSON.parse(options.body) - posts.push(parsed.pathname) - if (parsed.pathname === '/v1/admin/cell-admit-mode') { - // A pre-census director's router answers an unknown route with a plain-text 404. - if (!admitModes) return new Response('404 Not Found', { status: 404 }) - return response({ v: 1, cellId: body.cellId, admitMode: admitModes[body.cellId] ?? 'db' }) - } - if (parsed.pathname === '/v1/admin/cell-status') { - const state = cells.get(body.cellId) - const index = [...cells.keys()].indexOf(body.cellId) - return response({ - v: 1, - status: { - cellId: body.cellId, - enabled: state.enabled, - admissionState: - selectorGeneration > 0 - ? index === 0 - ? 'existing-only' - : index === 1 - ? 'migration-only' - : index === 2 - ? 'general' - : 'migration-only' - : state.enabled - ? 'general' - : 'existing-only', - assignments: 0, - reservedRequests: 0, - activityLeases: 0, - activityRequestUnits: 0, - outgoingMigrations: 0, - incomingMigrations: 0, - runtime: { - ready: state.targetSize === 1, - heartbeatFresh: state.targetSize === 1, - observedRequests: state.observedRequests - } - } - }) - } - if (parsed.pathname === '/v1/admin/admission-selector/status') { - return response({ - v: 1, - selector: { - generation: selectorGeneration, - attemptId: null, - membership: { - existingOnly: - selectorGeneration > 0 - ? ['staging-gce-c1'] - : [...cells] - .filter(([, state]) => !state.enabled) - .map(([cellId]) => cellId), - migrationOnly: - selectorGeneration > 0 ? ['staging-gce-c2', 'staging-gce-c4'] : [], - general: - selectorGeneration > 0 - ? ['staging-gce-c3'] - : [...cells] - .filter(([, state]) => state.enabled) - .map(([cellId]) => cellId) - } - }, - intent: null - }) - } - if (parsed.pathname === '/v1/admin/cell-state') { - cells.get(body.cellId).enabled = body.enabled - return response({ ok: true }) - } - throw new Error(`unexpected fetch ${parsed.pathname}`) - } - - return { - cells, - commands, - events, - posts, - admitModes, - deps: { - command, - commandJson, - fetch: fetchImpl, - adminToken: () => 'header.payload.signature', - emit: (event) => events.push(event), - now: () => clock, - wait: async (ms) => { - clock += ms - } - }, - sqlPolicy: () => activationPolicy - } -} - -test('accepts only explicit staging power arguments and topology', () => { - const file = topologyFile() - assert.equal(argumentConfig(file, 'status').mode, 'status') - assert.equal(readStagingTopology(file).at(-1).zone, 'asia-east2-a') - assert.throws(() => argumentConfig(file, 'destroy')) - assert.throws(() => parseArguments(['--mode', 'sleep', '--wake-cells', 'configured'])) - - const unsafe = topologyFile({ - 'staging-gce-c1': { - mig_name: 'orca-cloud-relay-gce-c1', - zone: 'us-central1-a', - origin: 'https://c1.relay.onorca.dev', - initially_enabled: true - } - }) - assert.throws(() => readStagingTopology(unsafe), /unsafe/) - - const unreviewedRegion = topologyFile({ - 'staging-gce-c4': { - mig_name: 'orca-cloud-staging-relay-gce-c4', - zone: 'europe-west1-b', - origin: 'https://c4.relay-staging.onorca.dev', - initially_enabled: false - } - }) - assert.throws(() => readStagingTopology(unreviewedRegion), /unsafe zone/) -}) - -test('refuses sleep before changing admission when a cell has active requests', async () => { - const testHarness = harness({ observedRequests: 1 }) - await assert.rejects( - runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps), - /still has active Relay work/ - ) - assert.equal(testHarness.commands.length, 0) - assert.equal(testHarness.cells.get('staging-gce-c1').enabled, true) -}) - -test('sleeps only after disabling admission and proving zero active work', async () => { - const testHarness = harness() - await runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps) - - assert.equal(testHarness.sqlPolicy(), 'NEVER') - assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [0, 0, 0, 0]) - assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), [false, false, false, false]) - assert.equal(testHarness.events.at(-1).event, 'staging_relay_slept') - assert.equal( - testHarness.commands.filter((args) => args[0] === 'run' && args[2] === 'update').length, - 2 - ) -}) - -test('sleeps under the selector without writing any cell admission', async () => { - for (const admitModes of [undefined, {}]) { - const testHarness = harness({ selectorGeneration: 3, admitModes }) - const before = [...testHarness.cells.values()].map((cell) => cell.enabled) - await runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps) - - assert.equal(testHarness.sqlPolicy(), 'NEVER') - assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [0, 0, 0, 0]) - assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), before) - assert.equal(testHarness.posts.includes('/v1/admin/cell-state'), false) - assert.equal(testHarness.posts.includes('/v1/admin/cell-admit-mode'), true) - assert.deepEqual(testHarness.events.at(-1), { - event: 'staging_relay_slept', - stoppedCells: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'], - selectorGeneration: 3 - }) - } -}) - -test('refuses a selector-era sleep while any cell is reserve or busy', async () => { - const reserve = harness({ selectorGeneration: 3, admitModes: { 'staging-gce-c2': 'reserve' } }) - await assert.rejects( - runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), reserve.deps), - /refuses reserve-mode cells: staging-gce-c2/ - ) - assert.equal(reserve.commands.length, 0) - - // A flip that lands after the first read, while Cloud Run scales down, still stops the resize. - const late = harness({ selectorGeneration: 3, admitModes: {} }) - const run = late.deps.command - late.deps.command = (args) => { - if (args[0] === 'run' && args[2] === 'update-traffic') late.admitModes['staging-gce-c4'] = 'reserve' - return run(args) - } - await assert.rejects( - runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), late.deps), - /refuses reserve-mode cells: staging-gce-c4/ - ) - assert.deepEqual([...late.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1]) - assert.equal(late.sqlPolicy(), 'ALWAYS') - - const busy = harness({ selectorGeneration: 3, observedRequests: 1 }) - await assert.rejects( - runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), busy.deps), - /still has active Relay work/ - ) - assert.equal(busy.commands.length, 0) - assert.deepEqual([...busy.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1]) -}) - -test('a selector-era sleep then wake restores the selector membership', async () => { - const testHarness = harness({ selectorGeneration: 3 }) - const file = topologyFile() - await runStagingRelayPower(argumentConfig(file, 'sleep'), testHarness.deps) - for (const state of testHarness.cells.values()) state.observedRequests = 0 - await runStagingRelayPower(argumentConfig(file, 'wake'), testHarness.deps) - - assert.equal(testHarness.sqlPolicy(), 'ALWAYS') - assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1]) - assert.equal(testHarness.posts.includes('/v1/admin/cell-state'), false) - assert.deepEqual(testHarness.events.at(-1), { - event: 'staging_relay_woke', - runningCells: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'], - admissionCells: ['staging-gce-c3'] - }) -}) - -test('refuses to terminate workers from an unknown partially asleep state', async () => { - const testHarness = harness({ sqlPolicy: 'NEVER', migSize: 1 }) - await assert.rejects( - runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps), - /partially asleep with running cells/ - ) - assert.equal(testHarness.commands.length, 0) - assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1]) -}) - -test('wakes SQL and configured cells while leaving the candidate off and disabled', async () => { - const testHarness = harness({ sqlPolicy: 'NEVER', migSize: 0 }) - for (const cell of testHarness.cells.values()) cell.enabled = false - for (const state of testHarness.cells.values()) state.observedRequests = 0 - await runStagingRelayPower(argumentConfig(topologyFile(), 'wake'), testHarness.deps) - - assert.equal(testHarness.sqlPolicy(), 'ALWAYS') - assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 0, 0]) - assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), [true, true, false, false]) - assert.deepEqual(testHarness.events.at(-1), { - event: 'staging_relay_woke', - runningCells: ['staging-gce-c1', 'staging-gce-c2'], - admissionCells: ['staging-gce-c1', 'staging-gce-c2'] - }) -}) - -test('wakes every retained cell without rewriting selector-era admission', async () => { - const testHarness = harness({ - sqlPolicy: 'NEVER', - migSize: 0, - selectorGeneration: 1 - }) - const states = [...testHarness.cells.values()] - states[0].enabled = false - states[1].enabled = true - states[2].enabled = true - states[3].enabled = true - await runStagingRelayPower(argumentConfig(topologyFile(), 'wake'), testHarness.deps) - - assert.deepEqual(states.map((cell) => cell.targetSize), [1, 1, 1, 1]) - assert.deepEqual(states.map((cell) => cell.enabled), [false, true, true, true]) - assert.deepEqual(testHarness.events.at(-1), { - event: 'staging_relay_woke', - runningCells: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'], - admissionCells: ['staging-gce-c3'] - }) -}) - -test('status is read-only and reports the current billable floor controls', async () => { - const testHarness = harness() - await runStagingRelayPower(argumentConfig(topologyFile(), 'status'), testHarness.deps) - assert.equal(testHarness.commands.length, 0) - assert.equal(testHarness.events[0].project, 'onorca-cloud-staging') - assert.equal(testHarness.events[0].sqlActivationPolicy, 'ALWAYS') - assert.equal(testHarness.events[0].cells.length, 4) -}) diff --git a/cloud/dev/scripts/prepare-relay-asia-director-cells.mjs b/cloud/dev/scripts/prepare-relay-asia-director-cells.mjs deleted file mode 100644 index 06022bc9371..00000000000 --- a/cloud/dev/scripts/prepare-relay-asia-director-cells.mjs +++ /dev/null @@ -1,86 +0,0 @@ -import { readFileSync, writeFileSync } from 'node:fs' -import { fileURLToPath } from 'node:url' - -function argumentsFrom(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of ['current-json', 'topology-json', 'output', 'cell-ids', 'image-digest']) { - if (!values[key]) throw new Error(`missing --${key}`) - } - return values -} - -// Production Asia pools sit 176 ms from Cloud SQL and run at 16; staging C4 and the production -// US cells at the Asia shape keep the default 10. -const DATABASE_POOL_MAX = { - production: { 'asia-east2': 16, 'us-central1': 10 }, - staging: { 'asia-east2': 10 } -} - -export function prepareRelayAsiaDirectorCells({ currentCells, topology, cellIds, imageDigest }) { - if (!Array.isArray(currentCells) || !topology || Array.isArray(topology)) { - throw new Error('director inputs are invalid') - } - const additions = cellIds.split(',').map((value) => value.trim()).filter(Boolean) - if ( - additions.length === 0 || - new Set(additions).size !== additions.length - ) throw new Error('director Asia additions are invalid') - const currentIds = new Set(currentCells.map((cell) => cell.id)) - if (currentIds.size !== currentCells.length) throw new Error('current director cells contain duplicates') - const normalizedCurrent = currentCells.map((cell) => ({ - ...cell, - region: cell.region ?? 'us-central1' - })) - const desiredCells = additions.map((cellId) => { - const cell = topology[cellId] - const pools = DATABASE_POOL_MAX[cellId.split('-')[0]] ?? {} - if ( - !cell || - !Object.hasOwn(pools, cell.region) || - cell.capacity_requests !== 6_000 || - cell.database_pool_max !== pools[cell.region] || - cell.connection_hard_cap !== 3_000 || - cell.connection_unobserved_bound !== 60 || - cell.initially_enabled !== false || - cell.image?.split('@')[1] !== imageDigest - ) throw new Error(`${cellId} state output does not match the reviewed Asia shape`) - return { - id: cellId, - url: cell.origin, - capacityRequests: cell.capacity_requests, - region: cell.region, - initiallyEnabled: false, - connectionHardCap: cell.connection_hard_cap, - connectionUnobservedBound: cell.connection_unobserved_bound - } - }) - const desiredById = new Map(desiredCells.map((cell) => [cell.id, cell])) - for (const current of normalizedCurrent) { - const desired = desiredById.get(current.id) - if (!desired) continue - for (const [key, value] of Object.entries(desired)) { - if (current[key] !== value) { - throw new Error(`${current.id} director configuration differs from the reviewed Asia shape`) - } - } - } - const configured = new Set(normalizedCurrent.map((cell) => cell.id)) - return [...normalizedCurrent, ...desiredCells.filter((cell) => !configured.has(cell.id))] -} - -if (process.argv[1] === fileURLToPath(import.meta.url)) { - const values = argumentsFrom(process.argv.slice(2)) - const result = prepareRelayAsiaDirectorCells({ - currentCells: JSON.parse(readFileSync(values['current-json'], 'utf8')), - topology: JSON.parse(readFileSync(values['topology-json'], 'utf8')), - cellIds: values['cell-ids'], - imageDigest: values['image-digest'] - }) - writeFileSync(values.output, `${JSON.stringify(result)}\n`, { mode: 0o600 }) -} diff --git a/cloud/dev/scripts/prepare-relay-asia-director-cells.test.mjs b/cloud/dev/scripts/prepare-relay-asia-director-cells.test.mjs deleted file mode 100644 index e2f0894648e..00000000000 --- a/cloud/dev/scripts/prepare-relay-asia-director-cells.test.mjs +++ /dev/null @@ -1,153 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { prepareRelayAsiaDirectorCells } from './prepare-relay-asia-director-cells.mjs' - -const digest = `sha256:${'a'.repeat(64)}` -const topologyCell = (ordinal, zone) => ({ - origin: `https://c${ordinal}.relay.onorca.dev`, region: 'asia-east2', zone, - capacity_requests: 6_000, database_pool_max: 16, - connection_hard_cap: 3_000, connection_unobserved_bound: 60, - initially_enabled: false, - image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@${digest}` -}) - -test('preserves current order, defaults predecessor regions, and appends exact Asia cells', () => { - const current = [{ - id: 'production-gce-c1', url: 'https://c1.relay.onorca.dev', - capacityRequests: 4_000, initiallyEnabled: false - }] - const result = prepareRelayAsiaDirectorCells({ - currentCells: current, - topology: { - 'production-gce-c27': topologyCell(27, 'asia-east2-a'), - 'production-gce-c28': topologyCell(28, 'asia-east2-b'), - 'production-gce-c29': topologyCell(29, 'asia-east2-c') - }, - cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', - imageDigest: digest - }) - assert.equal(result[0].region, 'us-central1') - assert.deepEqual(result.slice(1).map(({ id }) => id), [ - 'production-gce-c27', 'production-gce-c28', 'production-gce-c29' - ]) - assert.ok(result.slice(1).every((cell) => - cell.region === 'asia-east2' && cell.initiallyEnabled === false && - cell.connectionHardCap === 3_000 - )) -}) - -test('is idempotent for an exact existing Asia cell and rejects director drift', () => { - const topology = { 'production-gce-c27': topologyCell(27, 'asia-east2-a') } - const current = prepareRelayAsiaDirectorCells({ - currentCells: [], topology, cellIds: 'production-gce-c27', imageDigest: digest - }) - assert.deepEqual(prepareRelayAsiaDirectorCells({ - currentCells: current, topology, cellIds: 'production-gce-c27', imageDigest: digest - }), current) - assert.throws(() => prepareRelayAsiaDirectorCells({ - currentCells: [{ ...current[0], capacityRequests: 5_999 }], - topology, cellIds: 'production-gce-c27', imageDigest: digest - }), /director configuration differs/) -}) - -test('appends C30 after the configured launch cells without touching them', () => { - const launch = prepareRelayAsiaDirectorCells({ - currentCells: [], - topology: { - 'production-gce-c27': topologyCell(27, 'asia-east2-a'), - 'production-gce-c28': topologyCell(28, 'asia-east2-b'), - 'production-gce-c29': topologyCell(29, 'asia-east2-c') - }, - cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', - imageDigest: digest - }) - const result = prepareRelayAsiaDirectorCells({ - currentCells: launch, - topology: { 'production-gce-c30': topologyCell(30, 'asia-east2-a') }, - cellIds: 'production-gce-c30', - imageDigest: digest - }) - assert.deepEqual(result.slice(0, 3), launch) - assert.deepEqual(result[3], { - id: 'production-gce-c30', url: 'https://c30.relay.onorca.dev', capacityRequests: 6_000, - region: 'asia-east2', initiallyEnabled: false, connectionHardCap: 3_000, - connectionUnobservedBound: 60 - }) -}) - -test('checks C30 against its own digest, not the launch cells\' digest', () => { - const c30Digest = `sha256:${'b'.repeat(64)}` - const c30 = { - ...topologyCell(30, 'asia-east2-a'), - image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@${c30Digest}` - } - const launch = [27, 28, 29].map((ordinal) => ({ - id: `production-gce-c${ordinal}`, url: `https://c${ordinal}.relay.onorca.dev`, - capacityRequests: 6_000, region: 'asia-east2', initiallyEnabled: false, - connectionHardCap: 3_000, connectionUnobservedBound: 60 - })) - assert.equal(prepareRelayAsiaDirectorCells({ - currentCells: launch, topology: { 'production-gce-c30': c30 }, - cellIds: 'production-gce-c30', imageDigest: c30Digest - }).length, 4) - assert.throws(() => prepareRelayAsiaDirectorCells({ - currentCells: launch, topology: { 'production-gce-c30': c30 }, - cellIds: 'production-gce-c30', imageDigest: digest - }), /does not match/) -}) - -test('pins the Asia pool per environment', () => { - const staging = { ...topologyCell(4, 'asia-east2-a'), database_pool_max: 10 } - assert.equal(prepareRelayAsiaDirectorCells({ - currentCells: [], topology: { 'staging-gce-c4': staging }, - cellIds: 'staging-gce-c4', imageDigest: digest - }).length, 1) - assert.throws(() => prepareRelayAsiaDirectorCells({ - currentCells: [], topology: { 'staging-gce-c4': { ...staging, database_pool_max: 16 } }, - cellIds: 'staging-gce-c4', imageDigest: digest - }), /does not match/) - assert.throws(() => prepareRelayAsiaDirectorCells({ - currentCells: [], - topology: { 'production-gce-c30': { ...topologyCell(30, 'asia-east2-a'), database_pool_max: 10 } }, - cellIds: 'production-gce-c30', imageDigest: digest - }), /does not match/) -}) - -test('rejects a mismatching topology state output', () => { - const wrong = topologyCell(27, 'asia-east2-a') - wrong.database_pool_max = 20 - assert.throws(() => prepareRelayAsiaDirectorCells({ - currentCells: [], topology: { 'production-gce-c27': wrong }, - cellIds: 'production-gce-c27', imageDigest: digest - }), /does not match/) -}) - -test('appends a US cell at the Asia shape with the default pool, never the Asia one', () => { - const usCell = { - ...topologyCell(32, 'us-central1-a'), region: 'us-central1', database_pool_max: 10 - } - const current = [{ - id: 'production-gce-c26', url: 'https://c26.relay.onorca.dev', capacityRequests: 4_000, - initiallyEnabled: false, connectionHardCap: 1_000, connectionUnobservedBound: 60 - }] - const result = prepareRelayAsiaDirectorCells({ - currentCells: current, topology: { 'production-gce-c32': usCell }, - cellIds: 'production-gce-c32', imageDigest: digest - }) - assert.deepEqual(result.at(-1), { - id: 'production-gce-c32', url: 'https://c32.relay.onorca.dev', capacityRequests: 6_000, - region: 'us-central1', initiallyEnabled: false, connectionHardCap: 3_000, - connectionUnobservedBound: 60 - }) - for (const drift of [{ database_pool_max: 16 }, { region: 'europe-west1' }]) { - assert.throws(() => prepareRelayAsiaDirectorCells({ - currentCells: current, topology: { 'production-gce-c32': { ...usCell, ...drift } }, - cellIds: 'production-gce-c32', imageDigest: digest - }), /reviewed Asia shape/, JSON.stringify(drift)) - } - // Staging has no US cell at this shape. - assert.throws(() => prepareRelayAsiaDirectorCells({ - currentCells: [], topology: { 'staging-gce-c4': { ...usCell, database_pool_max: 10 } }, - cellIds: 'staging-gce-c4', imageDigest: digest - }), /reviewed Asia shape/) -}) diff --git a/cloud/dev/scripts/prepare-relay-asia-topology-input.mjs b/cloud/dev/scripts/prepare-relay-asia-topology-input.mjs deleted file mode 100644 index 084d3bee692..00000000000 --- a/cloud/dev/scripts/prepare-relay-asia-topology-input.mjs +++ /dev/null @@ -1,140 +0,0 @@ -import { readFileSync } from 'node:fs' -import { fileURLToPath } from 'node:url' - -const BOOT_IMAGE = 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13' -const SHAPES = { - staging: { - project: 'onorca-cloud-staging', - databasePoolMax: { 'asia-east2': 10 }, - cells: { 'staging-gce-c4': 'asia-east2-a' }, - waves: [['staging-gce-c4']] - }, - production: { - project: 'onorca-cloud', - // 16 covers asia-east2's round trip to us-central1 Postgres; US cells keep the default. - databasePoolMax: { 'asia-east2': 16, 'us-central1': 10 }, - cells: { - 'production-gce-c27': 'asia-east2-a', - 'production-gce-c28': 'asia-east2-b', - 'production-gce-c29': 'asia-east2-c', - 'production-gce-c30': 'asia-east2-a', - 'production-gce-c31': 'asia-east2-b', - 'production-gce-c32': 'us-central1-a', - 'production-gce-c33': 'us-central1-b', - 'production-gce-c34': 'asia-east2-c', - 'production-gce-c35': 'asia-east2-a' - }, - // The launch set, then each later additive cell; a plan targets one wave, never live cells. - waves: [ - ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - ['production-gce-c30'], - ['production-gce-c31'], - // Declared together, so they plan together: a lone C32 plan would hit C33's missing template. - ['production-gce-c32', 'production-gce-c33'], - ['production-gce-c34'], - ['production-gce-c35'] - ] - } -} - -function argumentsFrom(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of ['existing-json', 'environment', 'cell-ids', 'image']) { - if (!values[key]) throw new Error(`missing --${key}`) - } - return values -} - -function canonical(value) { - if (Array.isArray(value)) return value.map(canonical) - if (value && typeof value === 'object') { - return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])])) - } - return value -} - -export function prepareRelayAsiaTopologyInput({ - existingCells, - existingAdditionalRegions, - environment, - cellIds, - image -}) { - const shape = SHAPES[environment] - if (!shape) throw new Error('invalid environment') - const requested = cellIds.split(',').map((value) => value.trim()).filter(Boolean).sort() - const expected = shape.waves - .map((wave) => [...wave].sort()) - .find((wave) => JSON.stringify(wave) === JSON.stringify(requested)) - if (new Set(requested).size !== requested.length || !expected) { - throw new Error('cell IDs do not match the reviewed Asia topology') - } - const prefix = `us-central1-docker.pkg.dev/${shape.project}/orca-cloud/relay@sha256:` - if (!image.startsWith(prefix) || !/sha256:[a-f0-9]{64}$/.test(image)) { - throw new Error('image is not the environment Relay image pinned by digest') - } - if (!existingCells || Array.isArray(existingCells) || typeof existingCells !== 'object') { - throw new Error('existing Relay cells must be an object') - } - if ( - !existingAdditionalRegions || - Array.isArray(existingAdditionalRegions) || - typeof existingAdditionalRegions !== 'object' || - JSON.stringify(canonical(existingAdditionalRegions)) !== - JSON.stringify(canonical({ 'asia-east2': '10.42.1.0/24' })) - ) { - throw new Error('Asia subnet must be committed before topology planning') - } - const additions = Object.fromEntries(expected.map((cellId) => { - const hostname = cellId.split('-').at(-1) - const zone = shape.cells[cellId] - const region = zone.slice(0, zone.lastIndexOf('-')) - return [cellId, { - hostname, - region, - zone, - machine_type: 'e2-standard-4', - boot_disk_gb: 30, - boot_image: BOOT_IMAGE, - capacity_requests: 6_000, - database_pool_max: shape.databasePoolMax[region], - image, - initially_enabled: false, - connection_hard_cap: 3_000, - connection_unobserved_bound: 60 - }] - })) - for (const cellId of expected) { - if (!existingCells[cellId]) { - throw new Error('Asia cells must be committed before topology planning') - } - if ( - JSON.stringify(canonical(existingCells[cellId])) !== - JSON.stringify(canonical(additions[cellId])) - ) { - throw new Error('committed Asia cell differs from the reviewed topology') - } - } - return { - relay_gce_additional_region_subnetwork_cidrs: existingAdditionalRegions, - relay_gce_cells: existingCells - } -} - -if (process.argv[1] === fileURLToPath(import.meta.url)) { - const values = argumentsFrom(process.argv.slice(2)) - const existing = JSON.parse(readFileSync(values['existing-json'], 'utf8')) - prepareRelayAsiaTopologyInput({ - existingCells: existing.relay_gce_cells, - existingAdditionalRegions: existing.relay_gce_additional_region_subnetwork_cidrs, - environment: values.environment, - cellIds: values['cell-ids'], - image: values.image - }) -} diff --git a/cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs b/cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs deleted file mode 100644 index 0686df2a394..00000000000 --- a/cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs +++ /dev/null @@ -1,211 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import { test } from 'node:test' -import { prepareRelayAsiaTopologyInput } from './prepare-relay-asia-topology-input.mjs' - -const image = `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:${'a'.repeat(64)}` - -const additionalRegions = { 'asia-east2': '10.42.1.0/24' } - -const productionCells = () => Object.fromEntries([ - [27, 'asia-east2-a'], - [28, 'asia-east2-b'], - [29, 'asia-east2-c'], - [30, 'asia-east2-a'], - [31, 'asia-east2-b'], - [32, 'us-central1-a'], - [33, 'us-central1-b'], - [34, 'asia-east2-c'], - [35, 'asia-east2-a'] -].map(([ordinal, zone]) => [`production-gce-c${ordinal}`, { - hostname: `c${ordinal}`, region: zone.slice(0, -2), zone, - machine_type: 'e2-standard-4', boot_disk_gb: 30, - boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13', - capacity_requests: 6_000, database_pool_max: zone.startsWith('asia-') ? 16 : 10, image, - initially_enabled: false, - connection_hard_cap: 3_000, connection_unobserved_bound: 60 - }])) - -test('accepts the exact production topology only after it is durably committed', () => { - const existing = { - 'production-gce-c26': { hostname: 'c26', image: 'existing' }, - ...productionCells() - } - const result = prepareRelayAsiaTopologyInput({ existingCells: existing, - existingAdditionalRegions: additionalRegions, environment: 'production', - cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image }) - assert.equal(result.relay_gce_cells, existing) - assert.equal(result.relay_gce_additional_region_subnetwork_cidrs, additionalRegions) - assert.deepEqual(existing['production-gce-c27'], { - hostname: 'c27', region: 'asia-east2', zone: 'asia-east2-a', - machine_type: 'e2-standard-4', boot_disk_gb: 30, - boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13', - capacity_requests: 6_000, database_pool_max: 16, image, initially_enabled: false, - connection_hard_cap: 3_000, connection_unobserved_bound: 60 - }) -}) - -test('accepts the additive C30 wave without re-planning the launch cells', () => { - const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(), - existingAdditionalRegions: additionalRegions, environment: 'production', - cellIds: 'production-gce-c30', image }) - assert.equal(result.relay_gce_cells['production-gce-c30'].zone, 'asia-east2-a') -}) - -test('accepts the additive C31 wave in the next zone of the rotation', () => { - const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(), - existingAdditionalRegions: additionalRegions, environment: 'production', - cellIds: 'production-gce-c31', image }) - assert.equal(result.relay_gce_cells['production-gce-c31'].zone, 'asia-east2-b') -}) - -test('accepts the additive C34 spare wave in asia-east2-c at the Asia pool', () => { - const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(), - existingAdditionalRegions: additionalRegions, environment: 'production', - cellIds: 'production-gce-c34', image }) - assert.equal(result.relay_gce_cells['production-gce-c34'].zone, 'asia-east2-c') - assert.equal(result.relay_gce_cells['production-gce-c34'].database_pool_max, 16) -}) - -test('accepts the additive C35 spare wave in asia-east2-a at the Asia pool', () => { - const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(), - existingAdditionalRegions: additionalRegions, environment: 'production', - cellIds: 'production-gce-c35', image }) - assert.equal(result.relay_gce_cells['production-gce-c35'].zone, 'asia-east2-a') - assert.equal(result.relay_gce_cells['production-gce-c35'].database_pool_max, 16) -}) - -test('accepts the additive US C32+C33 wave at the default pool only', () => { - const cellIds = 'production-gce-c32,production-gce-c33' - for (const [cellId, zone] of [ - ['production-gce-c32', 'us-central1-a'], ['production-gce-c33', 'us-central1-b'] - ]) { - const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(), - existingAdditionalRegions: additionalRegions, environment: 'production', cellIds, image }) - assert.equal(result.relay_gce_cells[cellId].zone, zone) - assert.equal(result.relay_gce_cells[cellId].region, 'us-central1') - // Copying the Asia pool onto a US cell is drift, not the reviewed shape. - const asiaPool = productionCells() - asiaPool[cellId].database_pool_max = 16 - assert.throws(() => prepareRelayAsiaTopologyInput({ existingCells: asiaPool, - existingAdditionalRegions: additionalRegions, environment: 'production', cellIds, image - }), /differs from the reviewed topology/) - const asiaRegion = productionCells() - asiaRegion[cellId].region = 'asia-east2' - assert.throws(() => prepareRelayAsiaTopologyInput({ existingCells: asiaRegion, - existingAdditionalRegions: additionalRegions, environment: 'production', cellIds, image - }), /differs from the reviewed topology/) - } -}) - -// Reads the committed file so a reviewed-shape constant cannot drift from what the plan reads. -test('matches every committed production Asia cell entry', () => { - const tfvars = readFileSync( - new URL('../../infra/terraform/environments/production.tfvars', import.meta.url), - 'utf8' - ) - const committed = {} - for (const cellId of Object.keys(productionCells())) { - const start = tfvars.indexOf(` "${cellId}" = {`) - assert.notEqual(start, -1, `${cellId} is not committed`) - const body = tfvars.slice(start, tfvars.indexOf('\n }', start)) - const cell = {} - for (const [, key, raw] of body.matchAll(/^\s+([a-z_]+)\s+=\s+("[^"]*"|\S+)/gm)) { - cell[key] = raw.startsWith('"') ? raw.slice(1, -1) - : raw === 'true' || raw === 'false' ? raw === 'true' : Number(raw) - } - committed[cellId] = cell - } - // Each wave is pinned on its own: C30-C33 launch on the director's digest, not C27's. - for (const wave of [ - 'production-gce-c27,production-gce-c28,production-gce-c29', - 'production-gce-c30', - 'production-gce-c31', - 'production-gce-c32,production-gce-c33', - 'production-gce-c34', - 'production-gce-c35' - ]) { - const committedImage = committed[wave.split(',')[0]].image - assert.doesNotThrow(() => prepareRelayAsiaTopologyInput({ - existingCells: committed, existingAdditionalRegions: additionalRegions, - environment: 'production', cellIds: wave, image: committedImage - }), wave) - } - assert.throws(() => prepareRelayAsiaTopologyInput({ - existingCells: committed, existingAdditionalRegions: additionalRegions, - environment: 'production', cellIds: 'production-gce-c30', - image - }), /differs from the reviewed topology/) - // The US cells and the C34 spare launch on the newest cell digest, the one C31 launched on. - for (const cellId of ['production-gce-c32', 'production-gce-c33', 'production-gce-c34']) { - assert.equal(committed[cellId].image, committed['production-gce-c31'].image, cellId) - } -}) - -test('accepts the one exact committed staging Asia cell', () => { - const stagingImage = image.replace('onorca-cloud/', 'onorca-cloud-staging/') - const stagingCell = { - hostname: 'c4', region: 'asia-east2', zone: 'asia-east2-a', - machine_type: 'e2-standard-4', boot_disk_gb: 30, - boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13', - capacity_requests: 6_000, database_pool_max: 10, image: stagingImage, - initially_enabled: false, connection_hard_cap: 3_000, - connection_unobserved_bound: 60 - } - const result = prepareRelayAsiaTopologyInput({ - existingCells: { 'staging-gce-c3': { hostname: 'c3' }, 'staging-gce-c4': stagingCell }, - existingAdditionalRegions: additionalRegions, - environment: 'staging', - cellIds: 'staging-gce-c4', - image: stagingImage - }) - assert.equal(result.relay_gce_cells['staging-gce-c4'].zone, 'asia-east2-a') - assert.equal(result.relay_gce_cells['staging-gce-c4'].image, stagingImage) -}) - -test('rejects an uncommitted subnet or cell, partial wave, wrong image, and drift', () => { - for (const cellIds of [ - 'production-gce-c27', - 'production-gce-c27,production-gce-c30', - 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30', - 'production-gce-c30,production-gce-c30', - 'production-gce-c30,production-gce-c31', - 'production-gce-c32', - 'production-gce-c33', - 'production-gce-c33,production-gce-c34', - 'production-gce-c34,production-gce-c35', - 'production-gce-c36' - ]) { - assert.throws(() => prepareRelayAsiaTopologyInput({ - existingCells: productionCells(), existingAdditionalRegions: additionalRegions, - environment: 'production', cellIds, image - }), /cell IDs/, cellIds) - } - const poolDrift = productionCells() - poolDrift['production-gce-c30'].database_pool_max = 10 - assert.throws(() => prepareRelayAsiaTopologyInput({ - existingCells: poolDrift, existingAdditionalRegions: additionalRegions, - environment: 'production', cellIds: 'production-gce-c30', image - }), /differs from the reviewed topology/) - assert.throws(() => prepareRelayAsiaTopologyInput({ - existingCells: productionCells(), existingAdditionalRegions: additionalRegions, - environment: 'production', - cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', - image: image.replace('onorca-cloud/', 'other-project/') - }), /environment Relay image/) - assert.throws(() => prepareRelayAsiaTopologyInput({ - existingCells: productionCells(), existingAdditionalRegions: {}, environment: 'production', - cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image - }), /subnet must be committed/) - const missing = productionCells() - delete missing['production-gce-c29'] - assert.throws(() => prepareRelayAsiaTopologyInput({ - existingCells: missing, existingAdditionalRegions: additionalRegions, environment: 'production', - cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image - }), /cells must be committed/) - assert.throws(() => prepareRelayAsiaTopologyInput({ - existingCells: { ...productionCells(), 'production-gce-c27': {} }, - existingAdditionalRegions: additionalRegions, environment: 'production', - cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image - }), /differs from the reviewed topology/) -}) diff --git a/cloud/dev/scripts/prepare-relay-capacity-canary.mjs b/cloud/dev/scripts/prepare-relay-capacity-canary.mjs deleted file mode 100644 index 7bb7574d302..00000000000 --- a/cloud/dev/scripts/prepare-relay-capacity-canary.mjs +++ /dev/null @@ -1,183 +0,0 @@ -import { pathToFileURL } from 'node:url' -import { - applyExactAdmissionSelector, - inspectAdmissionSelector, - membershipWithStates, - selectorCellState -} from './relay-admission-selector.mjs' - -function parseArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of ['director-origin', 'cell-id', 'mode']) { - if (!values[key]) throw new Error(`missing --${key}`) - } - const origin = new URL(values['director-origin']) - if (origin.protocol !== 'https:' || origin.origin !== values['director-origin']) { - throw new Error('--director-origin must be a canonical HTTPS origin') - } - if (!['isolate', 'activate', 'restore-fallback', 'restore'].includes(values.mode)) { - throw new Error('--mode must be isolate, activate, restore-fallback, or restore') - } - const cellOrigin = values['cell-origin'] ? new URL(values['cell-origin']) : null - if ( - values.mode === 'isolate' && - (!cellOrigin || cellOrigin.protocol !== 'https:' || cellOrigin.origin !== values['cell-origin']) - ) { - throw new Error('--cell-origin must be a canonical HTTPS origin for isolate mode') - } - const restoreGeneralCellIds = values['general-cell-ids']?.split(',').filter(Boolean) ?? [] - if ( - ['restore-fallback', 'restore'].includes(values.mode) && - restoreGeneralCellIds.length === 0 - ) { - throw new Error('--general-cell-ids is required for restore modes') - } - if (values.mode === 'restore' && !restoreGeneralCellIds.includes(values['cell-id'])) { - throw new Error('--general-cell-ids must include the canary for restore mode') - } - if (values.mode === 'restore-fallback' && restoreGeneralCellIds.includes(values['cell-id'])) { - throw new Error('--general-cell-ids cannot include the canary for fallback restore') - } - if (new Set(restoreGeneralCellIds).size !== restoreGeneralCellIds.length) { - throw new Error('--general-cell-ids must be distinct') - } - return { - directorOrigin: origin.origin, - cellOrigin: cellOrigin?.origin, - cellId: values['cell-id'], - mode: values.mode, - restoreGeneralCellIds - } -} - -async function responseJson(response, label) { - const body = await response.json().catch(() => ({})) - if (!response.ok) throw new Error(`${label} returned ${response.status}`) - return body -} - -function sameMembership(left, right) { - return JSON.stringify(left) === JSON.stringify(right) -} - -async function legacyCellState(post, cellId) { - const result = await post('/v1/admin/cell-status', { v: 1, cellId }) - if (result.status?.cellId !== cellId) throw new Error('legacy admission status is invalid') - const state = result.status.admissionState - if (!['existing-only', 'migration-only', 'general'].includes(state)) { - throw new Error('legacy admission status is invalid') - } - return state -} - -async function applyLegacyStates(post, before, states, order) { - const expected = membershipWithStates(before.selector, states) - let changed = false - for (const cellId of order) { - const desired = states[cellId] - const current = await legacyCellState(post, cellId) - if (current === 'existing-only' && desired !== current) { - throw new Error(`legacy admission cannot re-enable existing-only cell ${cellId}`) - } - if (current === desired) continue - let cause - try { - await post('/v1/admin/cell-state', { v: 1, cellId, state: desired }) - } catch (error) { - cause = error - } - if ((await legacyCellState(post, cellId)) !== desired) { - const detail = cause instanceof Error ? `: ${cause.message}` : '' - throw new Error(`legacy admission did not commit ${cellId} exactly${detail}`, { cause }) - } - changed = true - } - const verified = await inspectAdmissionSelector(post) - if (verified.selector.generation !== 0 || - !sameMembership(verified.selector.membership, expected)) { - throw new Error('legacy admission membership changed unexpectedly') - } - return { changed, selector: verified.selector } -} - -export async function prepareCapacityCanary(config, overrides = {}) { - const fetchImpl = overrides.fetch ?? fetch - const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN - if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') - const postAt = async (origin, path, body) => - await responseJson( - await fetchImpl(`${origin}${path}`, { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(30_000) - }), - path - ) - const post = async (path, body) => await postAt(config.directorOrigin, path, body) - const before = await inspectAdmissionSelector(post) - const state = selectorCellState(before.selector, config.cellId) - if (state === 'existing-only' && config.mode !== 'restore-fallback') { - throw new Error('capacity canary cannot restore existing-only admission') - } - const states = - config.mode === 'isolate' - ? { [config.cellId]: 'migration-only' } - : config.mode === 'activate' - ? Object.fromEntries([ - ...before.selector.membership.general.map((cellId) => [ - cellId, - cellId === config.cellId ? 'general' : 'migration-only' - ]), - [config.cellId, 'general'] - ]) - : Object.fromEntries([ - ...config.restoreGeneralCellIds.map((cellId) => [cellId, 'general']), - ...(config.mode === 'restore-fallback' - ? [[config.cellId, state === 'existing-only' ? 'existing-only' : 'migration-only']] - : []) - ]) - const membership = membershipWithStates(before.selector, states) - const legacyOrder = - config.mode === 'activate' - ? [config.cellId, ...before.selector.membership.general.filter((id) => id !== config.cellId)] - : config.mode === 'restore-fallback' - ? [...config.restoreGeneralCellIds, config.cellId] - : Object.keys(states) - const result = before.selector.generation === 0 - ? await applyLegacyStates(post, before, states, legacyOrder) - : sameMembership(membership, before.selector.membership) - ? { changed: false, selector: before.selector } - : await applyExactAdmissionSelector(post, membership, { - expectedCurrentSelector: before.selector - }) - if (config.mode === 'isolate') { - await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 }) - } - return { - changed: result.changed, - generation: result.selector.generation, - ...(config.mode === 'isolate' ? { drained: true } : {}) - } -} - -export async function main(argv = process.argv.slice(2)) { - const config = parseArguments(argv) - const result = await prepareCapacityCanary(config) - process.stdout.write( - `${JSON.stringify({ event: 'relay_capacity_canary_admission', cellId: config.cellId, mode: config.mode, ...result })}\n` - ) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/prepare-relay-capacity-canary.test.mjs b/cloud/dev/scripts/prepare-relay-capacity-canary.test.mjs deleted file mode 100644 index d8e752aa500..00000000000 --- a/cloud/dev/scripts/prepare-relay-capacity-canary.test.mjs +++ /dev/null @@ -1,300 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import { test } from 'node:test' -import { relayWorkflowUrl } from './relay-repository.mjs' -import { prepareCapacityCanary } from './prepare-relay-capacity-canary.mjs' - -function harness(initialState, options = {}) { - const { - generation = 4, - ambiguousCellState = false, - rejectCellState = false, - fallbackState = 'general', - extraGeneralCellIds = [] - } = options - let selector = { - generation, - attemptId: 'initial', - membership: { - existingOnly: [ - 'staging-gce-c1', - ...(initialState === 'existing-only' ? ['staging-gce-c3'] : []) - ], - migrationOnly: [ - ...(fallbackState === 'migration-only' ? ['staging-gce-c2'] : []), - ...(initialState === 'migration-only' ? ['staging-gce-c3'] : []) - ], - general: [ - ...extraGeneralCellIds, - ...(fallbackState === 'general' ? ['staging-gce-c2'] : []), - ...(initialState === 'general' ? ['staging-gce-c3'] : []) - ].sort() - } - } - let intent = null - let applies = 0 - let drains = 0 - const cellStateChanges = [] - const fetch = async (url, options) => { - const path = new URL(url).pathname - const body = JSON.parse(options.body) - if (path === '/v1/admin/drain') { - assert.deepEqual(body, { v: 1, graceMs: 0 }) - drains++ - return Response.json({ ok: true }) - } - if (path === '/v1/admin/cell-status') { - const state = selector.membership.existingOnly.includes(body.cellId) - ? 'existing-only' - : selector.membership.migrationOnly.includes(body.cellId) - ? 'migration-only' - : 'general' - return Response.json({ status: { cellId: body.cellId, admissionState: state } }) - } - if (path === '/v1/admin/cell-state') { - assert.equal(selector.generation, 0) - if (rejectCellState) { - return Response.json({ error: 'invalid_token' }, { status: 401 }) - } - const keys = { - 'existing-only': 'existingOnly', - 'migration-only': 'migrationOnly', - general: 'general' - } - for (const cells of Object.values(selector.membership)) { - const index = cells.indexOf(body.cellId) - if (index !== -1) cells.splice(index, 1) - } - selector.membership[keys[body.state]].push(body.cellId) - for (const cells of Object.values(selector.membership)) cells.sort() - cellStateChanges.push({ cellId: body.cellId, state: body.state }) - if (ambiguousCellState) throw new Error('response lost') - return Response.json({ ok: true }) - } - if (path.endsWith('/status')) return Response.json({ selector, intent }) - applies++ - selector = { - generation: body.expectedGeneration + 1, - attemptId: body.attemptId, - membership: body.membership - } - intent = { - attemptId: body.attemptId, - expectedGeneration: body.expectedGeneration, - intendedGeneration: selector.generation, - membership: selector.membership, - state: 'committed' - } - return Response.json({ changed: true, selector }) - } - return { - fetch, - selector: () => selector, - applies: () => applies, - drains: () => drains, - cellStateChanges - } -} - -const config = { - directorOrigin: 'https://relay.example.com', - cellOrigin: 'https://c3.relay.example.com', - cellId: 'staging-gce-c3', - mode: 'isolate', - restoreGeneralCellIds: [] -} - -test('isolates a general canary as migration-only', async () => { - const testHarness = harness('general') - assert.deepEqual( - await prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }), - { changed: true, generation: 5, drained: true } - ) - assert.deepEqual(testHarness.selector().membership.migrationOnly, [config.cellId]) - assert.equal(testHarness.applies(), 1) - assert.equal(testHarness.drains(), 1) -}) - -test('activates the canary as the only general cell', async () => { - const testHarness = harness('migration-only') - assert.deepEqual( - await prepareCapacityCanary( - { ...config, mode: 'activate' }, - { fetch: testHarness.fetch, token: 'masked' } - ), - { changed: true, generation: 5 } - ) - assert.deepEqual(testHarness.selector().membership, { - existingOnly: ['staging-gce-c1'], - migrationOnly: ['staging-gce-c2'], - general: [config.cellId] - }) - assert.equal(testHarness.applies(), 1) -}) - -test('restores the reviewed staging general membership', async () => { - const testHarness = harness('migration-only') - assert.deepEqual( - await prepareCapacityCanary( - { - ...config, - mode: 'restore', - restoreGeneralCellIds: ['staging-gce-c2', config.cellId] - }, - { fetch: testHarness.fetch, token: 'masked' } - ), - { changed: true, generation: 5 } - ) - assert.deepEqual(testHarness.selector().membership, { - existingOnly: ['staging-gce-c1'], - migrationOnly: [], - general: ['staging-gce-c2', config.cellId] - }) -}) - -test('restores the fallback without promoting a possibly drained canary', async () => { - const testHarness = harness('general') - await prepareCapacityCanary( - { - ...config, - mode: 'restore-fallback', - restoreGeneralCellIds: ['staging-gce-c2'] - }, - { fetch: testHarness.fetch, token: 'masked' } - ) - assert.deepEqual(testHarness.selector().membership, { - existingOnly: ['staging-gce-c1'], - migrationOnly: [config.cellId], - general: ['staging-gce-c2'] - }) -}) - -test('restores the fallback while preserving an irreversible canary', async () => { - const testHarness = harness('existing-only') - await prepareCapacityCanary( - { - ...config, - mode: 'restore-fallback', - restoreGeneralCellIds: ['staging-gce-c2'] - }, - { fetch: testHarness.fetch, token: 'masked' } - ) - assert.deepEqual(testHarness.selector().membership, { - existingOnly: ['staging-gce-c1', config.cellId], - migrationOnly: [], - general: ['staging-gce-c2'] - }) -}) - -test('uses exact legacy admission writes before the selector boundary', async () => { - const testHarness = harness('general', { generation: 0 }) - assert.deepEqual( - await prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }), - { changed: true, generation: 0, drained: true } - ) - assert.deepEqual(testHarness.cellStateChanges, [ - { cellId: config.cellId, state: 'migration-only' } - ]) - assert.equal(testHarness.drains(), 1) -}) - -test('promotes a legacy canary before demoting its fallback', async () => { - const testHarness = harness('migration-only', { generation: 0 }) - await prepareCapacityCanary( - { ...config, mode: 'activate' }, - { fetch: testHarness.fetch, token: 'masked' } - ) - assert.deepEqual(testHarness.cellStateChanges, [ - { cellId: config.cellId, state: 'general' }, - { cellId: 'staging-gce-c2', state: 'migration-only' } - ]) -}) - -test('makes the canary sole general with the live legacy membership shape', async () => { - const extraGeneralCellIds = ['combined', 'staging-c1', 'staging-c2'] - const testHarness = harness('migration-only', { generation: 0, extraGeneralCellIds }) - const activate = { ...config, mode: 'activate' } - await prepareCapacityCanary(activate, { fetch: testHarness.fetch, token: 'masked' }) - assert.deepEqual(testHarness.cellStateChanges, [ - { cellId: config.cellId, state: 'general' }, - { cellId: 'combined', state: 'migration-only' }, - { cellId: 'staging-c1', state: 'migration-only' }, - { cellId: 'staging-c2', state: 'migration-only' }, - { cellId: 'staging-gce-c2', state: 'migration-only' } - ]) -}) - -test('restores a legacy fallback before demoting the target', async () => { - const testHarness = harness('general', { - generation: 0, - fallbackState: 'migration-only' - }) - await prepareCapacityCanary( - { - ...config, - mode: 'restore-fallback', - restoreGeneralCellIds: ['staging-gce-c2'] - }, - { fetch: testHarness.fetch, token: 'masked' } - ) - assert.deepEqual(testHarness.cellStateChanges, [ - { cellId: 'staging-gce-c2', state: 'general' }, - { cellId: config.cellId, state: 'migration-only' } - ]) -}) - -test('keeps an already restored legacy fallback unchanged', async () => { - const testHarness = harness('migration-only', { generation: 0 }) - assert.deepEqual( - await prepareCapacityCanary( - { - ...config, - mode: 'restore-fallback', - restoreGeneralCellIds: ['staging-gce-c2'] - }, - { fetch: testHarness.fetch, token: 'masked' } - ), - { changed: false, generation: 0 } - ) - assert.deepEqual(testHarness.cellStateChanges, []) - assert.deepEqual(testHarness.selector().membership, { - existingOnly: ['staging-gce-c1'], - migrationOnly: [config.cellId], - general: ['staging-gce-c2'] - }) -}) - -test('recovers an ambiguous legacy admission response by exact readback', async () => { - const testHarness = harness('general', { generation: 0, ambiguousCellState: true }) - await assert.doesNotReject( - prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }) - ) - assert.equal(testHarness.drains(), 1) -}) - -test('reports a rejected legacy admission write without draining', async () => { - const testHarness = harness('general', { generation: 0, rejectCellState: true }) - const operation = prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }) - await assert.rejects(operation, /cell-state returned 401/) - assert.equal(testHarness.drains(), 0) -}) - -test('the staging workflow supplies every required capacity transition argument', () => { - const workflow = readFileSync( - relayWorkflowUrl('prove-relay-staging-capacity.yml'), - 'utf8' - ) - const verifyCalls = workflow.match( - /node dev\/scripts\/verify-relay-capacity-transition\.mjs[\s\S]*?(?=\n\s*\n|\n\s*- name:)/g - ) - assert.ok(verifyCalls?.length >= 5) - for (const call of verifyCalls) { - for (const flag of ['--cell-origin', '--heartbeat', '--admission', '--draining', '--activity']) { - assert.match(call, new RegExp(flag)) - } - } - const isolate = workflow.match( - /node dev\/scripts\/prepare-relay-capacity-canary\.mjs[\s\S]*?--mode isolate/ - )?.[0] - assert.match(isolate, /--cell-origin/) -}) diff --git a/cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs b/cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs deleted file mode 100644 index fb13cecc59f..00000000000 --- a/cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs +++ /dev/null @@ -1,196 +0,0 @@ -import { pathToFileURL } from 'node:url' -import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' -import { - applyExactAdmissionSelector, - inspectAdmissionSelector, - membershipWithStates, - selectorCellState -} from './relay-admission-selector.mjs' -import { SAME_CAP_CELLS } from './relay-production-same-cap-wave.mjs' - -const DIRECTOR_ORIGIN = 'https://relay.onorca.dev' -export const PRODUCTION_CAPACITY_CELL_IDS = [ - 'production-gce-c7', - 'production-gce-c8', - 'production-gce-c9', - 'production-gce-c10', - 'production-gce-c13', - 'production-gce-c14', - 'production-gce-c15', - 'production-gce-c16', - 'production-gce-c19', - 'production-gce-c20', - 'production-gce-c21', - 'production-gce-c22', - 'production-gce-c23', - 'production-gce-c24', - 'production-gce-c25', - 'production-gce-c26' -] - -function cellOrigin(cellId) { - return `https://${cellId.slice('production-gce-'.length)}.relay.onorca.dev` -} - -// The same-cap roll covers the Asia cells the US-only capacity rollout never touches. -const APPROVED_CELL_LISTS = { 'same-cap': SAME_CAP_CELLS } - -// Matches the cell's own cap on /v1/admin/drain. -const MAX_PACE_WINDOW_MS = 20 * 60 * 1_000 -// Every paced image before the cap rose rejects a longer window with the same 400 an unpaced -// image gives, and an unpaced drain is the opposite of what a slower window asked for. -const LEGACY_MAX_PACE_WINDOW_MS = 5 * 60 * 1_000 - -export function parseProductionCapacityCellArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - if (!['pace-check', 'isolate', 'drain', 'activate'].includes(values.mode)) { - throw new Error('--mode must be pace-check, isolate, drain, or activate') - } - const approvedList = values['approved-cells'] - if (approvedList !== undefined && !APPROVED_CELL_LISTS[approvedList]) { - throw new Error('--approved-cells is not a known allowlist') - } - const approvedCellIds = approvedList === undefined - ? PRODUCTION_CAPACITY_CELL_IDS - : APPROVED_CELL_LISTS[approvedList] - const cellId = values['cell-id'] - if (!approvedCellIds.includes(cellId)) { - throw new Error('production capacity target is not approved') - } - const expectedCellOrigin = cellOrigin(cellId) - if ( - values['director-origin'] !== DIRECTOR_ORIGIN || - values['cell-origin'] !== expectedCellOrigin - ) { - throw new Error('production capacity target origin is not exact') - } - const paceWindowMs = values['pace-window-ms'] === undefined - ? 0 - : Number(values['pace-window-ms']) - if ( - !Number.isSafeInteger(paceWindowMs) || - paceWindowMs < 0 || - paceWindowMs > MAX_PACE_WINDOW_MS - ) { - throw new Error('--pace-window-ms must be an integer between 0 and 1200000') - } - return { - directorOrigin: DIRECTOR_ORIGIN, - cellOrigin: expectedCellOrigin, - cellId, - mode: values.mode, - paceWindowMs - } -} - -async function responseJson(response, label) { - const body = await response.json().catch(() => ({})) - if (!response.ok) throw new Error(`${label} returned ${response.status}`) - return body -} - -// Read-only, and run before the isolate: an image that does not advertise its cap has the -// legacy one, so a slower pace stops here with the cell untouched instead of isolated. -async function checkDrainPace(fetchImpl, config, wait) { - const health = await responseJson( - await fetchAdminOnceMore(fetchImpl, `${config.cellOrigin}/health`, {}, { wait }), - 'cell health' - ) - if (health.ok !== true) throw new Error('cell health is not ok') - const maxPaceWindowMs = Number.isSafeInteger(health.drainPaceWindowMaxMs) - ? health.drainPaceWindowMaxMs - : LEGACY_MAX_PACE_WINDOW_MS - if (config.paceWindowMs > maxPaceWindowMs) { - throw new Error( - `cell accepts drain paces up to ${maxPaceWindowMs} ms, not ${config.paceWindowMs} ms; ` + - 'it was not isolated' - ) - } - return { paceWindowMs: config.paceWindowMs, maxPaceWindowMs } -} - -export async function prepareProductionCapacityCell(config, overrides = {}) { - const fetchImpl = overrides.fetch ?? fetch - if (config.mode === 'pace-check') return await checkDrainPace(fetchImpl, config, overrides.wait) - const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN - if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') - const postRaw = async (origin, path, body) => - await fetchAdminOnceMore( - fetchImpl, - `${origin}${path}`, - { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify(body) - }, - { wait: overrides.wait } - ) - const postAt = async (origin, path, body) => - await responseJson(await postRaw(origin, path, body), path) - const post = async (path, body) => await postAt(config.directorOrigin, path, body) - if (config.mode === 'drain') { - const paceWindowMs = config.paceWindowMs ?? 0 - if (paceWindowMs > 0) { - const paced = await postRaw(config.cellOrigin, '/v1/admin/drain', { - v: 1, - graceMs: 0, - paceWindowMs - }) - if (paced.ok) { - await paced.json().catch(() => ({})) - return { changed: false, drained: true, paceWindowMs } - } - // A cell still on an image without paced drain rejects the unknown field outright. - // An unpaced drain is the behaviour that cell already has, so fall back to it. - if (paced.status !== 400) throw new Error(`/v1/admin/drain returned ${paced.status}`) - await paced.json().catch(() => ({})) - if (paceWindowMs > LEGACY_MAX_PACE_WINDOW_MS) { - throw new Error( - `cell rejected a ${paceWindowMs} ms drain pace; refusing to drain faster than asked` - ) - } - } - await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 }) - return { changed: false, drained: true, paceWindowMs: 0 } - } - const before = await inspectAdmissionSelector(post) - const state = selectorCellState(before.selector, config.cellId) - if (state === 'existing-only') throw new Error('production capacity target is irreversible') - const desiredState = config.mode === 'isolate' ? 'migration-only' : 'general' - const membership = membershipWithStates(before.selector, { [config.cellId]: desiredState }) - const result = await applyExactAdmissionSelector(post, membership, { - expectedCurrentSelector: before.selector, - // The stamp that tells the director this cell is parked for a restart rather - // than held back as capacity, so it may re-place the hosts still on it. The - // activate branch omits it, and moving to 'general' clears it in the same - // statement that writes the state. - ...(config.mode === 'isolate' ? { rollIsolatedCells: [config.cellId] } : {}) - }) - return { - changed: result.changed, - generation: result.selector.generation, - admissionState: desiredState, - rollIsolated: config.mode === 'isolate' - } -} - -export async function main(argv = process.argv.slice(2)) { - const config = parseProductionCapacityCellArguments(argv) - const result = await prepareProductionCapacityCell(config) - process.stdout.write( - `${JSON.stringify({ event: 'relay_production_capacity_canary', cellId: config.cellId, mode: config.mode, ...result })}\n` - ) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs b/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs deleted file mode 100644 index 80866fef433..00000000000 --- a/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs +++ /dev/null @@ -1,391 +0,0 @@ -import assert from 'node:assert/strict' -import { describe, it } from 'node:test' -import { - parseProductionCapacityCellArguments, - prepareProductionCapacityCell -} from './prepare-relay-production-capacity-canary.mjs' - -const config = { - directorOrigin: 'https://relay.onorca.dev', - cellOrigin: 'https://c26.relay.onorca.dev', - cellId: 'production-gce-c26' -} - -const membership = { - existingOnly: ['production-gce-c1'], - migrationOnly: ['production-gce-c17'], - general: ['production-gce-c25', 'production-gce-c26'] -} - -function response(body, status = 200) { - return new Response(JSON.stringify(body), { - status, - headers: { 'content-type': 'application/json' } - }) -} - -function canaryFetch() { - let selector = { generation: 20, attemptId: null, membership } - const calls = [] - const fetch = async (url, init) => { - const path = new URL(url).pathname - const body = JSON.parse(init.body) - calls.push({ path, body }) - if (path === '/v1/admin/admission-selector/status') { - return response({ - v: 1, - selector, - intent: body.attemptId - ? { - attemptId: body.attemptId, - state: 'committed', - expectedGeneration: selector.generation - 1, - intendedGeneration: selector.generation, - membership: selector.membership - } - : null - }) - } - if (path === '/v1/admin/admission-selector/apply') { - selector = { - generation: selector.generation + 1, - attemptId: body.attemptId, - membership: body.membership - } - return response({ v: 1, changed: true, selector }) - } - if (path === '/v1/admin/drain') return response({ v: 1, draining: true }) - throw new Error(`unexpected ${path}`) - } - return { calls, fetch, selector: () => selector } -} - -describe('production Relay capacity cell admission', () => { - it('allows only the serving rollout cells', () => { - assert.deepEqual(parseProductionCapacityCellArguments([ - '--director-origin', 'https://relay.onorca.dev', - '--cell-origin', 'https://c7.relay.onorca.dev', - '--cell-id', 'production-gce-c7', - '--mode', 'isolate' - ]), { - directorOrigin: 'https://relay.onorca.dev', - cellOrigin: 'https://c7.relay.onorca.dev', - cellId: 'production-gce-c7', - mode: 'isolate', - paceWindowMs: 0 - }) - assert.throws(() => parseProductionCapacityCellArguments([ - '--director-origin', 'https://relay.onorca.dev', - '--cell-origin', 'https://c17.relay.onorca.dev', - '--cell-id', 'production-gce-c17', - '--mode', 'isolate' - ]), /not approved/) - assert.throws(() => parseProductionCapacityCellArguments([ - '--director-origin', 'https://relay.onorca.dev', - '--cell-origin', 'https://c8.relay.onorca.dev', - '--cell-id', 'production-gce-c7', - '--mode', 'isolate' - ]), /origin is not exact/) - assert.throws(() => parseProductionCapacityCellArguments([ - '--director-origin', 'https://relay.onorca.dev', - '--cell-origin', 'https://c27.relay.onorca.dev', - '--cell-id', 'production-gce-c27', - '--mode', 'isolate' - ]), /not approved/) - }) - - it('admits the same-cap Asia and migration-only cells only under the same-cap allowlist', () => { - for (const cellId of [ - 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30', - 'production-gce-c31', 'production-gce-c34', 'production-gce-c35', - // Migration-only canaries: the US-only capacity rollout never touches them either. - 'production-gce-c17', 'production-gce-c18', - // US cells at the Asia shape: the 1,000-cap capacity rollout never touches them. - 'production-gce-c32', 'production-gce-c33' - ]) { - const hostname = cellId.slice('production-gce-'.length) - assert.deepEqual(parseProductionCapacityCellArguments([ - '--director-origin', 'https://relay.onorca.dev', - '--cell-origin', `https://${hostname}.relay.onorca.dev`, - '--cell-id', cellId, - '--approved-cells', 'same-cap', - '--mode', 'isolate' - ]), { - directorOrigin: 'https://relay.onorca.dev', - cellOrigin: `https://${hostname}.relay.onorca.dev`, - cellId, - mode: 'isolate', - paceWindowMs: 0 - }) - } - for (const cellId of ['production-gce-c12', 'production-gce-c36']) { - const hostname = cellId.slice('production-gce-'.length) - assert.throws(() => parseProductionCapacityCellArguments([ - '--director-origin', 'https://relay.onorca.dev', - '--cell-origin', `https://${hostname}.relay.onorca.dev`, - '--cell-id', cellId, - '--approved-cells', 'same-cap', - '--mode', 'isolate' - ]), /not approved/) - } - assert.throws(() => parseProductionCapacityCellArguments([ - '--director-origin', 'https://relay.onorca.dev', - '--cell-origin', 'https://c27.relay.onorca.dev', - '--cell-id', 'production-gce-c27', - '--approved-cells', 'every-cell', - '--mode', 'isolate' - ]), /not a known allowlist/) - }) - - it('isolates only the selected cell without depending on its runtime', async () => { - const fake = canaryFetch() - const result = await prepareProductionCapacityCell( - { ...config, mode: 'isolate' }, - { fetch: fake.fetch, token: 'token' } - ) - assert.equal(result.admissionState, 'migration-only') - assert.deepEqual(fake.selector().membership, { - existingOnly: ['production-gce-c1'], - migrationOnly: ['production-gce-c17', 'production-gce-c26'], - general: ['production-gce-c25'] - }) - assert.doesNotMatch(fake.calls.map(({ path }) => path).join(','), /\/v1\/admin\/drain/) - }) - - it('stamps the roll isolation on isolate and never on activate', async () => { - // The stamp is what lets the director tell a cell parked for a restart from - // an evacuation target or an Asia rollback, both of which must keep their - // hosts. Only this call site may send it. - const isolate = canaryFetch() - await prepareProductionCapacityCell( - { ...config, mode: 'isolate' }, - { fetch: isolate.fetch, token: 'token' } - ) - const isolateApply = isolate.calls.find( - ({ path }) => path === '/v1/admin/admission-selector/apply' - ) - assert.deepEqual(isolateApply.body.rollIsolatedCells, [config.cellId]) - - // Restore has to be a real apply, not the no-op an already-general cell - // takes, or the assertion below proves nothing. - isolate.calls.length = 0 - const restored = await prepareProductionCapacityCell( - { ...config, mode: 'activate' }, - { fetch: isolate.fetch, token: 'token' } - ) - assert.equal(restored.admissionState, 'general') - const restoreApply = isolate.calls.find( - ({ path }) => path === '/v1/admin/admission-selector/apply' - ) - assert.ok(restoreApply, 'restore must issue an apply') - assert.equal(restoreApply.body.rollIsolatedCells, undefined) - assert.ok(restoreApply.body.membership.general.includes(config.cellId)) - }) - - it('drains the selected cell independently after durable isolation', async () => { - const fake = canaryFetch() - const result = await prepareProductionCapacityCell( - { ...config, mode: 'drain' }, - { fetch: fake.fetch, token: 'token' } - ) - assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 }) - assert.deepEqual(fake.calls, [{ - path: '/v1/admin/drain', - body: { v: 1, graceMs: 0 } - }]) - }) - - it('paces the drain send when the roll asks for a window', async () => { - const fake = canaryFetch() - const result = await prepareProductionCapacityCell( - { ...config, mode: 'drain', paceWindowMs: 120_000 }, - { fetch: fake.fetch, token: 'token' } - ) - assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 120_000 }) - assert.deepEqual(fake.calls, [{ - path: '/v1/admin/drain', - body: { v: 1, graceMs: 0, paceWindowMs: 120_000 } - }]) - }) - - it('drains unpaced when the cell image rejects the pacing field', async () => { - const bodies = [] - const result = await prepareProductionCapacityCell( - { ...config, mode: 'drain', paceWindowMs: 120_000 }, - { - token: 'token', - wait: async () => {}, - fetch: async (url, init) => { - assert.equal(new URL(url).pathname, '/v1/admin/drain') - const body = JSON.parse(init.body) - bodies.push(body) - if (body.paceWindowMs !== undefined) return response({ error: 'invalid_request' }, 400) - return response({ v: 1, draining: true }) - } - } - ) - assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 }) - assert.deepEqual(bodies, [ - { v: 1, graceMs: 0, paceWindowMs: 120_000 }, - { v: 1, graceMs: 0 } - ]) - }) - - it('refuses to fall back to an unpaced drain when the cell rejects a slower window', async () => { - for (const paceWindowMs of [900_000, 1_200_000]) { - const bodies = [] - await assert.rejects( - prepareProductionCapacityCell( - { ...config, mode: 'drain', paceWindowMs }, - { - token: 'token', - wait: async () => {}, - fetch: async (_url, init) => { - bodies.push(JSON.parse(init.body)) - return response({ error: 'invalid_request' }, 400) - } - } - ), - /refusing to drain faster than asked/ - ) - assert.deepEqual(bodies, [{ v: 1, graceMs: 0, paceWindowMs }]) - } - }) - - it('stops a slower pace than an old-cap cell accepts before anything is isolated', async () => { - const check = async (health, paceWindowMs) => { - const requests = [] - const result = prepareProductionCapacityCell( - { ...config, mode: 'pace-check', paceWindowMs }, - { - wait: async () => {}, - fetch: async (url, init) => { - requests.push({ path: new URL(url).pathname, method: init.method ?? 'GET' }) - return response(health) - } - } - ) - return { result, requests } - } - // Every image before the cap rose answers /health without it. - const old = { ok: true, connectionCapacityProtocol: 2 } - for (const pace of [900_000, 1_200_000]) { - const { result, requests } = await check(old, pace) - await assert.rejects(result, /up to 300000 ms, not \d+ ms; it was not isolated/) - assert.deepEqual(requests, [{ path: '/health', method: 'GET' }]) - } - assert.deepEqual(await (await check(old, 300_000)).result, { - paceWindowMs: 300_000, maxPaceWindowMs: 300_000 - }) - const current = { ...old, drainPaceWindowMaxMs: 1_200_000 } - assert.deepEqual(await (await check(current, 1_200_000)).result, { - paceWindowMs: 1_200_000, maxPaceWindowMs: 1_200_000 - }) - await assert.rejects((await check({ ok: false }, 300_000)).result, /cell health is not ok/) - }) - - it('fails a paced drain that the cell rejects for any other reason', async () => { - await assert.rejects( - prepareProductionCapacityCell( - { ...config, mode: 'drain', paceWindowMs: 120_000 }, - { - token: 'token', - wait: async () => {}, - fetch: async () => response({ error: 'invalid_token' }, 401) - } - ), - /returned 401/ - ) - }) - - it('refuses a pacing window that is not a bounded integer', () => { - const argv = (value) => [ - '--director-origin', 'https://relay.onorca.dev', - '--cell-origin', 'https://c26.relay.onorca.dev', - '--cell-id', 'production-gce-c26', - '--mode', 'drain', - '--pace-window-ms', value - ] - for (const value of ['-1', '1200001', '1.5', 'soon']) { - assert.throws(() => parseProductionCapacityCellArguments(argv(value)), /pace-window-ms/) - } - for (const value of [300_000, 900_000, 1_200_000]) { - assert.equal(parseProductionCapacityCellArguments(argv(String(value))).paceWindowMs, value) - } - }) - - it('restores only the selected cell to general admission', async () => { - const fake = canaryFetch() - await prepareProductionCapacityCell( - { ...config, mode: 'isolate' }, - { fetch: fake.fetch, token: 'token' } - ) - const result = await prepareProductionCapacityCell( - { ...config, mode: 'activate' }, - { fetch: fake.fetch, token: 'token' } - ) - assert.equal(result.admissionState, 'general') - assert.deepEqual(fake.selector().membership, membership) - }) - - it('refuses an irreversible existing-only target', async () => { - const fetch = async () => response({ - v: 1, - selector: { - generation: 20, - attemptId: null, - membership: { - existingOnly: ['production-gce-c26'], - migrationOnly: ['production-gce-c17'], - general: ['production-gce-c25'] - } - }, - intent: null - }) - await assert.rejects( - prepareProductionCapacityCell( - { ...config, mode: 'isolate' }, - { fetch, token: 'token' } - ), - /irreversible/ - ) - }) - - it('retries a transient 503 on the cell drain endpoint', async () => { - let calls = 0 - const result = await prepareProductionCapacityCell( - { ...config, mode: 'drain' }, - { - token: 'token', - wait: async () => {}, - fetch: async (url) => { - assert.equal(new URL(url).pathname, '/v1/admin/drain') - calls += 1 - if (calls === 1) return response({ error: 'warming up' }, 503) - return response({ v: 1, draining: true }) - } - } - ) - assert.equal(calls, 2) - assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 }) - }) - - it('fails when both drain attempts return a transient 503', async () => { - let calls = 0 - await assert.rejects( - prepareProductionCapacityCell( - { ...config, mode: 'drain' }, - { - token: 'token', - wait: async () => {}, - fetch: async () => { - calls += 1 - return response({ error: 'warming up' }, 503) - } - } - ), - /returned 503/ - ) - assert.equal(calls, 2) - }) -}) diff --git a/cloud/dev/scripts/probe-relay-legacy-admission.mjs b/cloud/dev/scripts/probe-relay-legacy-admission.mjs deleted file mode 100644 index 3529c7d70ac..00000000000 --- a/cloud/dev/scripts/probe-relay-legacy-admission.mjs +++ /dev/null @@ -1,73 +0,0 @@ -import { randomBytes } from 'node:crypto' -import { pathToFileURL } from 'node:url' - -const WRONG_CELL = 4409 -const DRAINING = 4503 - -function once(socket, event, listener) { - if (typeof socket.once === 'function') { - socket.once(event, listener) - return - } - if (typeof socket.addEventListener !== 'function') { - throw new Error('WebSocket event API is unavailable') - } - socket.addEventListener(event, (value) => { - if (event === 'close') listener(value.code) - else if (event === 'error') listener(value.error ?? new Error(value.message)) - else listener() - }, { once: true }) -} - -export function parseLegacyAdmissionProbeArguments(argv) { - if (argv.length !== 2 || argv[0] !== '--cell-origin') throw new Error('invalid arguments') - const origin = new URL(argv[1]) - if (origin.protocol !== 'https:' || origin.origin !== argv[1]) { - throw new Error('--cell-origin must be a canonical HTTPS origin') - } - return { cellOrigin: origin.origin } -} - -export async function probeLegacyAdmission(config, overrides = {}) { - const Socket = overrides.WebSocket ?? globalThis.WebSocket - if (typeof Socket !== 'function') throw new Error('WebSocket is unavailable') - const random = overrides.randomBytes ?? randomBytes - const timeoutMs = overrides.timeoutMs ?? 15_000 - const hostId = random(12).toString('base64url') - const credential = random(32).toString('base64url') - const url = `${config.cellOrigin.replace('https://', 'wss://')}/v1/connect/${hostId}` - await new Promise((resolve, reject) => { - const socket = new Socket(url) - const timer = setTimeout(() => { - if (typeof socket.terminate === 'function') socket.terminate() - else socket.close() - reject(new Error('legacy admission probe timed out')) - }, timeoutMs) - once(socket, 'open', () => { - socket.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential })) - }) - once(socket, 'close', (code) => { - clearTimeout(timer) - if (code === WRONG_CELL) resolve() - else if (code === DRAINING) reject(new Error('legacy cell is draining')) - else reject(new Error(`legacy admission probe closed with ${code}`)) - }) - once(socket, 'error', (error) => { - clearTimeout(timer) - reject(new Error(`legacy admission probe failed: ${error.message}`)) - }) - }) - return { accepting: true } -} - -export async function main(argv = process.argv.slice(2)) { - const result = await probeLegacyAdmission(parseLegacyAdmissionProbeArguments(argv)) - process.stdout.write(`${JSON.stringify({ event: 'relay_legacy_admission_verified', ...result })}\n`) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/probe-relay-legacy-admission.test.mjs b/cloud/dev/scripts/probe-relay-legacy-admission.test.mjs deleted file mode 100644 index 7c0d078b4f0..00000000000 --- a/cloud/dev/scripts/probe-relay-legacy-admission.test.mjs +++ /dev/null @@ -1,89 +0,0 @@ -import assert from 'node:assert/strict' -import { EventEmitter } from 'node:events' -import { test } from 'node:test' -import { - parseLegacyAdmissionProbeArguments, - probeLegacyAdmission -} from './probe-relay-legacy-admission.mjs' - -function socketClosingWith(code, observed) { - return class extends EventEmitter { - constructor(url) { - super() - observed.url = url - queueMicrotask(() => this.emit('open')) - } - - send(payload) { - observed.payload = JSON.parse(payload) - queueMicrotask(() => this.emit('close', code)) - } - - terminate() {} - } -} - -function nativeSocketClosingWith(code) { - return class extends EventTarget { - constructor() { - super() - queueMicrotask(() => this.dispatchEvent(new Event('open'))) - } - - send() { - const event = new Event('close') - Object.defineProperty(event, 'code', { value: code }) - queueMicrotask(() => this.dispatchEvent(event)) - } - - close() {} - } -} - -const config = { cellOrigin: 'https://c2.relay.example.com' } -const random = (length) => Buffer.alloc(length, length) - -test('accepts only a canonical cell origin', () => { - assert.deepEqual( - parseLegacyAdmissionProbeArguments(['--cell-origin', config.cellOrigin]), - config - ) - assert.throws( - () => parseLegacyAdmissionProbeArguments(['--cell-origin', `${config.cellOrigin}/path`]), - /canonical/ - ) -}) - -test('proves admission with a synthetic invalid credential and exposes no identifier', async () => { - const observed = {} - assert.deepEqual( - await probeLegacyAdmission(config, { - WebSocket: socketClosingWith(4409, observed), - randomBytes: random - }), - { accepting: true } - ) - assert.match(observed.url, /^wss:\/\/c2\.relay\.example\.com\/v1\/connect\/[A-Za-z0-9_-]{16}$/) - assert.deepEqual(Object.keys(observed.payload).sort(), ['credential', 'mode', 'type', 'v']) -}) - -test('uses the dependency-free Node WebSocket event API', async () => { - await assert.doesNotReject( - probeLegacyAdmission(config, { - WebSocket: nativeSocketClosingWith(4409), - randomBytes: random - }) - ) -}) - -test('rejects the legacy draining close and any unknown outcome', async () => { - for (const [code, message] of [[4503, /draining/], [4401, /closed with 4401/]]) { - await assert.rejects( - probeLegacyAdmission(config, { - WebSocket: socketClosingWith(code, {}), - randomBytes: random - }), - message - ) - } -}) diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.mjs deleted file mode 100644 index 96c6ccf2f22..00000000000 --- a/cloud/dev/scripts/probe-relay-rehome-trust.mjs +++ /dev/null @@ -1,112 +0,0 @@ -import { pathToFileURL } from 'node:url' -import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' - -// Every cell that carries the rehome identity: the eighteen US cells and the -// seven asia-east2 cells that drain mis-homed hosts back the other way. -const PRODUCTION_CELL = - /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35)$/ -const DIRECTOR_ORIGIN = 'https://relay.onorca.dev' -const SOURCE_TRANSIENT_RETRIES = 12 -const SOURCE_TRANSIENT_RETRY_DELAY_MS = 10_000 - -export function parseRehomeTrustProbeArguments(argv, environment = process.env) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of ['director-origin', 'cell-id', 'cell-incarnation']) { - if (!values[key]) throw new Error(`missing --${key}`) - } - if (values['director-origin'] !== DIRECTOR_ORIGIN) { - throw new Error('--director-origin must be the production Relay origin') - } - if (!PRODUCTION_CELL.test(values['cell-id'])) throw new Error('--cell-id is not approved') - if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( - values['cell-incarnation'] - )) throw new Error('--cell-incarnation is invalid') - const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN - if (!token || token.length > 8_192 || !/^[^.]+\.[^.]+\.[^.]+$/.test(token)) { - throw new Error('admin identity token is unavailable') - } - return { - directorOrigin: DIRECTOR_ORIGIN, - cellId: values['cell-id'], - cellIncarnation: values['cell-incarnation'], - token - } -} - -export async function probeRehomeTrust(config, dependencies = {}) { - const fetchImpl = dependencies.fetch ?? fetch - const request = () => fetchAdminOnceMore( - fetchImpl, - `${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`, - { - method: 'POST', - headers: { - authorization: `Bearer ${config.token}`, - 'content-type': 'application/json' - }, - body: JSON.stringify({ - v: 1, - sourceCellId: config.cellId, - sourceCellIncarnation: config.cellIncarnation - }) - }, - { wait: dependencies.wait } - ) - const wait = dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) - let response = await request() - let body = await response.json().catch(() => ({})) - // The director wraps source HTTP failures in 409; retry only explicit transient statuses. The - // load balancer can answer 503 for a minute after a fresh instance is ready (c10, 2026-10-20). - for ( - let retry = 0; - retry < SOURCE_TRANSIENT_RETRIES && - response.status === 409 && - /^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? ''); - retry++ - ) { - await wait(SOURCE_TRANSIENT_RETRY_DELAY_MS) - response = await request() - body = await response.json().catch(() => ({})) - } - if (!response.ok) { - const safeReasons = new Set([ - 'invalid_token', 'director_only', 'invalid_request', - 'regional_rehome_trust_not_configured', - 'regional_rehome_trust_probe_source_unavailable', - 'regional_rehome_trust_probe_source_invalid_response', - 'regional_rehome_trust_probe_not_proven', - ...[400, 401, 403, 404, 409, 429, 500, 502, 503, 504] - .map((status) => `regional_rehome_trust_probe_source_${status}`) - ]) - const reason = safeReasons.has(body?.error) ? body.error : 'unrecognized_error' - throw new Error(`application-mediated rehome trust probe returned ${response.status}: ${reason}`) - } - if ( - body.v !== 1 || - body.dedicatedIdentity?.firstOutcome !== 'host-not-connected' || - body.dedicatedIdentity?.secondOutcome !== 'host-not-connected' || - body.dedicatedIdentity?.accepted !== true || - body.dedicatedIdentity?.idempotent !== true || - body.sharedRuntimeIdentityRejected !== true || - body.proven !== true - ) throw new Error('application-mediated rehome trust proof is incomplete') - return body -} - -export async function main(argv = process.argv.slice(2)) { - const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv)) - process.stdout.write(`${JSON.stringify({ event: 'relay_rehome_trust_verified', ...result })}\n`) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs deleted file mode 100644 index 9e31864956d..00000000000 --- a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs +++ /dev/null @@ -1,217 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import { test } from 'node:test' -import { - parseRehomeTrustProbeArguments, - probeRehomeTrust -} from './probe-relay-rehome-trust.mjs' - -const argv = [ - '--director-origin', 'https://relay.onorca.dev', - '--cell-id', 'production-gce-c7', - '--cell-incarnation', '11111111-1111-4111-8111-111111111111' -] -const environment = { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' } - -test('binds the application-mediated probe to an exact approved cell incarnation', () => { - assert.equal(parseRehomeTrustProbeArguments(argv, environment).cellId, 'production-gce-c7') - assert.throws(() => parseRehomeTrustProbeArguments( - argv.with(1, 'https://other.example.test'), - environment - )) - assert.throws(() => parseRehomeTrustProbeArguments(argv, { - ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-token' - })) -}) - -test('requires complete aggregate application-mediated trust proof', async () => { - const config = parseRehomeTrustProbeArguments(argv, environment) - const result = await probeRehomeTrust(config, { - fetch: async (url, init) => { - assert.equal(url, 'https://relay.onorca.dev/v1/admin/regional-rehome-trust-probe') - assert.deepEqual(JSON.parse(init.body), { - v: 1, - sourceCellId: 'production-gce-c7', - sourceCellIncarnation: '11111111-1111-4111-8111-111111111111' - }) - return Response.json({ - v: 1, - dedicatedIdentity: { - firstOutcome: 'host-not-connected', - secondOutcome: 'host-not-connected', - accepted: true, - idempotent: true - }, - sharedRuntimeIdentityRejected: true, - proven: true - }) - } - }) - assert.equal(result.proven, true) -}) - -test('rejects partial or mismatched proof', async () => { - const config = parseRehomeTrustProbeArguments(argv, environment) - await assert.rejects( - probeRehomeTrust(config, { - fetch: async () => Response.json({ - v: 1, - dedicatedIdentity: { - firstOutcome: 'host-not-connected', - secondOutcome: 'host-not-connected', - accepted: true, - idempotent: true - }, - sharedRuntimeIdentityRejected: false, - proven: false - }) - }), - /incomplete/ - ) -}) - -const provenProbe = { - v: 1, - dedicatedIdentity: { - firstOutcome: 'host-not-connected', - secondOutcome: 'host-not-connected', - accepted: true, - idempotent: true - }, - sharedRuntimeIdentityRejected: true, - proven: true -} - -test('retries a transient 503 on the trust probe and proves on the second answer', async () => { - const config = parseRehomeTrustProbeArguments(argv, environment) - let calls = 0 - const result = await probeRehomeTrust(config, { - wait: async () => {}, - fetch: async () => { - calls += 1 - if (calls === 1) return new Response('warming up', { status: 503 }) - return Response.json(provenProbe) - } - }) - assert.equal(calls, 2) - assert.equal(result.proven, true) -}) - -test('fails when both trust-probe attempts return a transient 503', async () => { - const config = parseRehomeTrustProbeArguments(argv, environment) - let calls = 0 - await assert.rejects( - probeRehomeTrust(config, { - wait: async () => {}, - fetch: async () => { - calls += 1 - return new Response('warming up', { status: 503 }) - } - }), - /returned 503/ - ) - assert.equal(calls, 2) -}) - -test('approves the asia-east2 and US 3,000 rehome sources and still rejects unlisted cells', () => { - for (const cellId of [ - 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30', - 'production-gce-c31', 'production-gce-c32', 'production-gce-c33', 'production-gce-c34', - 'production-gce-c35' - ]) { - const parsed = parseRehomeTrustProbeArguments( - argv.map((value) => (value === 'production-gce-c7' ? cellId : value)), - environment - ) - assert.equal(parsed.cellId, cellId) - } - for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c36']) { - assert.throws( - () => - parseRehomeTrustProbeArguments( - argv.map((value) => (value === 'production-gce-c7' ? cellId : value)), - environment - ), - /--cell-id is not approved/ - ) - } -}) - -test('retries one director-wrapped source 503 without relaxing the proof', async () => { - let calls = 0 - const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), { - wait: async () => {}, - fetch: async () => ++calls === 1 - ? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) - : Response.json(provenProbe) - }) - assert.equal(calls, 2) - assert.equal(result.proven, true) -}) - -test('reports safe trust reasons, keeps rejection final, and redacts arbitrary error text', async () => { - for (const reason of ['regional_rehome_trust_probe_source_403', 'secret-token-example']) { - let calls = 0 - await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), { - wait: async () => { throw new Error('must not retry') }, - fetch: async () => { calls++; return Response.json({ error: reason }, { status: 409 }) } - }), error => { - assert.match(error.message, /returned 409/) - assert.ok(!error.message.includes('secret-token-example')) - if (reason.endsWith('_403')) assert.match(error.message, /source_403/) - return true - }) - assert.equal(calls, 1) - } -}) - -test('outlasts a load-balancer warm-up of consecutive wrapped source 503s', async () => { - let calls = 0 - const waits = [] - const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), { - wait: async (ms) => { waits.push(ms) }, - fetch: async () => ++calls <= 6 - ? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) - : Response.json(provenProbe) - }) - assert.equal(calls, 7) - assert.deepEqual(waits, Array(6).fill(10_000)) - assert.equal(result.proven, true) -}) - -test('stops after two minutes of wrapped transient failures', async () => { - let calls = 0 - await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), { - wait: async () => {}, - fetch: async () => { calls++; return Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) } - }), /returned 409.*source_503/) - assert.equal(calls, 13) -}) - -// The probe allowlist is the committed rehome source list, so a declared source can always probe. -test('approves exactly the committed production rehome source cells', () => { - const tfvars = readFileSync( - new URL('../../infra/terraform/environments/production.tfvars', import.meta.url), - 'utf8' - ) - const start = tfvars.indexOf('relay_region_rehome_source_cell_ids = [') - const sources = new Set( - [...tfvars.slice(start, tfvars.indexOf(']', start)).matchAll(/"([^"]+)"/g)].map(([, cell]) => cell) - ) - assert.ok(sources.has('production-gce-c35')) - for (let ordinal = 1; ordinal <= 40; ordinal++) { - const cellId = `production-gce-c${ordinal}` - const approved = (() => { - try { - parseRehomeTrustProbeArguments( - argv.map((value) => (value === 'production-gce-c7' ? cellId : value)), - environment - ) - return true - } catch { - return false - } - })() - assert.equal(approved, sources.has(cellId), cellId) - } -}) diff --git a/cloud/dev/scripts/production-cell-image-digest-consistency.test.mjs b/cloud/dev/scripts/production-cell-image-digest-consistency.test.mjs deleted file mode 100644 index 612d48abdcd..00000000000 --- a/cloud/dev/scripts/production-cell-image-digest-consistency.test.mjs +++ /dev/null @@ -1,85 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs' -import { readRelayWorkflow } from './relay-repository.mjs' - -const production = source('infra/terraform/environments/production.tfvars') -const dispatchWorkflow = readRelayWorkflow('deploy-relay-production-capacity.yml') -const jobWorkflow = readRelayWorkflow('deploy-relay-production-capacity-job.yml') - -const RELAY_REPOSITORY = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay' - -function source(path) { - return readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8') -} - -// Slice each "" = { ... } entry out of relay_gce_cells. -function productionCells() { - const block = production.slice(production.indexOf('relay_gce_cells = {')) - const cells = new Map() - for (const match of block.matchAll(/"(production-gce-c\d+)" = \{([\s\S]*?)\n {2}\}/g)) { - cells.set(match[1], match[2]) - } - assert.ok(cells.size > 0, 'relay_gce_cells parsed empty') - return cells -} - -function hardCap(body) { - const match = body.match(/connection_hard_cap\s*=\s*(\d+)/) - return match ? Number(match[1]) : undefined -} - -function imageDigest(body) { - const match = body.match(/^\s*image\s*=\s*"([^"]+)"/m) - assert.ok(match, 'cell entry has no image') - const [repository, digest] = match[1].split('@') - assert.equal(repository, RELAY_REPOSITORY) - assert.match(digest, /^sha256:[0-9a-f]{64}$/) - return digest -} - -function workflowPin(workflow, name) { - const match = workflow.match(new RegExp(`${name}: (sha256:[0-9a-f]{64})`)) - assert.ok(match, `${name} is missing or not a full digest`) - return match[1] -} - -test('every production cell pins a full relay image digest', () => { - for (const [cellId, body] of productionCells()) { - assert.match(imageDigest(body), /^sha256:[0-9a-f]{64}$/, `${cellId} image digest`) - } -}) - -test('the 1,000-cap cells are exactly the canonical capacity set', () => { - const thousandCap = [...productionCells()] - .filter(([, body]) => hardCap(body) === 1000) - .map(([cellId]) => cellId) - assert.deepEqual([...thousandCap].sort(), [...PRODUCTION_CAPACITY_CELL_IDS].sort()) -}) - -test('the 1,000-cap cells all serve one image digest', () => { - const digests = new Map() - for (const [cellId, body] of productionCells()) { - if (hardCap(body) !== 1000) continue - const digest = imageDigest(body) - if (!digests.has(digest)) digests.set(digest, []) - digests.get(digest).push(cellId) - } - assert.equal( - digests.size, - 1, - `1,000-cap cells split across digests: ${JSON.stringify([...digests])}` - ) - assert.equal([...digests.values()][0].length, PRODUCTION_CAPACITY_CELL_IDS.length) -}) - -// COMPATIBLE_CELL_IMAGE_DIGEST is one half of a reviewed (director, cell) skew pair, not a -// claim about what the fleet serves; it is re-derived by hand for each capacity wave. So it -// is deliberately NOT tied to the tfvars digest — only to its twin in the dispatch workflow. -test('both capacity workflows declare the same reviewed image pins', () => { - for (const name of ['PREDECESSOR_IMAGE_DIGEST', 'COMPATIBLE_CELL_IMAGE_DIGEST']) { - assert.equal(workflowPin(dispatchWorkflow, name), workflowPin(jobWorkflow, name), name) - } - workflowPin(jobWorkflow, 'COMPATIBLE_DIRECTOR_IMAGE_DIGEST') -}) diff --git a/cloud/dev/scripts/production-cloud-sql-rollout-lock.test.mjs b/cloud/dev/scripts/production-cloud-sql-rollout-lock.test.mjs deleted file mode 100644 index 628b09176da..00000000000 --- a/cloud/dev/scripts/production-cloud-sql-rollout-lock.test.mjs +++ /dev/null @@ -1,182 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { - LEASED_WORKFLOWS, - LOCK_GROUPS, - NOT_A_CLOUD_SQL_CANDIDATE, - PRODUCTION_LEASE, - SELECTABLE_LEASE, - STAGING_LEASE, - concurrencyBlocks, - entrypointsFor, - jobIf, - jobNeeds, - jobs, - leaseSteps, - leaseStepsByJob, - mutatesSharedInstance, - readWorkflow, - reusableCalls, - revisionMintingScripts, - workflowFiles -} from './cloud-sql-rollout-lock-census.mjs' - -const expectedLease = { production: PRODUCTION_LEASE, staging: STAGING_LEASE, selectable: SELECTABLE_LEASE } -const leasedFiles = Object.keys(LEASED_WORKFLOWS) - -function contractFiles(file) { - return [file, ...(LEASED_WORKFLOWS[file].leaseFiles ?? [])] -} - -test('every locked workflow declares exactly one lock group that never cancels', () => { - for (const file of leasedFiles) { - const blocks = concurrencyBlocks(readWorkflow(file)) - assert.equal(blocks.length, 1, `${file} must declare exactly one concurrency block`) - assert.equal(blocks[0].group, LEASED_WORKFLOWS[file].group, file) - assert.equal(blocks[0].cancelInProgress, 'false', file) - } -}) - -test('every locked workflow takes the lease for its own environment', () => { - for (const file of leasedFiles) { - const lease = expectedLease[LEASED_WORKFLOWS[file].env] - const steps = contractFiles(file).flatMap((member) => leaseSteps(readWorkflow(member))) - assert.ok(steps.length > 0, `${file} must use the Cloud SQL rollout lease action`) - for (const step of steps) { - assert.equal(step.bucket, lease.bucket, file) - assert.equal(step.object, lease.object, file) - } - } -}) - -test('the lease step runs after the credential that authorizes it', () => { - for (const file of leasedFiles) { - for (const member of contractFiles(file)) { - const text = readWorkflow(member) - const lines = text.split('\n') - for (const step of leaseSteps(text)) { - const before = lines.slice(0, step.line - 1) - const gcloud = before.lastIndexOf(' - uses: google-github-actions/setup-gcloud@v2') - assert.notEqual(gcloud, -1, `${member}: lease step at line ${step.line} has no setup-gcloud before it`) - assert.ok( - before.lastIndexOf(' - uses: actions/checkout@v4') !== -1, - `${member}: lease step at line ${step.line} runs before the local action is checked out` - ) - } - } - } -}) - -test('multi-wave workflows hold one lease per run and free it exactly once', () => { - for (const file of leasedFiles) { - const entry = LEASED_WORKFLOWS[file] - const waveFiles = entry.leaseFiles ?? [] - const callCount = waveFiles.reduce( - (total, member) => total + (reusableCalls(readWorkflow(file)).get(member) ?? 0), - 0 - ) - if (!entry.reentrant) { - assert.ok(callCount <= 1, `${file} calls a leased reusable job ${callCount} times; it needs a release job`) - for (const member of contractFiles(file)) { - for (const step of leaseSteps(readWorkflow(member))) { - assert.equal(step.release, undefined, `${member} must leave release at its default`) - } - } - continue - } - - assert.ok(callCount > 1, `${file} no longer calls its reusable job more than once`) - const steps = contractFiles(file).flatMap((member) => leaseSteps(readWorkflow(member))) - const released = steps.filter((step) => step.release === "'true'") - assert.equal(released.length, 1, `${file} must free the run lease exactly once`) - for (const step of steps) { - if (step === released[0]) continue - assert.equal(step.release, "'false'", `${file} wave jobs must hold the lease`) - } - - const callerJobs = jobs(readWorkflow(file)) - const releaseJob = leaseStepsByJob(file).find((job) => - job.steps.some((step) => step.release === "'true'") - ) - assert.ok(releaseJob, `${file} must free the lease from its own job`) - const guard = jobIf(callerJobs.find((job) => job.id === releaseJob.id).text) - assert.match(guard, /always\(\)/, `${file}: the release job must run on failure and cancellation`) - - const holders = callerJobs - .filter( - (job) => - job.id !== releaseJob.id && - (waveFiles.some((member) => job.text.includes(`uses: ./.github/workflows/${member}`)) || - leaseStepsByJob(file).find((entry) => entry.id === job.id)?.steps.length > 0) - ) - .map((job) => job.id) - const needs = jobNeeds(callerJobs.find((job) => job.id === releaseJob.id).text) - for (const holder of holders) { - assert.ok(needs.includes(holder), `${file}: the release job must need ${holder}`) - } - } -}) - -test('workflows with two lease-holding jobs can never run them together', () => { - for (const file of leasedFiles) { - const entry = LEASED_WORKFLOWS[file] - if (entry.reentrant) continue - const holding = leaseStepsByJob(file).filter((job) => job.steps.length > 0) - if (holding.length <= 1) continue - assert.ok(entry.exclusiveBy, `${file} has ${holding.length} lease-holding jobs and no exclusivity guard`) - const guards = holding.map((job) => jobIf(jobs(readWorkflow(file)).find((j) => j.id === job.id).text)) - assert.equal( - guards.filter((guard) => guard.includes(entry.exclusiveBy)).length, - 1, - `${file}: exactly one job may run when ${entry.exclusiveBy}` - ) - assert.equal( - guards.filter((guard) => guard.includes(entry.exclusiveBy.replace('==', '!='))).length, - guards.length - 1, - `${file}: every other lease-holding job must be excluded when ${entry.exclusiveBy}` - ) - } -}) - -test('census: no workflow rolls out against the shared instance outside the lease', () => { - const minters = revisionMintingScripts() - assert.ok(minters.size > 0, 'the revision-minting script scan found nothing and is vacuous') - const flagged = new Map() - for (const file of workflowFiles()) { - const reason = mutatesSharedInstance(readWorkflow(file), minters) - if (reason) flagged.set(file, reason) - } - assert.ok(flagged.size > 0, 'the rollout census found no candidates and is vacuous') - - for (const [file, reason] of flagged) { - for (const entrypoint of entrypointsFor(file)) { - assert.ok( - entrypoint in LEASED_WORKFLOWS || entrypoint in NOT_A_CLOUD_SQL_CANDIDATE, - `${entrypoint} reaches ${file} (${reason}) but is neither leased nor recorded as a non-candidate` - ) - } - } - - for (const file of workflowFiles()) { - const groups = concurrencyBlocks(readWorkflow(file)).map((block) => block.group) - if (!groups.some((group) => LOCK_GROUPS.has(group))) continue - assert.ok( - file in LEASED_WORKFLOWS || file in NOT_A_CLOUD_SQL_CANDIDATE, - `${file} sits in a Cloud SQL lock group but is neither leased nor recorded as a non-candidate` - ) - } - - for (const [file, reason] of Object.entries(NOT_A_CLOUD_SQL_CANDIDATE)) { - assert.ok(typeof reason === 'string' && reason.length > 40, `${file} needs a real reason`) - const groups = concurrencyBlocks(readWorkflow(file)).map((block) => block.group) - assert.ok( - flagged.has(file) || groups.some((group) => LOCK_GROUPS.has(group)), - `${file} is recorded as a non-candidate but nothing would have flagged it` - ) - } - - for (const file of leasedFiles) { - assert.doesNotThrow(() => readWorkflow(file), `${file} is leased but does not exist`) - assert.ok(!(file in NOT_A_CLOUD_SQL_CANDIDATE), `${file} cannot be both leased and a non-candidate`) - } -}) diff --git a/cloud/dev/scripts/push-cloud-run-model.mjs b/cloud/dev/scripts/push-cloud-run-model.mjs deleted file mode 100644 index fea782e625d..00000000000 --- a/cloud/dev/scripts/push-cloud-run-model.mjs +++ /dev/null @@ -1,140 +0,0 @@ -import { readFileSync, writeFileSync } from 'node:fs' - -// Executable fake gcloud: revision deletion obeys the platform's latest/traffic constraints. -const path = process.env.MODEL_STATE -const state = JSON.parse(readFileSync(path, 'utf8')) -const args = process.argv.slice(2) -const option = (name) => args[args.indexOf(name) + 1] -const has = (name) => args.includes(name) -const fail = (message) => { - throw new Error(message) -} -const persist = () => writeFileSync(path, JSON.stringify(state)) -const output = (value) => console.log(typeof value === 'string' ? value : JSON.stringify(value)) -const revision = (name) => state.revisions[name] ?? fail(`missing revision ${name}`) -const traffic = () => [ - { revisionName: state.serving, percent: 100 }, - ...Object.entries(state.tags).map(([tag, name]) => ({ - tag, - revisionName: name, - url: `https://${tag}.test` - })) -] -state.trace.push(args.join(' ')) -try { - if (args[0] === 'curl') { - if (state.failure === 'public' && args.some((arg) => arg.includes('https://public.test'))) { - fail('public check failed') - } - const url = args.find((arg) => arg.startsWith('https://')) - const tag = new URL(url).hostname.split('.')[0] - const name = state.tags[tag] ?? state.serving - if (has('-w')) { - output('200') - } else { - output({ - ok: true, - deliveryProtocol: 2, - mode: revision(name).spec.containers[0].env.some((entry) => entry.value === 'validation') - ? 'validation' - : 'active' - }) - } - } else if (args.slice(0, 2).join(' ') === 'run deploy') { - const name = `${option('deploy')}-${option('--revision-suffix')}` - if (state.failure === 'deploy-before') { - fail('deploy failed before create') - } - const item = structuredClone(revision(state.latest)) - item.metadata.name = name - item.spec.containers[0].image = option('--image') - item.status.imageDigest = option('--image') - item.spec.containers[0].env = item.spec.containers[0].env.filter( - (entry) => entry.name !== 'ORCA_PUSH_MODE' - ) - if (has('--update-env-vars')) { - item.spec.containers[0].env.push({ name: 'ORCA_PUSH_MODE', value: 'validation' }) - } - state.revisions[name] = item - state.latest = name - if (has('--tag')) { - state.tags[option('--tag')] = name - } - state.peak = Math.max(state.peak, Object.keys(state.revisions).length) - if (state.peak > 3) { - fail('three-revision budget exceeded') - } - if (state.failure === 'deploy-after') { - fail('deploy failed after create') - } - } else if (args.slice(0, 3).join(' ') === 'run services describe') { - if (state.failure === 'describe') { - fail('describe failed') - } - if (args.some((arg) => arg.includes('value(status.latestCreatedRevisionName)'))) { - output(state.latest) - } else { - const template = structuredClone(revision(state.latest)) - delete template.spec.containers[0].name - output({ - spec: { template }, - status: { latestCreatedRevisionName: state.latest, traffic: traffic() } - }) - } - } else if (args.slice(0, 3).join(' ') === 'run services update-traffic') { - if (has('--to-revisions')) { - const name = option('--to-revisions').split('=')[0] - revision(name) - state.serving = name - } - if (has('--remove-tags')) { - for (const tag of option('--remove-tags').split(',')) { - delete state.tags[tag] - } - } - if (has('--clear-tags')) { - state.tags = {} - } - if (state.failure === 'traffic-after') { - fail('traffic changed but response failed') - } - } else if (args.slice(0, 3).join(' ') === 'run revisions list') { - const names = Object.keys(state.revisions) - output( - (has('--filter') - ? names.filter((name) => name === option('--filter').split('=')[1]) - : names - ).join('\n') - ) - } else if (args.slice(0, 3).join(' ') === 'run revisions describe') { - const item = revision(args[3]) - const format = args.find((arg) => arg.startsWith('--format=')) ?? option('--format') - if (format.includes('minScale')) { - output(item.metadata.annotations['autoscaling.knative.dev/minScale']) - } else if (format.includes('maxScale')) { - output(item.metadata.annotations['autoscaling.knative.dev/maxScale']) - } else { - output(item) - } - } else if (args.slice(0, 3).join(' ') === 'run revisions delete') { - const name = args[3] - if (name === state.latest) { - fail('FAILED_PRECONDITION: latest created Revision cannot be directly deleted') - } - if (name === state.serving || Object.values(state.tags).includes(name)) { - fail('revision has traffic or tags') - } - if (state.failure === 'delete') { - fail('delete failed') - } - revision(name) - delete state.revisions[name] - } else { - fail(`unmodeled gcloud call ${args.join(' ')}`) - } -} catch (error) { - console.error(error.message) - process.exitCode = 1 -} finally { - persist() -} diff --git a/cloud/dev/scripts/push-gateway-recovery.test.mjs b/cloud/dev/scripts/push-gateway-recovery.test.mjs deleted file mode 100644 index 97d9c4b5539..00000000000 --- a/cloud/dev/scripts/push-gateway-recovery.test.mjs +++ /dev/null @@ -1,373 +0,0 @@ -import assert from 'node:assert/strict' -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { fileURLToPath } from 'node:url' -import { spawnSync } from 'node:child_process' -import test from 'node:test' -import { readRelayWorkflow } from './relay-repository.mjs' - -const workflow = readRelayWorkflow('push-deploy.yml') -function step(name) { - const start = workflow.indexOf(` - name: ${name}\n`) - assert.notEqual(start, -1) - const end = workflow.indexOf('\n - ', start + 1) - const block = workflow.slice(start, end === -1 ? undefined : end) - return block - .slice(block.indexOf(' run: |\n') + ' run: |\n'.length) - .split('\n') - .filter((line) => line.startsWith(' ')) - .map((line) => line.slice(10)) - .join('\n') -} -const names = { - preflight: 'Record the serving revision and require its Terraform-owned scaling', - candidate: 'Deploy the candidate revision with no traffic', - activate: 'Retire inert validation and activate the verified image', - shift: 'Shift all traffic to the verified candidate', - public: 'Verify the public origin after the shift', - rollback: 'Roll traffic back to the previous revision', - restore: 'Restore the known-good service template', - promoteRecovery: 'Promote and verify the known-good recovery revision', - cleanup: 'Delete the rejected candidate revision', - retire: 'Retire previous consumers after public checks' -} -const image = `registry/push@sha256:${'a'.repeat(64)}` -const spec = { - serviceAccountName: 'runtime@test', - containerConcurrency: 40, - containers: [ - { - image, - name: 'push-test-1', - env: [ - { - name: 'ORCA_PUSH_DATABASE_URL', - valueFrom: { secretKeyRef: { name: 'database', key: '7' } } - }, - { name: 'ORCA_PUSH_DATABASE_POOL_MAX', value: '2' } - ] - } - ] -} -const prior = { - metadata: { - name: 'push-test-old', - annotations: { - 'autoscaling.knative.dev/minScale': '1', - 'autoscaling.knative.dev/maxScale': '2' - } - }, - spec, - status: { imageDigest: image } -} -const model = fileURLToPath(new URL('./push-cloud-run-model.mjs', import.meta.url)) -const options = { skip: process.platform === 'win32' } -function exercise(callback) { - const dir = mkdtempSync(join(tmpdir(), 'push-workflow-')) - const statePath = join(dir, 'state.json') - writeFileSync( - statePath, - JSON.stringify({ - revisions: { 'push-test-old': prior }, - latest: 'push-test-old', - serving: 'push-test-old', - tags: {}, - peak: 1, - trace: [] - }) - ) - writeFileSync(join(dir, 'env'), '') - const env = { - ...process.env, - SERVICE_NAME: 'push-test', - GCP_PROJECT_ID: 'test', - GCP_REGION: 'test', - GITHUB_RUN_ID: '123', - GITHUB_RUN_ATTEMPT: '1', - IMAGE: `registry/push@sha256:${'b'.repeat(64)}`, - PUSH_MIN_INSTANCES: '1', - PUSH_MAX_INSTANCES: '2', - PUSH_RUNTIME_SERVICE_ACCOUNT: 'runtime@test', - PUSH_ORIGIN: 'https://public.test', - MODEL_STATE: statePath, - MODEL_SCRIPT: model, - RUNNER_TEMP: dir, - GITHUB_ENV: join(dir, 'env'), - GITHUB_STEP_SUMMARY: join(dir, 'summary') - } - const state = () => JSON.parse(readFileSync(statePath, 'utf8')) - const change = (edit) => { - const value = state() - edit(value) - writeFileSync(statePath, JSON.stringify(value)) - } - const run = (key, ok = true, extra = '') => { - const result = spawnSync( - 'bash', - [ - '-c', - ` - set -a - source "$GITHUB_ENV" - gcloud() { node "$MODEL_SCRIPT" "$@"; } - curl() { node "$MODEL_SCRIPT" curl "$@"; } - sleep() { :; } - ${extra} - ${names[key] ? step(names[key]) : key} - ` - ], - { cwd: dir, env, encoding: 'utf8', timeout: 30000 } - ) - assert.equal(result.status === 0, ok, `${key}: ${result.stderr}\n${result.stdout}`) - return result - } - try { - callback({ run, state, change, dir }) - } finally { - rmSync(dir, { recursive: true, force: true }) - } -} -function recover(h) { - h.change((state) => { - delete state.failure - }) - h.run('restore') - h.run('promoteRecovery') - h.run('cleanup') - h.run('retire') - const state = h.state() - assert.equal(state.serving, 'push-test-r123-1') - assert.equal(state.latest, state.serving) - assert.deepEqual(Object.keys(state.revisions), [state.serving]) - assert.equal(state.revisions[state.serving].spec.containers[0].image, image) - assert.ok(state.peak <= 3) -} - -test( - 'the executable Cloud Run model rejects deleting latest even without tags or traffic', - options, - () => - exercise((h) => { - h.run('preflight') - h.run('candidate') - h.run('gcloud run services update-traffic "$SERVICE_NAME" --clear-tags') - const result = h.run('gcloud run revisions delete "$CANDIDATE_REVISION"', false) - assert.match(result.stderr, /FAILED_PRECONDITION: latest created Revision/) - }) -) - -test( - 'success creates successor before retirement and repeated rollouts retain one consumer', - options, - () => - exercise((h) => { - for (const attempt of ['1', '2']) { - if (attempt === '2') { - writeFileSync(join(h.dir, 'env'), 'GITHUB_RUN_ATTEMPT=2\n') - } - for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public', 'retire']) { - h.run(key) - } - const state = h.state() - assert.equal(state.serving, `push-test-a123-${attempt}`) - assert.deepEqual(Object.keys(state.revisions), [state.serving]) - assert.equal(state.peak, 3) - } - }) -) - -for (const failure of ['deploy-before', 'deploy-after', 'describe']) { - test(`validation ${failure} recovers without deleting latest`, options, () => - exercise((h) => { - h.run('preflight') - h.change((state) => { - state.failure = failure - }) - h.run('candidate', false) - recover(h) - }) - ) -} -for (const failure of ['deploy-before', 'deploy-after', 'delete', 'describe']) { - test( - `activation ${failure} frees validation slot before recovery and stays within three`, - options, - () => - exercise((h) => { - h.run('preflight') - h.run('candidate') - h.change((state) => { - state.failure = failure - }) - h.run('activate', false) - recover(h) - }) - ) -} - -test( - 'ambiguous traffic shift records intent before mutation, rolls back and recovers', - options, - () => - exercise((h) => { - h.run('preflight') - h.run('candidate') - h.run('activate') - h.change((state) => { - state.failure = 'traffic-after' - }) - h.run('shift', false) - assert.match(readFileSync(join(h.dir, 'env'), 'utf8'), /TRAFFIC_SHIFT_ATTEMPTED=true/) - h.change((state) => { - delete state.failure - }) - h.run('rollback') - recover(h) - }) -) - -test('failed public check rolls back and recovers', options, () => - exercise((h) => { - for (const key of ['preflight', 'candidate', 'activate', 'shift']) { - h.run(key) - } - h.change((state) => { - state.failure = 'public' - }) - h.run('public', false) - h.change((state) => { - delete state.failure - }) - h.run('rollback') - recover(h) - }) -) - -for (const defect of [ - 'runtime', - 'secret', - 'mode', - 'image', - 'traffic', - 'scaling', - 'deploy-before', - 'deploy-after' -]) { - test(`recovery rejects ${defect} and preserves partial-create state`, options, () => - exercise((h) => { - h.run('preflight') - h.run('candidate') - h.change((state) => { - const revision = state.revisions[state.latest] - if (defect === 'runtime') { - revision.spec.serviceAccountName = 'wrong@test' - } - if (defect === 'secret') { - revision.spec.containers[0].env[0].valueFrom.secretKeyRef.key = '8' - } - if (defect === 'scaling') { - revision.metadata.annotations['autoscaling.knative.dev/maxScale'] = '3' - } - if (defect === 'traffic') { - state.serving = state.latest - } - if (defect.startsWith('deploy-')) { - state.failure = defect - } - }) - // Corrupt the recovery response after the modeled deploy while keeping real jq assertions. - const extra = ['mode', 'image'].includes(defect) - ? ` - gcloud() { - node "$MODEL_SCRIPT" "$@" > "$RUNNER_TEMP/out" || return $? - if [[ "$*" == 'run services describe '* && "$*" == *'--format=json'* ]]; then - jq '${defect === 'mode' ? '.spec.template.spec.containers[0].env += [{name:"ORCA_PUSH_MODE",value:"validation"}]' : '.spec.template.spec.containers[0].image = "wrong"'}' "$RUNNER_TEMP/out" - else cat "$RUNNER_TEMP/out"; fi - }` - : '' - h.run('restore', false, extra) - const recorded = readFileSync(join(h.dir, 'env'), 'utf8') - assert.match(recorded, /TEMPLATE_RECOVERY_REVISION=push-test-r123-1/) - assert.doesNotMatch(recorded, /TEMPLATE_RESTORED=true/) - assert.ok(h.state().peak <= 3) - }) - ) -} - -test('failed validation retirement blocks a fourth revision during recovery', options, () => - exercise((h) => { - h.run('preflight') - h.run('candidate') - h.change((state) => { - state.failure = 'delete' - }) - h.run('activate', false) - h.run('restore', false) - assert.equal(h.state().peak, 3) - assert.equal(h.state().revisions['push-test-r123-1'], undefined) - }) -) - -test( - 'failed retirement after public checks leaves verified serving and blocks the next run', - options, - () => - exercise((h) => { - for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public']) { - h.run(key) - } - h.change((state) => { - state.failure = 'delete' - }) - h.run('retire', false) - assert.equal(h.state().serving, 'push-test-a123-1') - h.run('preflight', false) - assert.match(workflow, /env.ROLLOUT_VERIFIED != 'true'/) - }) -) - -test( - 'recovery promotion failure keeps consumers for operator diagnosis and blocks new rollout', - options, - () => - exercise((h) => { - h.run('preflight') - h.run('candidate') - h.run('restore') - h.change((state) => { - state.failure = 'public' - }) - h.run('promoteRecovery', false) - assert.doesNotMatch(readFileSync(join(h.dir, 'env'), 'utf8'), /RECOVERY_VERIFIED=true/) - h.run('preflight', false) - }) -) - -const capability = step('Require image support for inert validation') -for (const [label, source, ok] of [ - ['old image', 'export function loadPushConfig() { return {}; }', false], - [ - 'invalid mode accepted', - 'export function loadPushConfig(env) { return { mode: env.ORCA_PUSH_MODE }; }', - false - ], - [ - 'validation supported', - `export function loadPushConfig(env) { - if (env.ORCA_PUSH_MODE !== 'validation') throw new Error('invalid mode'); - return { mode: 'validation' }; - }`, - true - ] -]) { - test(`pre-production image smoke: ${label}`, options, () => - exercise((h) => { - const dist = join(h.dir, 'apps', 'push', 'dist') - mkdirSync(dist, { recursive: true }) - writeFileSync(join(h.dir, 'package.json'), '{"type":"module"}') - writeFileSync(join(dist, 'config.js'), source) - h.run(capability, ok, 'docker() { node "${@: -3}"; }') - }) - ) -} diff --git a/cloud/dev/scripts/push-gateway-workflow.test.mjs b/cloud/dev/scripts/push-gateway-workflow.test.mjs deleted file mode 100644 index 5a8349c1855..00000000000 --- a/cloud/dev/scripts/push-gateway-workflow.test.mjs +++ /dev/null @@ -1,329 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { - concurrencyBlocks, - jobIf, - jobs, - LEASE_ACTION, - leaseSteps -} from './cloud-sql-rollout-lock-census.mjs' -import { readRelayWorkflow, relayWorkflowFile } from './relay-repository.mjs' - -// Why: the push gateway holds the APNs key and is the only thing standing between a paired -// phone and a silent notification pipeline. Its deploy is a blue/green rollout against the -// dedicated Cloud SQL instance, and each of the guarantees below is one careless edit from gone. -const WORKFLOW = 'push-deploy.yml' -const workflow = readRelayWorkflow(WORKFLOW) -const deploy = () => { - const job = jobs(workflow).find((entry) => entry.id === 'deploy') - assert.ok(job, 'the workflow no longer declares a deploy job') - return job -} - -function terraform(file) { - return readFileSync(new URL(`../../infra/terraform/${file}`, import.meta.url), 'utf8') -} - -// The ordered step names; every assertion below reads positions out of this list rather than -// restating them, so a reordering that breaks the no-traffic guarantee fails here. -const stepNames = () => [...workflow.matchAll(/^ {6}- name: (.+)$/gm)].map((match) => match[1]) - -const indexOfStep = (name) => { - const index = stepNames().indexOf(name) - assert.notEqual(index, -1, `the workflow no longer has a "${name}" step`) - return index -} - -test('the whole surface stays inert until the owner enables cloud operations', () => { - const guard = jobIf(deploy().text) - assert.ok(guard.includes("vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'"), guard) - assert.ok(guard.includes("github.ref == 'refs/heads/main'"), guard) - assert.equal(jobs(workflow).length, 1, 'a second job would need its own gate') -}) - -test('it authenticates through Workload Identity and holds no repository secret', () => { - assert.match(workflow, /uses: google-github-actions\/auth@v2/) - assert.match(workflow, /workload_identity_provider: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER \}\}/) - assert.match(workflow, /service_account: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT \}\}/) - assert.match(workflow, /environment: production/) - for (const [, name] of workflow.matchAll(/secrets\.([A-Za-z_][A-Za-z0-9_]*)/g)) { - assert.equal(name, 'GITHUB_TOKEN', `the workflow reads secrets.${name}`) - } -}) - -// Why: Terraform trusts exact workflow filenames, not a prefix. A rename here without the -// matching tfvars-independent list entry would fail authentication at dispatch time only. -test('Terraform trusts this exact workflow file on the production deploy provider', () => { - assert.match(terraform('push-deploy-identity.tf'), /push-deploy\.yml@refs\/heads\/main/) - assert.doesNotMatch(terraform('relay-github-actions.tf'), /push-deploy\.yml/) - assert.equal(relayWorkflowFile(WORKFLOW), 'cloud-push-deploy.yml') -}) - -test('the rollout is serialized and leases its dedicated push rollout lock', () => { - const blocks = concurrencyBlocks(workflow) - assert.equal(blocks.length, 1) - assert.equal(blocks[0].group, 'production-push-rollout') - assert.equal(blocks[0].cancelInProgress, 'false') - const steps = leaseSteps(workflow) - assert.equal(steps.length, 1, 'exactly one lease step, held for the whole run') - assert.equal(steps[0].bucket, 'onorca-cloud-terraform-state') - assert.equal(steps[0].object, 'terraform/state/push-rollout/production.lock') - assert.equal(steps[0].release, undefined, 'release stays at its default for a single-job run') -}) - -// Why: the ops guardrail is that a piped command only fails the step when pipefail is set, and -// pipefail only applies under an explicit bash shell. Every multi-line body here opts in. -test('every multi-line command runs under bash with pipefail', () => { - const bodies = [...workflow.matchAll(/^ {8}(shell: bash\n {8})?run: \|\n((?: {10}.*\n|\n)+)/gm)] - assert.ok(bodies.length >= 8, `only ${bodies.length} multi-line commands were found`) - for (const match of bodies) { - assert.ok(match[1], `a multi-line command does not declare shell: bash:\n${match[2].slice(0, 120)}`) - assert.match(match[2], /^ {10}set -euo pipefail$/m) - } -}) - -test('the candidate revision takes no traffic and is addressed by its own tag', () => { - assert.match(workflow, /gcloud run deploy "\$\{SERVICE_NAME\}"/) - assert.match(workflow, /^ {12}--no-traffic \\$/m) - assert.match(workflow, /--tag "\$\{tag\}"/) - assert.match(workflow, /test "\$\{CANDIDATE_REVISION\}" != "\$\{ROLLBACK_REVISION\}"/) - assert.ok( - indexOfStep('Record the serving revision and require its Terraform-owned scaling') < - indexOfStep('Deploy the candidate revision with no traffic'), - 'the rollback target must be captured before the candidate exists' - ) -}) - -// Why: scaling is a Terraform-owned field that `lifecycle.ignore_changes` does not cover, so a -// deploy that passed --max-instances would revert a later push_max_instances raise on every run. -// The workflow asserts the shape instead of writing it, on the serving revision before the -// candidate exists and on the candidate that inherits it. -test('the deploy asserts the Terraform-owned scaling instead of mutating it', () => { - assert.doesNotMatch(workflow, /--max-instances/, 'the deploy must not write a scaling field') - assert.doesNotMatch(workflow, /--min-instances "/, 'the deploy must not write a scaling field') - // The floor is the variables.tf default; production.tfvars overrides only the ceiling, down to - // the two instances the Cloud SQL connection budget leaves room for. - assert.match(workflow, /PUSH_MIN_INSTANCES: 1$/m) - assert.match(workflow, /PUSH_MAX_INSTANCES: 2$/m) - assert.match(terraform('variables.tf'), /variable "push_min_instances"[\s\S]*?default {5}= 1/) - assert.match(terraform('environments/production.tfvars'), /^push_max_instances {9}= 2$/m) - const gate = indexOfStep('Record the serving revision and require its Terraform-owned scaling') - assert.ok(gate < indexOfStep('Deploy the candidate revision with no traffic')) - assert.match(workflow, /autoscaling\.knative\.dev\/minScale/) - assert.match(workflow, /\[\[ "\$\{floor:-0\}" -lt "\$\{PUSH_MIN_INSTANCES\}" \]\]/) - assert.match(workflow, /test "\$\{ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/) - assert.match(workflow, /test "\$\{candidate_ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/) -}) - -// Why: the image build is not a Cloud SQL operation, and the lease is a global serialization -// point. A build inside it blocks every relay deploy and rehome for its duration. -test('the image is built before the rollout lease is taken', () => { - const lease = workflow.indexOf(`- uses: ${LEASE_ACTION}`) - assert.notEqual(lease, -1) - const build = workflow.indexOf('- name: Build and publish the immutable gateway image') - const deployCandidate = workflow.indexOf('- name: Deploy the candidate revision with no traffic') - assert.ok(build < lease, 'the build must finish before the run takes the lease') - assert.ok(lease < deployCandidate, 'the lease must still cover the deploy, probe, and shift') -}) - -// Why: the gateway's Cloud SQL draw is instances x pool, and the root that takes the rollout -// lease can only account for a pool it declares. Leaving it at the application default hid it. -test('the database pool size is Terraform-owned and bounded at plan time', () => { - const source = terraform('push-gateway.tf') - assert.match(source, /name {2}= "ORCA_PUSH_DATABASE_POOL_MAX"/) - assert.match(source, /value = tostring\(var\.push_database_pool_max\)/) - assert.match(terraform('variables.tf'), /variable "push_database_pool_max"[\s\S]*?default {5}= 2/) - const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n lifecycle \{([\s\S]*?)\n \}/.exec(source) - assert.ok(block, 'the push service no longer declares a lifecycle block') - assert.match( - block[1], - /var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/, - 'instances x pool must be bounded at plan time' - ) - assert.match( - readFileSync(new URL('../../apps/push/src/config.ts', import.meta.url), 'utf8'), - /ORCA_PUSH_DATABASE_POOL_MAX/, - 'the gateway must read the variable Terraform sets' - ) -}) - -test('the candidate is probed on its own URL before any traffic moves', () => { - const probe = indexOfStep('Probe the candidate readiness endpoint') - assert.ok(probe > indexOfStep('Deploy the candidate revision with no traffic')) - assert.ok(probe < indexOfStep('Shift all traffic to the verified candidate')) - assert.match(workflow, /"\$\{CANDIDATE_URL\}\/ready"/) - assert.match(workflow, /test "\$\{code\}" = 200/) - assert.ok(workflow.indexOf('${CANDIDATE_URL}/ready') < workflow.indexOf('${CANDIDATE_URL}/health')) - assert.match(workflow, /\.deliveryProtocol == 2/, 'verify the durable gateway after readiness') -}) - -// Why: a gateway that answers /ready can still hold no usable FCM credential. The probe must be -// validate-only, must use a token that cannot exist, and must treat a denied credential as the -// failure. Accepting PERMISSION_DENIED would make the whole step decorative. -test('the FCM probe is validate-only and separates a bad token from a bad credential', () => { - const fcm = indexOfStep('Prove the runtime identity can reach FCM') - assert.ok(fcm > indexOfStep('Probe the candidate readiness endpoint')) - assert.ok(fcm < indexOfStep('Shift all traffic to the verified candidate')) - assert.match(workflow, /"validate_only":true/) - assert.match(workflow, /https:\/\/fcm\.googleapis\.com\/v1\/projects\/\$\{GCP_PROJECT_ID\}\/messages:send/) - assert.match(workflow, /GCP_PROJECT_ID: onorca-cloud$/m) - assert.match(workflow, /orca-push-deploy-probe-invalid-token/) - assert.match(workflow, /test "\$\{status\}" = INVALID_ARGUMENT/) - assert.match(workflow, /test "\$\{status\}" = PERMISSION_DENIED/) - // Only those four answers are conclusive; a 429 or a 5xx says nothing about the credential, so - // it is retried rather than read as either verdict. A denied credential still fails at once. - assert.match(workflow, /for attempt in \$\(seq 1 5\); do/) - const probe = workflow.slice( - workflow.indexOf('- name: Prove the runtime identity can reach FCM'), - workflow.indexOf('- name: Shift all traffic to the verified candidate') - ) - assert.match(probe, /for attempt in \$\(seq 1 5\); do/) - assert.match(probe, /test "\$\{code\}" = 401 \|\| test "\$\{code\}" = 403; then\n {14}break/) - assert.match( - workflow, - /--impersonate-service-account "\$\{PUSH_RUNTIME_SERVICE_ACCOUNT\}"/, - 'the probe must exercise the runtime credential, not the deploy identity' - ) - // Why: that token reads the Apple signing key. Masking it means a later `set -x` or a - // debug re-run cannot print it into a public log. - assert.match( - probe, - /test -n "\$\{token\}"\n {10}echo "::add-mask::\$\{token\}"/, - 'the impersonated token must be masked before anything else runs' - ) - assert.match(workflow, /PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud\.iam\.gserviceaccount\.com/) -}) - -// Why: a deploy ends with traffic pinned to an exact revision, and a rollback pins it to the -// previous one. Terraform reverting the service to 100% LATEST would undo either silently. -test('Terraform does not own the image or the traffic split', () => { - const source = terraform('push-gateway.tf') - const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n lifecycle \{([\s\S]*?)\n \}/.exec(source) - assert.ok(block, 'the push service no longer declares a lifecycle block') - assert.match(block[1], /template\[0\]\.containers\[0\]\.image/) - assert.match(block[1], /^\s*traffic$/m) -}) - -test('impersonating the runtime identity is a Terraform-declared grant', () => { - const source = terraform('push-gateway.tf') - assert.match(source, /resource "google_service_account_iam_member" "github_production_push_runtime_token_creator"/) - assert.match(source, /role\s+= "roles\/iam\.serviceAccountTokenCreator"/) - assert.match(source, /resource "google_cloud_run_v2_service_iam_member" "github_production_push_developer"/) -}) - -test('the traffic shift is all-or-nothing and is verified after the fact', () => { - const shift = indexOfStep('Shift all traffic to the verified candidate') - assert.match(workflow, /gcloud run services update-traffic "\$\{SERVICE_NAME\}"/) - assert.match(workflow, /--to-revisions "\$\{CANDIDATE_REVISION\}=100"/) - assert.match(workflow, /test "\$\{serving\}" = "\$\{CANDIDATE_REVISION\}"/) - assert.ok(shift < indexOfStep('Verify the public origin after the shift')) - assert.match(workflow, /PUSH_ORIGIN: https:\/\/push\.onorca\.dev/) - assert.match(workflow, /"\$\{PUSH_ORIGIN\}\/ready"/) -}) - -// Why: the origin can lag the traffic move by seconds, and a single unlucky curl would otherwise -// roll a healthy deploy back. It retries on the same schedule as the candidate probe. -test('the post-shift origin check retries like the candidate probe', () => { - const check = workflow.slice( - workflow.indexOf('- name: Verify the public origin after the shift'), - workflow.indexOf('- name: Roll traffic back to the previous revision') - ) - assert.match(check, /for attempt in \$\(seq 1 30\); do/) - assert.match(check, /sleep 5/) - assert.match(check, /test "\$\{code\}" = 200/) -}) - -// Why: the summary carries the rollback target. Writing it after the origin check meant the one -// run that needed it, the run whose check failed, was the one run that never got it. -test('the summary is written before anything that can fail after the shift', () => { - const summary = indexOfStep('Publish the rollout summary') - assert.ok(summary > indexOfStep('Shift all traffic to the verified candidate')) - assert.ok(summary < indexOfStep('Verify the public origin after the shift')) - assert.match(workflow, /Known-good image:/) - assert.match(workflow, /GITHUB_STEP_SUMMARY/) -}) - -// Why: everything after the shift runs with production on the candidate, so a failure there is a -// live gateway that has to go back. The marker is what separates that case from a failure before -// the shift, where production never moved and the candidate is the thing to clean up. -test('a failure after the shift rolls production back automatically', () => { - const rollback = indexOfStep('Roll traffic back to the previous revision') - assert.ok(rollback > indexOfStep('Verify the public origin after the shift')) - assert.match(workflow, /echo "TRAFFIC_SHIFTED=true" >> "\$\{GITHUB_ENV\}"/) - const shift = workflow.indexOf('- name: Shift all traffic to the verified candidate') - assert.ok( - workflow.indexOf('echo "TRAFFIC_SHIFTED=true"') > shift, - 'the success marker follows the shift step' - ) - const body = workflow.slice( - workflow.indexOf('- name: Roll traffic back to the previous revision'), - workflow.indexOf('- name: Delete the rejected candidate revision') - ) - assert.match( - body, - /if: \$\{\{ \(failure\(\) \|\| cancelled\(\)\) && env\.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env\.ROLLOUT_VERIFIED != 'true' \}\}/, - 'the rollback must be conditioned on both failure and the shift marker' - ) - assert.match(body, /test -n "\$\{ROLLBACK_REVISION:-\}"/) - assert.match(body, /--to-revisions "\$\{ROLLBACK_REVISION\}=100"/) - assert.match(body, /test "\$\{serving\}" = "\$\{ROLLBACK_REVISION\}"/) - assert.match(body, /GITHUB_STEP_SUMMARY/, 'the rollback must be reported in the summary') -}) - -// Why: a candidate that never took traffic still holds a warm instance and a Cloud SQL pool. Its -// tag comes off first, because Cloud Run refuses to delete a revision a traffic target names. -test('verified recovery authorizes rejected candidate deletion', () => { - const body = workflow.slice( - workflow.indexOf('- name: Delete the rejected candidate revision'), - workflow.indexOf('- name: Drop the candidate traffic tag') - ) - assert.match( - body, - /env\.RECOVERY_VERIFIED == 'true'/, - 'cleanup must wait for verified recovery traffic and public checks' - ) - assert.match(body, /if test -z "\$\{CANDIDATE_REVISION:-\}"; then/) - assert.ok( - body.indexOf('--remove-tags') < body.indexOf('gcloud run revisions delete'), - 'the tag must come off before the revision is deleted' - ) - assert.match(body, /echo "CANDIDATE_TAG=" >> "\$\{GITHUB_ENV\}"/) -}) - -test('the run always drops its traffic tag', () => { - const cleanup = indexOfStep('Drop the candidate traffic tag') - assert.equal(cleanup, stepNames().length - 1, 'tag cleanup must be the last step') - assert.match(workflow, /--remove-tags "\$\{CANDIDATE_TAG\}"/) - const body = workflow.slice(workflow.indexOf('- name: Drop the candidate traffic tag')) - assert.match(body, /if: always\(\)/) - assert.match(body, /test -n "\$\{CANDIDATE_TAG:-\}" \|\| exit 0/) -}) - -test('push credentials cannot assume the shared Relay deploy identity', () => { - const source = terraform('push-deploy-identity.tf') - assert.match(source, /"attribute.push_deploy"\s*=\s*"'production'"/) - assert.doesNotMatch(source, /"attribute.repository"\s*=/) - assert.match(source, /attribute\.push_deploy\/production/) - assert.doesNotMatch(workflow, /PRODUCTION_GCP_RELAY_DEPLOY_/) - assert.doesNotMatch(terraform('push-gateway.tf'), /member\s*=\s*local\.relay_github_deploy_service_account_member/) -}) - -test('push has only a dedicated database attachment and a narrowly scoped deployment lease', () => { - const service = terraform('push-gateway.tf') - const database = terraform('push-dedicated-database.tf') - assert.match(service, /instances = \[google_sql_database_instance\.push_dedicated\[0\]\.connection_name\]/) - assert.match(service, /secret\s*= google_secret_manager_secret\.push_dedicated_database_url\[0\]\.secret_id/) - assert.match(service, /version = google_secret_manager_secret_version\.push_dedicated_database_url\[0\]\.version/) - assert.doesNotMatch(service + database, /push_dedicated_database_(?:active|enabled)|local\.relay_database_connection_name|resource "google_sql_database" "push"/) - assert.match(database, /tier\s*= "db-custom-2-7680"/) - assert.match(database, /availability_type = "REGIONAL"/) - assert.match(database, /deletion_protection\s*= true/) - assert.match(database, /deletion_protection_enabled = true/) - const identity = terraform('push-deploy-identity.tf') - const lease = identity.match(/resource "google_storage_bucket_iam_member" "github_push_rollout_lease" \{([\s\S]*?)\n\}/)?.[1] - assert.ok(lease) - assert.match(lease, /member = local\.push_deploy_member/) - assert.match(lease, /role\s*= "roles\/storage.objectAdmin"/) - assert.match(lease, /resource.name == 'projects\/_\/buckets\/\$\{var.project_id\}-terraform-state\/objects\/terraform\/state\/push-rollout\/production.lock'/) -}) diff --git a/cloud/dev/scripts/read-relay-production-capacity-identity.mjs b/cloud/dev/scripts/read-relay-production-capacity-identity.mjs deleted file mode 100644 index 764b2732cfb..00000000000 --- a/cloud/dev/scripts/read-relay-production-capacity-identity.mjs +++ /dev/null @@ -1,31 +0,0 @@ -import { readFileSync } from 'node:fs' -import { pathToFileURL } from 'node:url' - -const CAPACITY_IDENTITY_NAME = 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT' - -export function readProductionCapacityIdentity(revision) { - const env = revision?.spec?.containers?.[0]?.env - if (!Array.isArray(env)) throw new Error('director revision environment is missing') - const matches = env.filter((entry) => entry?.name === CAPACITY_IDENTITY_NAME) - if (matches.length === 0) return null - if (matches.length !== 1) throw new Error('duplicate capacity identity') - const value = matches[0]?.value - if (typeof value !== 'string' || value.length === 0) { - throw new Error('capacity identity is not a literal string') - } - return value -} - -export function main() { - const revision = JSON.parse(readFileSync(0, 'utf8')) - process.stdout.write(`${JSON.stringify(readProductionCapacityIdentity(revision))}\n`) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - try { - main() - } catch (error) { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - } -} diff --git a/cloud/dev/scripts/read-relay-production-capacity-identity.test.mjs b/cloud/dev/scripts/read-relay-production-capacity-identity.test.mjs deleted file mode 100644 index c75ddcbaf77..00000000000 --- a/cloud/dev/scripts/read-relay-production-capacity-identity.test.mjs +++ /dev/null @@ -1,44 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { readProductionCapacityIdentity } from './read-relay-production-capacity-identity.mjs' - -function revision(env) { - return { spec: { containers: [{ env }] } } -} - -test('reads absent, exact, and foreign literal capacity identities', () => { - assert.equal(readProductionCapacityIdentity(revision([])), null) - assert.equal( - readProductionCapacityIdentity(revision([ - { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'capacity@example.test' } - ])), - 'capacity@example.test' - ) - assert.equal( - readProductionCapacityIdentity(revision([ - { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'foreign@example.test' } - ])), - 'foreign@example.test' - ) -}) - -test('rejects malformed or duplicate capacity identity entries', () => { - for (const entry of [ - { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: null }, - { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: '' }, - { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', valueSource: { secretKeyRef: {} } } - ]) { - assert.throws( - () => readProductionCapacityIdentity(revision([entry])), - /not a literal string/ - ) - } - assert.throws( - () => readProductionCapacityIdentity(revision([ - { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'one@example.test' }, - { name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'two@example.test' } - ])), - /duplicate capacity identity/ - ) - assert.throws(() => readProductionCapacityIdentity({}), /environment is missing/) -}) diff --git a/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs b/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs deleted file mode 100644 index b8e368f6e1b..00000000000 --- a/cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs +++ /dev/null @@ -1,140 +0,0 @@ -import { spawnSync } from 'node:child_process' -import { fileURLToPath } from 'node:url' - -const SECRET = 'orca-cloud-relay-regional-placement-enabled' - -function validate(input) { - for (const key of ['project', 'region', 'service', 'bootstrap_version']) { - if (typeof input?.[key] !== 'string' || !input[key] || /[\r\n]/.test(input[key])) { - throw new Error(`invalid ${key}`) - } - } - if (!/^[a-z][a-z0-9-]{0,62}$/.test(input.service)) throw new Error('invalid service') - if (!/^[1-9][0-9]*$/.test(input.bootstrap_version)) { - throw new Error('invalid bootstrap_version') - } -} - -export function classifyRelayServiceDescribeFailure(args, stderr) { - const serviceDescribe = args[0] === 'run' && args[1] === 'services' && args[2] === 'describe' - if (serviceDescribe && (stderr.includes('NOT_FOUND') || /Cannot find service \[[^\]\r\n]+\]/.test(stderr))) { - return 'NOT_FOUND' - } - return 'GCLOUD_FAILED' -} - -function defaultRun(args) { - const result = spawnSync('gcloud', args, { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'pipe'], - maxBuffer: 10 * 1024 * 1024 - }) - if (result.status !== 0) { - const error = new Error('gcloud read failed') - error.code = classifyRelayServiceDescribeFailure(args, result.stderr) - throw error - } - return JSON.parse(result.stdout) -} - -function gcloudArguments(kind, input, revision) { - return [ - 'run', kind, 'describe', revision ?? input.service, - '--project', input.project, - '--region', input.region, - '--format=json' - ] -} - -export function readRelayServingRegionalPlacementVersion(input, dependencies = {}) { - validate(input) - const run = dependencies.run ?? defaultRun - let service - try { - service = run(gcloudArguments('services', input)) - } catch (error) { - if (error?.code === 'NOT_FOUND') { - return { version: input.bootstrap_version, cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' } - } - throw error - } - const serving = (service.status?.traffic ?? []).filter( - (entry) => Number(entry.percent ?? 0) > 0 - ) - if ( - serving.length !== 1 || - Number(serving[0].percent) !== 100 || - typeof serving[0].revisionName !== 'string' - ) { - throw new Error('Relay director must have exactly one revision serving 100% traffic') - } - const revision = run(gcloudArguments('revisions', input, serving[0].revisionName)) - const cohortSettings = (revision.spec?.containers ?? []).flatMap((container) => - (container.env ?? []).filter((environment) => - environment.name === 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT') - ) - if (cohortSettings.length > 1 || (cohortSettings.length === 1 && - (typeof cohortSettings[0].value !== 'string' || - !/^(?:[0-9]|[1-9][0-9]|100)$/.test(cohortSettings[0].value)))) { - throw new Error('serving region correction cohort is invalid') - } - const cohort_percent = cohortSettings[0]?.value ?? '0' - // Set by the director deploy (placement) or by hand (shadow cells), never by Terraform: carried - // so an apply keeps them, and empty when the serving revision has none. - const reserve_placement = servingPlainSetting(revision, 'ORCA_RELAY_RESERVE_PLACEMENT', /^(?:off|dry-run|on)$/) - const shadow_seat_feed_cells = servingPlainSetting( - revision, - 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', - /^(?:all|[A-Za-z0-9_-]+(?:, ?[A-Za-z0-9_-]+)*)$/ - ) - const carried = { cohort_percent, reserve_placement, shadow_seat_feed_cells } - const references = (revision.spec?.containers ?? []).flatMap((container) => - (container.env ?? []).filter( - (environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED' - ) - ) - if (references.length === 0) return { version: input.bootstrap_version, ...carried } - const reference = normalizeSecretReference(references[0]) - if ( - references.length !== 1 || - reference?.secret !== SECRET || - !/^[1-9][0-9]*$/.test(reference?.version ?? '') - ) { - throw new Error('serving regional placement secret reference is invalid') - } - return { version: reference.version, ...carried } -} - -// One plain value, or none; anything else (a secret, a duplicate, a malformed value) fails the -// plan rather than letting an apply drop or reset the setting. -function servingPlainSetting(revision, name, valid) { - const settings = (revision.spec?.containers ?? []).flatMap((container) => - (container.env ?? []).filter((environment) => environment.name === name) - ) - if (settings.length === 0) return '' - if (settings.length > 1 || typeof settings[0].value !== 'string' || !valid.test(settings[0].value)) { - throw new Error(`serving ${name} is invalid`) - } - return settings[0].value -} - -// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but -// `gcloud run revisions describe --format=json` emits the Knative v1 shape -// `valueFrom.secretKeyRef.{name,key}`, where `name` may be a full resource path. -// A `key` of "latest" is deliberately left invalid: the director's serving -// version must be a pinned integer for this data source to mean anything. -export function normalizeSecretReference(environment) { - const v2 = environment?.valueSource?.secretKeyRef - if (v2) return { secret: v2.secret, version: v2.version } - const v1 = environment?.valueFrom?.secretKeyRef - if (!v1) return undefined - const secret = typeof v1.name === 'string' ? v1.name.replace(/^projects\/[^/]+\/secrets\//, '') : undefined - return { secret, version: v1.key } -} - -if (process.argv[1] === fileURLToPath(import.meta.url)) { - const chunks = [] - for await (const chunk of process.stdin) chunks.push(chunk) - const input = JSON.parse(Buffer.concat(chunks).toString('utf8')) - process.stdout.write(`${JSON.stringify(readRelayServingRegionalPlacementVersion(input))}\n`) -} diff --git a/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs b/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs deleted file mode 100644 index 8fd0c3a551e..00000000000 --- a/cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs +++ /dev/null @@ -1,209 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { fileURLToPath } from 'node:url' -import { - classifyRelayServiceDescribeFailure, - readRelayServingRegionalPlacementVersion -} from './read-relay-serving-regional-placement-version.mjs' - -const input = { - project: 'onorca-cloud', - region: 'us-central1', - service: 'orca-cloud-relay', - bootstrap_version: '7' -} - -function revision(version = '11') { - return { - spec: { - containers: [{ - env: [{ - name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED', - valueSource: { - secretKeyRef: { - secret: 'orca-cloud-relay-regional-placement-enabled', - version - } - } - }] - }] - } - } -} - -// Why: `gcloud run revisions describe --format=json` emits the Knative v1 shape, where the -// secret lives in `name` and the version in `key`, and `name` may be the full resource path. -function v1Revision(name, key) { - return { - spec: { - containers: [{ - env: [{ - name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED', - valueFrom: { secretKeyRef: { name, key } } - }] - }] - } - } -} - -function serving() { - return { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } } -} - -test('reads the exact version from the sole traffic-serving revision', () => { - const calls = [] - const result = readRelayServingRegionalPlacementVersion(input, { - run: (args) => { - calls.push(args) - return calls.length === 1 - ? { - status: { - traffic: [ - { revisionName: 'relay-failed-latest', tag: 'candidate' }, - { revisionName: 'relay-serving', percent: 100 } - ] - } - } - : revision() - } - }) - - assert.deepEqual(result, { version: '11', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' }) - assert.equal(calls[1][3], 'relay-serving') -}) - -test('reads the gcloud v1 secret reference shape by bare id and by full resource path', () => { - for (const name of [ - 'orca-cloud-relay-regional-placement-enabled', - 'projects/120364513935/secrets/orca-cloud-relay-regional-placement-enabled' - ]) { - assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { - run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1') - }), { version: '1', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' }) - } -}) - -test('rejects a v1 reference that names another secret or a floating version', () => { - assert.throws(() => readRelayServingRegionalPlacementVersion(input, { - run: (args) => args[1] === 'services' - ? serving() - : v1Revision('projects/120364513935/secrets/some-other-secret', '1') - }), /secret reference is invalid/) - assert.throws(() => readRelayServingRegionalPlacementVersion(input, { - run: (args) => args[1] === 'services' - ? serving() - : v1Revision('orca-cloud-relay-regional-placement-enabled', 'latest') - }), /secret reference is invalid/) -}) - -test('falls back only when the service or setting is absent', () => { - const notFound = new Error('not found') - notFound.code = 'NOT_FOUND' - assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { - run: () => { throw notFound } - }), { version: '7', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' }) - assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { - run: (args) => args[1] === 'services' - ? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } } - : { spec: { containers: [{ env: [] }] } } - }), { version: '7', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' }) -}) - -test('classifies real absent-service stderr without weakening revision failures', () => { - const serviceArgs = ['run', 'services', 'describe', 'missing-service'] - const stderr = 'ERROR: (gcloud.run.services.describe) Cannot find service [missing-service]' - assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, stderr), 'NOT_FOUND') - assert.equal( - classifyRelayServiceDescribeFailure(['run', 'revisions', 'describe', 'missing-revision'], stderr), - 'GCLOUD_FAILED' - ) - assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, 'PERMISSION_DENIED'), 'GCLOUD_FAILED') -}) - -test('rejects ambiguous traffic, malformed references, and read failures', () => { - assert.throws(() => readRelayServingRegionalPlacementVersion(input, { - run: () => ({ - status: { traffic: [{ revisionName: 'a', percent: 50 }, { revisionName: 'b', percent: 50 }] } - }) - }), /exactly one revision/) - assert.throws(() => readRelayServingRegionalPlacementVersion(input, { - run: (args) => args[1] === 'services' - ? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } } - : revision('latest') - }), /secret reference is invalid/) - const denied = new Error('denied') - denied.code = 'GCLOUD_FAILED' - assert.throws(() => readRelayServingRegionalPlacementVersion(input, { - run: () => { throw denied } - }), denied) -}) - - -test('preserves the serving cohort including explicit disable across later Terraform plans', () => { - for (const value of ['0', '1', '17', '100']) { - const servingRevision = revision() - servingRevision.spec.containers[0].env.push({ name: 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT', value }) - assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { - run: (args) => args[1] === 'services' ? serving() : servingRevision - }), { version: '11', cohort_percent: value, reserve_placement: '', shadow_seat_feed_cells: '' }) - } -}) - -test('fails closed on malformed, secret-backed or duplicate cohorts rather than resetting them', () => { - const name = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT' - const cases = [ - [{ name, value: '101' }], [{ name, value: '-1' }], [{ name, value: '1.5' }], - [{ name, value: '' }], [{ name, value: '01' }], [{ name, value: 1 }], - [{ name, valueFrom: { secretKeyRef: { name: 'unexpected', key: '1' } } }], - [{ name, value: '1' }, { name, value: '2' }] - ] - for (const settings of cases) { - const servingRevision = revision() - servingRevision.spec.containers[0].env.push(...settings) - assert.throws(() => readRelayServingRegionalPlacementVersion(input, { - run: (args) => args[1] === 'services' ? serving() : servingRevision - }), /cohort is invalid/) - } -}) - -test('carries the deploy-owned placement and shadow cells, so an apply keeps them', () => { - const servingRevision = revision() - servingRevision.spec.containers[0].env.push( - { name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'on' }, - { name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: 'production-gce-c1,production-gce-c2' } - ) - assert.deepEqual(readRelayServingRegionalPlacementVersion(input, { - run: (args) => args[1] === 'services' ? serving() : servingRevision - }), { - version: '11', - cohort_percent: '0', - reserve_placement: 'on', - shadow_seat_feed_cells: 'production-gce-c1,production-gce-c2' - }) -}) - -test('fails closed on a malformed, secret-backed or duplicate placement or shadow-cell setting', () => { - const cases = [ - [{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'yes' }], - [{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'on' }, { name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'off' }], - [{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', valueFrom: { secretKeyRef: { name: 'x', key: '1' } } }], - [{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: 'c1;c2' }], - [{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: '' }] - ] - for (const settings of cases) { - const servingRevision = revision() - servingRevision.spec.containers[0].env.push(...settings) - assert.throws(() => readRelayServingRegionalPlacementVersion(input, { - run: (args) => args[1] === 'services' ? serving() : servingRevision - }), /is invalid/) - } -}) - -test('Terraform sets the carried settings only when the serving revision has them', () => { - const terraform = readFileSync(fileURLToPath(new URL('../../infra/terraform/relay.tf', import.meta.url)), 'utf8') - const block = /dynamic "env" \{([\s\S]*?)\n {6}\}/.exec(terraform)?.[1] ?? '' - assert.match(block, /ORCA_RELAY_RESERVE_PLACEMENT *= data\.external\.relay_serving_regional_placement_version\.result\.reserve_placement/) - assert.match(block, /ORCA_RELAY_SHADOW_SEAT_FEED_CELLS *= data\.external\.relay_serving_regional_placement_version\.result\.shadow_seat_feed_cells/) - assert.match(block, /if value != ""/) -}) diff --git a/cloud/dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs b/cloud/dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs deleted file mode 100644 index 0f5845a732c..00000000000 --- a/cloud/dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs +++ /dev/null @@ -1,137 +0,0 @@ -import assert from 'node:assert/strict' -import { spawn } from 'node:child_process' -import { mkdtempSync, readFileSync, rmSync } from 'node:fs' -import { createServer } from 'node:http' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { test } from 'node:test' -import { fileURLToPath } from 'node:url' -import { relayWorkflowUrl } from './relay-repository.mjs' - -const WORKFLOWS = [ - 'deploy-relay-production-same-cap-job.yml', - 'operate-relay-production-rehome-job.yml' -] - -function workflow(name) { - return readFileSync(fileURLToPath(relayWorkflowUrl(name)), 'utf8') -} - -const VERIFY_STEP = 'name: Verify new incarnation, exact image, protocol, and durable safety' -const STALL_WINDOW = 'local out="${RUNNER_TEMP}/$1.json" window=60 delay=5 status' - -function verifyStep() { - return workflow('deploy-relay-production-same-cap-job.yml') - .split(VERIFY_STEP)[1] - .split('\n - name:')[0] -} - -function verifyAdminPost() { - const step = verifyStep() - const start = step.indexOf('admin_post() {') - return step.slice(start, step.indexOf('\n }', start) + '\n }'.length) -} - -// curl 7.81 (the Ubuntu 22.04 runners) never retries under --fail-with-body, so the -// remaining --retry flags are inert and only the verify loop below really retries. -// What holds for every admin curl: it is bounded and never retries a final 4xx. -test('every admin curl is bounded and never retries a final 4xx', () => { - for (const name of WORKFLOWS) { - for (const invocation of workflow(name).split(/\bcurl\b/).slice(1)) { - const flags = invocation.split('\n }')[0] - assert.match(flags, /--max-time 30/, name) - // --retry-all-errors would also retry 401, 403, and 409, which are final. - assert.doesNotMatch(flags, /--retry-all-errors/, name) - } - } -}) - -test('every retried admin request captures only the final attempt body', () => { - const job = workflow('deploy-relay-production-same-cap-job.yml') - // --fail-with-body writes every failed attempt to stdout, so a retried - // request must land in a file curl truncates per attempt. - assert.match(job, /--output "\$\{out\}"/) - assert.equal(job.split('admin_post() {').length - 1, 2) - for (const call of [ - /CURRENT_RUNTIME="\$\(admin_post current-runtime/, - /CURRENT_DIRECTOR_STATUS="\$\(admin_post current-cell-status/, - /TARGET_RUNTIME="\$\(admin_post target-runtime/, - /TARGET_DIRECTOR_STATUS="\$\(admin_post target-cell-status/ - ]) assert.match(job, call) - // The verify loop captures only the status code; its body goes to the file. - assert.doesNotMatch(job.replace(verifyAdminPost(), ''), /\$\(curl /) -}) - -// c18, 2026-10-07 18:49Z: a DB stall made the fresh cell answer 503 "no healthy upstream"; -// three 2 s retries gave up and the cell was healthy 16 s later. -// 60 s covers a 7 s stall, 16 s readiness grace and two 10 s LB health checks. -test('the post-roll verify outlasts one DB stall and its health-check recovery', () => { - assert.ok(verifyAdminPost().includes(STALL_WINDOW)) - assert.match(verifyAdminPost(), /--max-time 30/) -}) - -// Runs the step's own admin_post, with its retry clock scaled down twelve-fold. -async function runVerifyAdminPost(respond) { - const scaled = verifyAdminPost().replace('window=60 delay=5', 'window=5 delay=1') - assert.notEqual(scaled, verifyAdminPost()) - let calls = 0 - const startedAt = Date.now() - const server = createServer((request, response) => { - calls += 1 - const { status, body, drop } = respond(Date.now() - startedAt) - if (drop) return void request.socket.destroy() - response.writeHead(status, { 'content-type': 'application/json' }) - response.end(body) - }) - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) - const runnerTemp = mkdtempSync(join(tmpdir(), 'relay-verify-admin-post-')) - try { - const url = `http://127.0.0.1:${server.address().port}/v1/admin/runtime-status` - const child = spawn( - 'bash', - ['-euo', 'pipefail', '-c', `${scaled}\nadmin_post target-runtime "${url}" '{"v":1}'`], - { env: { ...process.env, RUNNER_TEMP: runnerTemp, ORCA_RELAY_ADMIN_ID_TOKEN: 't' } } - ) - let stdout = '' - let stderr = '' - child.stdout.on('data', (chunk) => { stdout += chunk }) - child.stderr.on('data', (chunk) => { stderr += chunk }) - const code = await new Promise((resolve) => child.on('close', resolve)) - return { code, stdout, stderr, calls, elapsedMs: Date.now() - startedAt } - } finally { - server.close() - rmSync(runnerTemp, { recursive: true, force: true }) - } -} - -const UNHEALTHY = { status: 503, body: 'no healthy upstream' } - -test('the verify read recovers when the cell comes back inside the retry window', async () => { - const result = await runVerifyAdminPost((elapsedMs) => - elapsedMs < 2_500 ? UNHEALTHY : { status: 200, body: '{"ok":true}' } - ) - assert.equal(result.code, 0, result.stderr) - assert.equal(result.stdout, '{"ok":true}') - // The old three-retry budget would have given up here. - assert.ok(result.calls > 3, `calls ${result.calls}`) -}) - -test('the verify read still fails a cell that stays down past the window', async () => { - const result = await runVerifyAdminPost(() => UNHEALTHY) - assert.notEqual(result.code, 0) - assert.ok(result.elapsedMs >= 3_000, `gave up after ${result.elapsedMs} ms: ${result.stderr}`) - assert.ok(result.elapsedMs < 8_000, `bounded at ${result.elapsedMs} ms`) -}) - -test('the verify read fails a final 4xx without waiting out the window', async () => { - const result = await runVerifyAdminPost(() => ({ status: 409, body: '{"error":"generation"}' })) - assert.notEqual(result.code, 0) - assert.equal(result.calls, 1) -}) - -test('a final connection failure does not report the previous attempt body', async () => { - const result = await runVerifyAdminPost((elapsedMs) => (elapsedMs < 1_000 ? UNHEALTHY : { drop: true })) - assert.notEqual(result.code, 0) - assert.match(result.stderr, /admin_post target-runtime: HTTP 000/) - assert.doesNotMatch(result.stderr, /no healthy upstream/) -}) diff --git a/cloud/dev/scripts/relay-admin-transient-retry.mjs b/cloud/dev/scripts/relay-admin-transient-retry.mjs deleted file mode 100644 index 9995be96a8f..00000000000 --- a/cloud/dev/scripts/relay-admin-transient-retry.mjs +++ /dev/null @@ -1,29 +0,0 @@ -// A single transient 5xx (load-balancer warm-up behind a fresh instance) must not fail a -// deploy step. 4xx is never retried: auth and generation-mismatch answers are final. -const TRANSIENT_STATUSES = [500, 502, 503, 504] -const RETRY_DELAY_MS = 2_000 -const REQUEST_TIMEOUT_MS = 30_000 - -export function isTransientAdminStatus(status) { - return TRANSIENT_STATUSES.includes(status) -} - -// Each attempt gets its own timeout budget, so a reused signal cannot abort the retry. -export async function fetchAdminOnceMore(fetchImpl, url, init, overrides = {}) { - const wait = overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) - const timeoutMs = overrides.timeoutMs ?? REQUEST_TIMEOUT_MS - const retryDelayMs = overrides.retryDelayMs ?? RETRY_DELAY_MS - const attempt = async () => - await fetchImpl(url, { ...init, signal: AbortSignal.timeout(timeoutMs) }) - let response - try { - response = await attempt() - } catch { - await wait(retryDelayMs) - return await attempt() - } - if (!isTransientAdminStatus(response.status)) return response - await response.arrayBuffer?.().catch(() => undefined) - await wait(retryDelayMs) - return await attempt() -} diff --git a/cloud/dev/scripts/relay-admin-transient-retry.test.mjs b/cloud/dev/scripts/relay-admin-transient-retry.test.mjs deleted file mode 100644 index ec041084344..00000000000 --- a/cloud/dev/scripts/relay-admin-transient-retry.test.mjs +++ /dev/null @@ -1,130 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' - -const url = 'https://relay.onorca.dev/v1/admin/cell-status' -const init = { method: 'POST', body: '{"v":1}' } - -function recordingWait(waits) { - return async (ms) => { waits.push(ms) } -} - -test('a single transient 5xx is retried and the second answer is returned', async () => { - const waits = [] - const statuses = [503, 200] - let calls = 0 - const response = await fetchAdminOnceMore( - async () => { - calls += 1 - const status = statuses.shift() - return new Response(JSON.stringify({ ok: status === 200 }), { status }) - }, - url, - init, - { wait: recordingWait(waits) } - ) - assert.equal(calls, 2) - assert.equal(response.status, 200) - assert.deepEqual(waits, [2_000]) - assert.deepEqual(await response.json(), { ok: true }) -}) - -test('a connection failure is retried and the second answer is returned', async () => { - const waits = [] - let calls = 0 - const response = await fetchAdminOnceMore( - async () => { - calls += 1 - if (calls === 1) throw new TypeError('fetch failed') - return Response.json({ ok: true }) - }, - url, - init, - { wait: recordingWait(waits) } - ) - assert.equal(calls, 2) - assert.equal(response.status, 200) - assert.deepEqual(waits, [2_000]) -}) - -test('two transient failures surface the second answer without a third attempt', async () => { - let calls = 0 - const response = await fetchAdminOnceMore( - async () => { - calls += 1 - return new Response('down', { status: 503 }) - }, - url, - init, - { wait: async () => {} } - ) - assert.equal(calls, 2) - assert.equal(response.status, 503) -}) - -test('two connection failures rethrow the second error', async () => { - let calls = 0 - await assert.rejects( - fetchAdminOnceMore( - async () => { - calls += 1 - throw new TypeError(`fetch failed ${calls}`) - }, - url, - init, - { wait: async () => {} } - ), - /fetch failed 2/ - ) - assert.equal(calls, 2) -}) - -test('4xx is final: auth and generation-mismatch answers are never retried', async () => { - for (const status of [400, 401, 403, 404, 409, 429]) { - let calls = 0 - const response = await fetchAdminOnceMore( - async () => { - calls += 1 - return new Response('no', { status }) - }, - url, - init, - { wait: async () => { throw new Error('must not wait') } } - ) - assert.equal(calls, 1, `status ${status} must not be retried`) - assert.equal(response.status, status) - } -}) - -test('each attempt carries its own unexpired timeout signal', async () => { - const signals = [] - await fetchAdminOnceMore( - async (_url, attemptInit) => { - signals.push(attemptInit.signal) - return new Response('down', { status: 502 }) - }, - url, - init, - { wait: async () => {}, timeoutMs: 30_000 } - ) - assert.equal(signals.length, 2) - assert.notEqual(signals[0], signals[1]) - assert.equal(signals[1].aborted, false) -}) - -test('the caller init is forwarded unchanged apart from the signal', async () => { - let seen - await fetchAdminOnceMore( - async (seenUrl, attemptInit) => { - seen = { seenUrl, attemptInit } - return Response.json({}) - }, - url, - { method: 'POST', headers: { authorization: 'Bearer t' }, body: '{"v":1}' }, - { wait: async () => {} } - ) - assert.equal(seen.seenUrl, url) - assert.equal(seen.attemptInit.method, 'POST') - assert.deepEqual(seen.attemptInit.headers, { authorization: 'Bearer t' }) - assert.equal(seen.attemptInit.body, '{"v":1}') -}) diff --git a/cloud/dev/scripts/relay-admission-selector.mjs b/cloud/dev/scripts/relay-admission-selector.mjs deleted file mode 100644 index 9b993fdc9ae..00000000000 --- a/cloud/dev/scripts/relay-admission-selector.mjs +++ /dev/null @@ -1,280 +0,0 @@ -import { createHash } from 'node:crypto' - -const STATES = ['existing-only', 'migration-only', 'general'] - -function normalizeMembership(input) { - const membership = { - existingOnly: [...input.existingOnly].sort(), - migrationOnly: [...input.migrationOnly].sort(), - general: [...input.general].sort() - } - const all = [...membership.existingOnly, ...membership.migrationOnly, ...membership.general] - if (new Set(all).size !== all.length) throw new Error('selector membership contains duplicates') - return membership -} - -function encodedMembership(membership) { - return JSON.stringify(normalizeMembership(membership)) -} - -function membershipSha256(membership) { - return createHash('sha256').update(encodedMembership(membership)).digest('hex') -} - -function normalizeMigrationCells(input) { - const cells = [...input] - .map((cell) => ({ - cellId: cell.cellId, - cellUrl: cell.cellUrl, - capacityRequests: cell.capacityRequests, - ...(cell.region ? { region: cell.region } : {}), - connectionHardCap: cell.connectionHardCap, - connectionUnobservedBound: cell.connectionUnobservedBound - })) - .sort((left, right) => left.cellId.localeCompare(right.cellId)) - if ( - cells.length === 0 || - new Set(cells.map(({ cellId }) => cellId)).size !== cells.length || - new Set(cells.map(({ cellUrl }) => cellUrl)).size !== cells.length - ) { - throw new Error('migration cell registration must contain distinct cells') - } - return cells -} - -function membershipWithMigrationCells(membership, cells) { - const known = new Set([ - ...membership.existingOnly, - ...membership.migrationOnly, - ...membership.general - ]) - if (cells.some(({ cellId }) => known.has(cellId))) { - throw new Error('migration cell registration contains an existing selector cell') - } - return normalizeMembership({ - existingOnly: membership.existingOnly, - migrationOnly: [...membership.migrationOnly, ...cells.map(({ cellId }) => cellId)], - general: membership.general - }) -} - -function assertSelector(value) { - if ( - !value || - !Number.isSafeInteger(value.generation) || - value.generation < 0 || - !value.membership - ) { - throw new Error('director returned an invalid admission selector') - } - return { - generation: value.generation, - attemptId: value.attemptId ?? null, - membership: normalizeMembership(value.membership) - } -} - -export function selectorAttemptId(expectedGeneration, membership) { - const digest = createHash('sha256') - .update(`${expectedGeneration}:${encodedMembership(membership)}`) - .digest('hex') - .slice(0, 24) - return `selector_${expectedGeneration}_${digest}` -} - -export function membershipWithStates(selector, states) { - const byCell = new Map() - for (const [state, key] of [ - ['existing-only', 'existingOnly'], - ['migration-only', 'migrationOnly'], - ['general', 'general'] - ]) { - for (const cellId of selector.membership[key]) byCell.set(cellId, state) - } - for (const [cellId, state] of Object.entries(states)) { - if (!byCell.has(cellId)) throw new Error(`selector does not contain ${cellId}`) - if (!STATES.includes(state)) throw new Error(`invalid admission state for ${cellId}`) - if (byCell.get(cellId) === 'existing-only' && state !== 'existing-only') { - throw new Error(`selector cannot re-enable existing-only cell ${cellId}`) - } - byCell.set(cellId, state) - } - return normalizeMembership({ - existingOnly: [...byCell].filter(([, state]) => state === 'existing-only').map(([id]) => id), - migrationOnly: [...byCell].filter(([, state]) => state === 'migration-only').map(([id]) => id), - general: [...byCell].filter(([, state]) => state === 'general').map(([id]) => id) - }) -} - -export async function inspectAdmissionSelector(post, attemptId) { - const result = await post('/v1/admin/admission-selector/status', { - v: 1, - ...(attemptId ? { attemptId } : {}) - }) - return { - selector: assertSelector(result.selector), - intent: result.intent - ? { - ...result.intent, - previousMembership: result.intent.previousMembership - ? normalizeMembership(result.intent.previousMembership) - : undefined, - membership: normalizeMembership(result.intent.membership) - } - : null - } -} - -function exactSelector(actual, expected) { - return ( - actual.generation === expected.generation && - encodedMembership(actual.membership) === encodedMembership(expected.membership) - ) -} - -export async function applyExactAdmissionSelector(post, membership, options = {}) { - const before = await inspectAdmissionSelector(post) - const desired = normalizeMembership(membership) - if ( - options.expectedCurrentSelector && - !exactSelector(before.selector, options.expectedCurrentSelector) - ) { - throw new Error('admission selector changed before exact apply') - } - if (options.requireBoundary !== false && before.selector.generation < 1) { - throw new Error('admission selector boundary is not active') - } - if (encodedMembership(before.selector.membership) === encodedMembership(desired)) { - return { changed: false, selector: before.selector } - } - const attemptId = - options.attemptId ?? selectorAttemptId(before.selector.generation, desired) - const expected = { - generation: before.selector.generation + 1, - membership: desired - } - let result - try { - result = await post('/v1/admin/admission-selector/apply', { - v: 1, - attemptId, - expectedGeneration: before.selector.generation, - ...(before.selector.generation === 0 - ? { expectedMembershipSha256: membershipSha256(before.selector.membership) } - : {}), - // Only a same-cap roll's isolate passes this; every other caller omits it - // and leaves the cell's hosts pinned, which is the pre-existing behaviour. - ...(options.rollIsolatedCells ? { rollIsolatedCells: options.rollIsolatedCells } : {}), - membership: desired - }) - } catch (error) { - const inspected = await inspectAdmissionSelector(post, attemptId) - if ( - inspected.intent?.state === 'committed' && - exactSelector(inspected.selector, expected) - ) { - return { changed: true, selector: inspected.selector, recovered: true } - } - if ( - inspected.intent?.state === 'unchanged' && - exactSelector(inspected.selector, before.selector) - ) { - throw new Error('admission selector apply remained unchanged after an ambiguous response', { - cause: error - }) - } - throw new Error('admission selector apply diverged after an ambiguous response', { - cause: error - }) - } - const applied = assertSelector(result.selector) - if (!exactSelector(applied, expected)) { - throw new Error('admission selector apply returned unexpected membership') - } - const verified = await inspectAdmissionSelector(post, attemptId) - if ( - verified.intent?.state !== 'committed' || - !exactSelector(verified.selector, expected) - ) { - throw new Error('admission selector commit could not be verified') - } - return { changed: result.changed === true, selector: verified.selector } -} - -export async function addExactMigrationCells(post, input, options = {}) { - const cells = normalizeMigrationCells(input.cells) - const attemptId = input.attemptId - if (!/^[A-Za-z0-9_-]{8,128}$/.test(attemptId ?? '')) { - throw new Error('migration cell registration requires an exact attempt ID') - } - const before = await inspectAdmissionSelector(post, attemptId) - let expectedGeneration - let expectedMembership - if (before.intent) { - expectedGeneration = before.intent.expectedGeneration - expectedMembership = normalizeMembership(before.intent.membership) - } else { - if (before.selector.generation < 1) { - throw new Error('admission selector boundary is not active') - } - if ( - options.expectedCurrentSelector && - !exactSelector(before.selector, options.expectedCurrentSelector) - ) { - throw new Error('admission selector changed before cell registration') - } - expectedGeneration = before.selector.generation - expectedMembership = membershipWithMigrationCells(before.selector.membership, cells) - } - const expected = { - generation: expectedGeneration + 1, - membership: expectedMembership - } - let result - try { - result = await post('/v1/admin/admission-selector/add-migration-cells', { - v: 1, - attemptId, - expectedGeneration, - cells - }) - } catch (error) { - const inspected = await inspectAdmissionSelector(post, attemptId) - if ( - !before.intent && - inspected.intent?.state === 'committed' && - exactSelector(inspected.selector, expected) - ) { - return { changed: true, selector: inspected.selector, recovered: true } - } - throw new Error('migration cell registration did not commit exactly', { cause: error }) - } - const applied = assertSelector(result.selector) - if (!exactSelector(applied, expected)) { - throw new Error('migration cell registration returned unexpected membership') - } - const verified = await inspectAdmissionSelector(post, attemptId) - if (verified.intent?.state !== 'committed' || !exactSelector(verified.selector, expected)) { - throw new Error('migration cell registration commit could not be verified') - } - return { changed: result.changed === true, selector: verified.selector } -} - -export async function transitionAdmissionSelector(post, states, options = {}) { - const current = await inspectAdmissionSelector(post) - if (current.selector.generation < 1) { - throw new Error('admission selector boundary is not active') - } - return await applyExactAdmissionSelector( - post, - membershipWithStates(current.selector, states), - options - ) -} - -export function selectorCellState(selector, cellId) { - if (selector.membership.existingOnly.includes(cellId)) return 'existing-only' - if (selector.membership.migrationOnly.includes(cellId)) return 'migration-only' - if (selector.membership.general.includes(cellId)) return 'general' - throw new Error(`selector does not contain ${cellId}`) -} diff --git a/cloud/dev/scripts/relay-admission-selector.test.mjs b/cloud/dev/scripts/relay-admission-selector.test.mjs deleted file mode 100644 index b45df5ac34b..00000000000 --- a/cloud/dev/scripts/relay-admission-selector.test.mjs +++ /dev/null @@ -1,232 +0,0 @@ -import assert from 'node:assert/strict' -import { createHash } from 'node:crypto' -import { test } from 'node:test' -import { - addExactMigrationCells, - applyExactAdmissionSelector, - membershipWithStates, - selectorAttemptId, - transitionAdmissionSelector -} from './relay-admission-selector.mjs' - -const initialMembership = { - existingOnly: ['legacy'], - migrationOnly: ['target'], - general: ['general'] -} - -function selectorHarness({ ambiguous = null, generation = 1 } = {}) { - let selector = { generation, attemptId: 'initial', membership: initialMembership } - const intents = new Map() - const requests = [] - let applies = 0 - const post = async (path, body) => { - if (path.endsWith('/status')) { - return { - selector, - intent: body.attemptId ? intents.get(body.attemptId) ?? null : null - } - } - applies++ - requests.push(body) - const before = selector - const committed = { - generation: body.expectedGeneration + 1, - attemptId: body.attemptId, - membership: body.membership - } - intents.set(body.attemptId, { - attemptId: body.attemptId, - expectedGeneration: body.expectedGeneration, - intendedGeneration: committed.generation, - previousMembership: before.membership, - membership: body.membership, - state: ambiguous === 'unchanged' ? 'unchanged' : 'committed' - }) - if (ambiguous !== 'unchanged') selector = committed - if (ambiguous) throw new Error('lost selector response') - return { changed: true, selector } - } - return { post, selector: () => selector, applies: () => applies, requests } -} - -test('derives deterministic attempts and applies exact selector transitions', async () => { - const harness = selectorHarness() - const desired = membershipWithStates(harness.selector(), { target: 'general' }) - assert.equal( - selectorAttemptId(1, desired), - selectorAttemptId(1, { - existingOnly: ['legacy'], - migrationOnly: [], - general: ['target', 'general'] - }) - ) - const result = await transitionAdmissionSelector(harness.post, { target: 'general' }) - assert.equal(result.selector.generation, 2) - assert.deepEqual(result.selector.membership.general, ['general', 'target']) -}) - -test('accepts only an exact committed result after an ambiguous response', async () => { - const harness = selectorHarness({ ambiguous: 'committed' }) - const result = await applyExactAdmissionSelector(harness.post, { - existingOnly: ['legacy', 'target'], - migrationOnly: [], - general: ['general'] - }) - assert.equal(result.recovered, true) - assert.equal(harness.applies(), 1) - assert.equal(result.selector.generation, 2) -}) - -test('binds a generation-zero cutover to the inspected membership', async () => { - const harness = selectorHarness({ generation: 0 }) - await applyExactAdmissionSelector( - harness.post, - { - existingOnly: ['legacy', 'target'], - migrationOnly: [], - general: ['general'] - }, - { requireBoundary: false } - ) - assert.equal( - harness.requests[0].expectedMembershipSha256, - createHash('sha256').update(JSON.stringify(initialMembership)).digest('hex') - ) -}) - -test('stops on an unchanged ambiguous result without replaying', async () => { - const harness = selectorHarness({ ambiguous: 'unchanged' }) - await assert.rejects( - applyExactAdmissionSelector(harness.post, { - existingOnly: ['legacy', 'target'], - migrationOnly: [], - general: ['general'] - }), - /remained unchanged/ - ) - assert.equal(harness.applies(), 1) - assert.equal(harness.selector().generation, 1) -}) - -test('never restores an existing-only cell', () => { - assert.throws( - () => membershipWithStates({ membership: initialMembership }, { legacy: 'general' }), - /cannot re-enable/ - ) -}) - -test('refuses an exact apply after the inspected selector changes', async () => { - const harness = selectorHarness() - await assert.rejects( - applyExactAdmissionSelector( - harness.post, - { - existingOnly: ['legacy', 'target'], - migrationOnly: [], - general: ['general'] - }, - { - expectedCurrentSelector: { - generation: 0, - membership: { - existingOnly: ['target'], - migrationOnly: [], - general: ['general', 'legacy'] - } - } - } - ), - /changed before exact apply/ - ) - assert.equal(harness.applies(), 0) -}) - -test('adds exact migration cells and recovers a committed response loss', async () => { - let selector = { generation: 1, attemptId: 'initial', membership: initialMembership } - const intents = new Map() - let additions = 0 - const post = async (path, body) => { - if (path.endsWith('/status')) { - return { - selector, - intent: body.attemptId ? intents.get(body.attemptId) ?? null : null - } - } - additions++ - selector = { - generation: body.expectedGeneration + 1, - attemptId: body.attemptId, - membership: { - ...selector.membership, - migrationOnly: [ - ...selector.membership.migrationOnly, - ...body.cells.map(({ cellId }) => cellId) - ].sort() - } - } - intents.set(body.attemptId, { - attemptId: body.attemptId, - expectedGeneration: body.expectedGeneration, - intendedGeneration: selector.generation, - membership: selector.membership, - state: 'committed' - }) - throw new Error('lost cell registration response') - } - const result = await addExactMigrationCells(post, { - attemptId: 'add_cells_exact', - cells: [ - { - cellId: 'target-2', - cellUrl: 'https://target-2.example.com', - capacityRequests: 4_000, - connectionHardCap: 600, - connectionUnobservedBound: 60 - } - ] - }) - assert.equal(result.recovered, true) - assert.equal(additions, 1) - assert.deepEqual(result.selector.membership.migrationOnly, ['target', 'target-2']) -}) - -test('does not recover an attempt owned by another selector operation', async () => { - const selector = { - generation: 2, - attemptId: 'selector_collision', - membership: initialMembership - } - const post = async (path, body) => { - if (path.endsWith('/status')) { - return { - selector, - intent: body.attemptId - ? { - attemptId: body.attemptId, - expectedGeneration: 1, - intendedGeneration: 2, - membership: initialMembership, - state: 'committed' - } - : null - } - } - throw new Error('admission_selector_attempt_mismatch') - } - await assert.rejects( - addExactMigrationCells(post, { - attemptId: 'selector_collision', - cells: [ - { - cellId: 'target-2', - cellUrl: 'https://target-2.example.com', - capacityRequests: 4_000, - connectionHardCap: 600, - connectionUnobservedBound: 60 - } - ] - }), - /did not commit exactly/ - ) -}) diff --git a/cloud/dev/scripts/relay-asia-cloud-sql-metrics.mjs b/cloud/dev/scripts/relay-asia-cloud-sql-metrics.mjs deleted file mode 100644 index 489402b0399..00000000000 --- a/cloud/dev/scripts/relay-asia-cloud-sql-metrics.mjs +++ /dev/null @@ -1,33 +0,0 @@ -import { spawnSync } from 'node:child_process' - -function accessToken() { - const result = spawnSync('gcloud', ['auth', 'print-access-token'], { - encoding: 'utf8', timeout: 30_000 - }) - const token = result.stdout.trim() - if (result.status !== 0 || token.length < 20) { - throw new Error('Google access token is unavailable') - } - return token -} - -export async function readCloudSqlBackends(environment, startedAt, endedAt) { - const production = environment === 'production' - if (!production && environment !== 'staging') throw new Error('Cloud SQL environment is invalid') - const project = production ? 'onorca-cloud' : 'onorca-cloud-staging' - const instance = production ? 'orca-cloud-auth-db' : 'orca-cloud-staging-auth-db' - const url = new URL(`https://monitoring.googleapis.com/v3/projects/${project}/timeSeries`) - url.searchParams.set('filter', `metric.type = "cloudsql.googleapis.com/database/postgresql/num_backends" AND resource.labels.database_id = "${project}:${instance}"`) - url.searchParams.set('interval.startTime', startedAt) - url.searchParams.set('interval.endTime', endedAt) - url.searchParams.set('aggregation.alignmentPeriod', '60s') - url.searchParams.set('aggregation.perSeriesAligner', 'ALIGN_MAX') - url.searchParams.set('aggregation.crossSeriesReducer', 'REDUCE_MAX') - url.searchParams.set('view', 'FULL') - const response = await fetch(url, { - headers: { authorization: `Bearer ${accessToken()}` }, - redirect: 'error', signal: AbortSignal.timeout(30_000) - }) - if (!response.ok) throw new Error(`Cloud SQL metric query returned ${response.status}`) - return await response.json() -} diff --git a/cloud/dev/scripts/relay-asia-rollout-evidence.mjs b/cloud/dev/scripts/relay-asia-rollout-evidence.mjs deleted file mode 100644 index 850a9c98f11..00000000000 --- a/cloud/dev/scripts/relay-asia-rollout-evidence.mjs +++ /dev/null @@ -1,639 +0,0 @@ -import { readFileSync, writeFileSync } from 'node:fs' -import { fileURLToPath } from 'node:url' -import { readCloudSqlBackends } from './relay-asia-cloud-sql-metrics.mjs' -import { RELAY_GITHUB_REPOSITORY, relayWorkflowPath } from './relay-repository.mjs' - -const REPOSITORY = RELAY_GITHUB_REPOSITORY -const ADMISSION_WORKFLOW = relayWorkflowPath('operate-relay-asia-admission.yml') -const STAGING_WORKFLOW = relayWorkflowPath('prove-relay-asia-staging.yml') -const STAGING_CELL = 'staging-gce-c4' -const C27 = 'production-gce-c27' -const ASIA_REGION = 'asia-east2' -const US_REGION = 'us-central1' -// Each canary proves its own cell under production load; C28/C29 promotion consumes only C27's. -const PRODUCTION_CANARIES = { - [C27]: { kind: 'production-c27-canary', origin: 'https://c27.relay.onorca.dev', region: ASIA_REGION }, - 'production-gce-c30': { - kind: 'production-c30-canary', origin: 'https://c30.relay.onorca.dev', region: ASIA_REGION - }, - 'production-gce-c31': { - kind: 'production-c31-canary', origin: 'https://c31.relay.onorca.dev', region: ASIA_REGION - }, - 'production-gce-c32': { - kind: 'production-c32-canary', origin: 'https://c32.relay.onorca.dev', region: US_REGION - }, - 'production-gce-c33': { - kind: 'production-c33-canary', origin: 'https://c33.relay.onorca.dev', region: US_REGION - }, - 'production-gce-c34': { - kind: 'production-c34-canary', origin: 'https://c34.relay.onorca.dev', region: ASIA_REGION - }, - 'production-gce-c35': { - kind: 'production-c35-canary', origin: 'https://c35.relay.onorca.dev', region: ASIA_REGION - } -} -const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/ -const SHA_PATTERN = /^[a-f0-9]{40}$/ -const MAX_LOG_EDGE_GAP_MS = 120_000 -const MAX_LOG_SAMPLE_GAP_MS = 120_000 -const CLOUD_SQL_LIMIT = 320 -const CANARY_MINIMUM_MS = 5 * 60_000 -const GENERATOR_CPU_PERCENT_LIMIT = 80 -const GENERATOR_EVENT_LOOP_P99_MS_LIMIT = 100 -const GENERATOR_RSS_GROWTH_MIB_LIMIT = 512 -const DATABASE_POOL_TRANSIENT_WAITERS_MAX = 4 -const DATABASE_POOL_TRANSIENT_WAIT_MS_MAX = 50 - -function object(value, label) { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`${label} is invalid`) - } - return value -} - -function positiveInteger(value, label) { - const number = Number(value) - if (!Number.isSafeInteger(number) || number < 1) throw new Error(`${label} is invalid`) - return number -} - -function instant(value, label) { - const date = new Date(value) - if (!Number.isFinite(date.valueOf())) throw new Error(`${label} is invalid`) - return date -} - -function exactCells(actual, expected) { - return Array.isArray(actual) && - JSON.stringify([...actual].sort()) === JSON.stringify([...expected].sort()) -} - -function source(input, environment, workflow) { - if (input.repository !== REPOSITORY) throw new Error('repository is invalid') - if (!SHA_PATTERN.test(input.commitSha)) throw new Error('commit SHA is invalid') - return { - repository: input.repository, - workflow, - environment, - runId: positiveInteger(input.runId, 'run ID'), - runAttempt: positiveInteger(input.runAttempt, 'run attempt'), - commitSha: input.commitSha - } -} - -function baseEvidence(input, environment, cells, workflow = ADMISSION_WORKFLOW) { - if (!DIGEST_PATTERN.test(input.imageDigest)) throw new Error('image digest is invalid') - return { - version: 1, - source: source(input, environment, workflow), - imageDigest: input.imageDigest, - topology: { - cellIds: cells, - selectorGeneration: positiveInteger(input.selectorGeneration, 'selector generation') - } - } -} - -export function buildStagingEvidence(input) { - const start = instant(input.startedAt, 'staging proof start') - const end = instant(input.endedAt, 'staging proof end') - const launch = loadReports(input.launchReport, 'launch load report') - const expectedLoad = { - controls: 20, splices: 20, slowReaders: 4, wedgedReaders: 1, - minimumSeconds: 210 - } - assertLoadReports(launch, expectedLoad) - const metrics = runtimeMetrics(input.logs, start, end, STAGING_CELL) - assertPassingRuntimeMetrics(metrics, 'staging proof') - // Staging is quiet, so its one intentional sticky fallback must be the only one. - if ( - number(metrics.regionFallbacks, 'regionFallbacks') !== 1 || - number(metrics.usRegionFallbacks, 'usRegionFallbacks') !== 1 - ) throw new Error('staging proof regionFallbacks did not match the intentional probes') - const minimumConcurrentSplices = expectedLoad.splices - expectedLoad.wedgedReaders - if ( - metrics.targetControlsMax < expectedLoad.controls || - metrics.targetSplicesMax < minimumConcurrentSplices - ) { - throw new Error('staging launch load did not reach C4 at the reviewed levels') - } - metrics.cloudSqlBackendsMax = cloudSqlMaximum(input.cloudSql, start, end) - if (metrics.cloudSqlBackendsMax >= CLOUD_SQL_LIMIT) { - throw new Error(`Cloud SQL backends must remain below ${CLOUD_SQL_LIMIT}`) - } - return { - ...baseEvidence(input, 'staging', [STAGING_CELL], STAGING_WORKFLOW), - kind: 'staging-asia-readiness', - window: { startedAt: start.toISOString(), endedAt: end.toISOString() }, - load: { launch: loadSummary(launch) }, - metrics - } -} - -function loadReports(value, label) { - if (!Array.isArray(value) || value.length < 2) throw new Error(`${label} must be sharded`) - return value.map((report) => object(report, label)) -} - -function total(reports, key) { - return reports.reduce((sum, report) => sum + number(report[key], key), 0) -} - -function loadSummary(reports) { - return { - shards: reports.length, - controls: total(reports, 'controls'), - peakActive: total(reports, 'peakActive'), - steadyMinimumActive: total(reports, 'steadyMinimumActive'), - peakActiveSplices: total(reports, 'peakActiveSplices'), - completedSplices: total(reports, 'completedSplices'), - slowReaderSplicesCompleted: total(reports, 'slowReaderSplicesCompleted'), - wedgedReaderSplicesClosed: total(reports, 'wedgedReaderSplicesClosed'), - regionalFallbacksProved: total(reports, 'regionalFallbacksProved'), - oldClientUsFirstProved: total(reports, 'oldClientUsFirstProved'), - stickyAssignmentProved: total(reports, 'stickyAssignmentProved'), - requestUnitInvitesOpened: total(reports, 'requestUnitInvitesOpened'), - requestUnitPrincipalCounts: reports.map((report) => report.requestUnitPrincipalCount), - requestUnitOverflowReasons: - reports.map((report) => report.requestUnitOverflowReason).filter(Boolean), - requestUnitCleanupProved: total(reports, 'requestUnitCleanupProved'), - phaseBarrierPassed: reports.every((report) => report.phaseBarrierPassed === true), - rebindProbesOpened: total(reports, 'rebindProbesOpened'), - rebindOverflowReasons: reports.map((report) => report.rebindOverflowReason).filter(Boolean), - readerQueueEvidence: reports.flatMap((report) => report.readerQueueEvidence), - readerQueuedBytesPeak: Math.max(...reports.map((report) => report.readerQueuedBytesPeak)), - generatorCpuPercentMax: Math.max(...reports.map((report) => report.generatorCpuPercent)), - generatorEventLoopP99MsMax: Math.max( - ...reports.map((report) => report.generatorEventLoopP99Ms) - ), - generatorRssGrowthMiBMax: Math.max(...reports.map((report) => report.generatorRssGrowthMiB)), - configuredSteadySeconds: Math.min(...reports.map((report) => report.configuredSteadySeconds)) - } -} - -function assertLoadReports(reports, expected) { - const shardCount = reports.length - if ( - reports.some((report, index) => - report.event !== 'relay_load_complete' || - report.shardCount !== shardCount || report.shardIndex !== index || - report.relayAsiaLoadPrincipalCount !== 32 || - number(report.configuredSteadySeconds, 'staging steady seconds') < expected.minimumSeconds || - number(report.configuredSpliceHoldSeconds, 'staging splice hold seconds') < - expected.minimumSeconds || - report.requiredLeaseHorizons !== 2 || report.phaseBarrierPassed !== true - ) || - total(reports, 'controls') !== expected.controls - ) { - throw new Error('staging load profile does not match') - } - if ( - total(reports, 'peakActive') !== expected.controls || - total(reports, 'steadyMinimumActive') !== expected.controls - ) { - throw new Error('staging load did not sustain the required controls') - } - for (const key of [ - 'rampConnectionFailures', 'steadyConnectionFailures', 'transitionConnectionFailures', - 'unexpectedCloses', 'protocolErrors', 'refreshErrors', 'socketErrors', 'failedSplices' - ]) { - if (total(reports, key) !== 0) throw new Error(`staging load ${key} must be zero`) - } - const peakActiveSplices = total(reports, 'peakActiveSplices') - if ( - total(reports, 'configuredSplices') !== expected.splices || - total(reports, 'configuredSlowReaderSplices') !== expected.slowReaders || - total(reports, 'configuredWedgedReaderSplices') !== expected.wedgedReaders || - peakActiveSplices < expected.splices - expected.wedgedReaders || - peakActiveSplices > expected.splices || - total(reports, 'completedSplices') !== expected.splices - expected.wedgedReaders || - total(reports, 'slowReaderSplicesCompleted') !== expected.slowReaders || - total(reports, 'wedgedReaderSplicesClosed') !== expected.wedgedReaders - ) throw new Error('staging mixed load evidence does not match') - if ( - total(reports, 'regionalFallbacksProved') !== 0 || - total(reports, 'oldClientUsFirstProved') !== 1 || - total(reports, 'stickyAssignmentProved') !== 1 || - total(reports, 'requestUnitInvitesOpened') !== 0 || - reports.some((report) => report.requestUnitPrincipalCount !== 0) || - total(reports, 'requestUnitCleanupProved') !== 0 || - reports.some((report) => report.requestUnitOverflowReason !== null) || - total(reports, 'rebindProbesOpened') !== 2 || - reports.some((report) => report.rebindOverflowReason !== null) - ) throw new Error('staging launch-path evidence does not match') - const readerReports = reports.filter( - (report) => report.configuredSlowReaderSplices + report.configuredWedgedReaderSplices > 0 - ) - if (expected.slowReaders > 0 && readerReports.length !== 1) { - throw new Error('staging reader pressure must have one causal owner') - } - for (const report of reports) { - const queue = report.readerQueueEvidence - if (!Array.isArray(queue)) throw new Error('staging reader queue evidence is invalid') - if (expected.slowReaders === 0 && queue.length !== 0) { - throw new Error('control load unexpectedly contains reader queue evidence') - } - const ownsReaderPressure = readerReports.includes(report) - if (expected.slowReaders > 0 && ownsReaderPressure && ( - queue.length !== 1 || - queue[0]?.origin !== 'https://c4.relay-staging.onorca.dev' || - number(queue[0]?.baselineBytes, 'reader queue baseline') > - number(queue[0]?.peakBytes, 'reader queue peak') || - number(queue[0]?.increaseBytes, 'reader queue increase') <= 0 || - queue[0].peakBytes - queue[0].baselineBytes !== queue[0].increaseBytes || - report.readerQueuedBytesPeak !== queue[0].increaseBytes - )) throw new Error('staging reader queue evidence is not causal') - if (expected.slowReaders > 0 && !ownsReaderPressure && ( - queue.length !== 0 || report.readerQueuedBytesPeak !== 0 - )) throw new Error('non-owner shard contains reader queue evidence') - } - for (const report of reports) { - if ( - number(report.generatorCpuPercent, 'generator CPU') >= GENERATOR_CPU_PERCENT_LIMIT || - number(report.generatorEventLoopP99Ms, 'generator event loop') >= - GENERATOR_EVENT_LOOP_P99_MS_LIMIT || - number(report.generatorRssGrowthMiB, 'generator RSS growth') >= - GENERATOR_RSS_GROWTH_MIB_LIMIT - ) throw new Error('staging load generator has insufficient headroom') - const shutdown = object(report.shutdownEvidence, 'load shutdown evidence') - if ( - shutdown.peerShutdowns !== report.controls || shutdown.activeControls !== 0 || - shutdown.activeSplices !== 0 || shutdown.reconnectTimers !== 0 - ) throw new Error('staging load cleanup is incomplete') - } -} - -function number(value, label) { - if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { - throw new Error(`${label} is invalid`) - } - return value -} - -function sumMap(value, label) { - const entries = Object.entries(object(value ?? {}, label)) - return entries.reduce((total, [key, count]) => { - if (!key) throw new Error(`${label} is invalid`) - return total + number(count, label) - }, 0) -} - -function assertCoverage(timestamps, start, end, label) { - if (timestamps.length === 0) throw new Error(`${label} has no samples`) - const ordered = timestamps.map((value) => instant(value, `${label} timestamp`).valueOf()) - .sort((left, right) => left - right) - if ( - ordered[0] > start.valueOf() + MAX_LOG_EDGE_GAP_MS || - ordered.at(-1) < end.valueOf() - MAX_LOG_EDGE_GAP_MS - ) throw new Error(`${label} does not cover the canary window`) - if (ordered.some((timestamp, index) => index > 0 && timestamp - ordered[index - 1] > MAX_LOG_SAMPLE_GAP_MS)) { - throw new Error(`${label} has a sampling gap`) - } -} - -function pointValue(point) { - const value = point?.value - if (typeof value?.doubleValue === 'number') return value.doubleValue - if (typeof value?.int64Value === 'string') return Number(value.int64Value) - return NaN -} - -function cloudSqlMaximum(response, start, end) { - const points = (object(response, 'Cloud SQL response').timeSeries ?? []) - .flatMap((series) => series.points ?? []) - assertCoverage(points.map((point) => point.interval?.endTime), start, end, 'Cloud SQL metrics') - const values = points.map(pointValue) - if (values.some((value) => !Number.isFinite(value) || value < 0)) { - throw new Error('Cloud SQL backend metric is invalid') - } - return Math.max(...values) -} - -function productionCanary(cellId) { - const canary = PRODUCTION_CANARIES[cellId] - if (!canary) throw new Error('canary cell is not a reviewed production Asia canary') - return { ...canary, label: cellId.split('-').at(-1).toUpperCase() } -} - -export function buildProductionCanaryEvidence(input) { - const canary = productionCanary(input.cellId) - const start = instant(input.startedAt, 'canary start') - const end = instant(input.endedAt, 'canary end') - if (end.valueOf() - start.valueOf() < CANARY_MINIMUM_MS) { - throw new Error(`${canary.label} canary window is shorter than 5 minutes`) - } - const load = object(input.loadReport, `${canary.label} load report`) - assertCanaryLoad(load, input.cellId) - // Directors show a steady relay_cells lock and pool-wait baseline unrelated to the canary cell. - const metrics = runtimeMetrics(input.logs, start, end, input.cellId, { - gateDirectorDatabase: false, region: canary.region - }) - assertPassingRuntimeMetrics(metrics, `${canary.label} canary`, canary.region) - // Fallbacks are keyed by the host's target region. US-targeted ones come from unhinted or - // US-preferring hosts an Asia cell cannot cause, so they are recorded but not gated. A US - // cell gates neither: the US-targeted baseline is nonzero while the US fleet is full. - if ( - canary.region === ASIA_REGION && - number(metrics.asiaRegionFallbacks, 'asiaRegionFallbacks') !== 0 - ) { - throw new Error(`${canary.label} canary asiaRegionFallbacks must be zero`) - } - metrics.cloudSqlBackendsMax = cloudSqlMaximum(input.cloudSql, start, end) - assertPassingCanary(metrics, input.cellId) - return { - ...baseEvidence(input, 'production', [input.cellId]), - kind: canary.kind, - window: { startedAt: start.toISOString(), endedAt: end.toISOString() }, - load, - metrics - } -} - -function assertCanaryLoad(report, cellId) { - const { label, origin } = productionCanary(cellId) - if ( - report.event !== 'relay_load_complete' || - report.controls !== 1 || report.shardCount !== 1 || report.shardIndex !== 0 || - report.relayAsiaLoadPrincipalCount !== 1 || - number(report.configuredSteadySeconds, 'canary steady seconds') < 300 || - number(report.configuredSpliceHoldSeconds, 'canary splice hold seconds') < 60 || - number(report.requiredLeaseHorizons, 'canary lease horizons') < 2 || - report.peakActive !== 1 || report.steadyMinimumActive !== 1 || - report.configuredSplices !== 1 || report.peakActiveSplices !== 1 || - report.completedSplices !== 1 || report.failedSplices !== 0 - ) throw new Error(`${label} control and splice canary did not match`) - // Placement picks the least-loaded general cell in the canary's region, so the control lands on it. - if (JSON.stringify(report.assignedCellOrigins) !== JSON.stringify([origin])) { - throw new Error(`${label} canary load was not placed only on ${label}`) - } - for (const key of [ - 'connectionFailures', 'unexpectedCloses', 'protocolErrors', - 'refreshErrors', 'socketErrors' - ]) { - if (number(report[key], key) !== 0) throw new Error(`${label} canary ${key} must be zero`) - } - const shutdown = object(report.shutdownEvidence, `${label} load shutdown evidence`) - if ( - shutdown.peerShutdowns !== 1 || shutdown.activeControls !== 0 || - shutdown.activeSplices !== 0 || shutdown.reconnectTimers !== 0 - ) throw new Error(`${label} load cleanup is incomplete`) -} - -function runtimeMetrics( - logs, start, end, targetCellId, { gateDirectorDatabase = true, region = ASIA_REGION } = {} -) { - const entries = logs.map((entry) => object(entry, 'runtime metric entry')) - const directorEntries = entries.filter((entry) => entry.jsonPayload?.role === 'director') - const cellEntries = entries.filter((entry) => - entry.jsonPayload?.role === 'cell' && - entry.jsonPayload?.cellId === targetCellId && - entry.jsonPayload?.region === region - ) - assertCoverage(directorEntries.map((entry) => entry.timestamp), start, end, 'director metrics') - assertCoverage(cellEntries.map((entry) => entry.timestamp), start, end, `${targetCellId} metrics`) - const identifiedDirectors = Map.groupBy( - directorEntries.filter((entry) => entry.resource?.labels?.instance_id), - (entry) => entry.resource.labels.instance_id - ) - for (const [instanceId, instanceEntries] of identifiedDirectors) { - assertCoverage(instanceEntries.map((entry) => entry.timestamp), start, end, `director ${instanceId}`) - } - const directorPayloads = directorEntries.map((entry) => entry.jsonPayload) - const cellPayloads = cellEntries.map((entry) => entry.jsonPayload) - const placement = { - asiaSelections: directorPayloads.reduce( - (total, payload) => total + number(payload.selectedRegionsDelta?.['asia-east2'] ?? 0, 'Asia selections'), 0 - ), - regionFallbacks: directorPayloads.reduce( - (total, payload) => total + sumMap(payload.regionFallbacksDelta, 'region fallbacks'), 0 - ), - asiaRegionFallbacks: directorPayloads.reduce( - (total, payload) => total + number( - payload.regionFallbacksDelta?.['asia-east2'] ?? 0, - 'Asia region fallbacks' - ), 0 - ), - usRegionFallbacks: directorPayloads.reduce( - (total, payload) => total + number( - payload.regionFallbacksDelta?.['us-central1'] ?? 0, - 'US region fallbacks' - ), 0 - ), - unavailableRegions: directorPayloads.reduce( - (total, payload) => total + sumMap(payload.unavailableRegionsDelta, 'unavailable regions'), 0 - ), - // Only a US canary reads US selections, so Asia evidence keeps its exact field set. - ...(region === US_REGION ? { - usSelections: directorPayloads.reduce( - (total, payload) => - total + number(payload.selectedRegionsDelta?.[US_REGION] ?? 0, 'US selections'), - 0 - ) - } : {}) - } - // Director before cell per metric keeps the old validation order. - const split = (read) => ({ director: read(directorPayloads), cell: read(cellPayloads) }) - const sqlFailures = split((list) => list.reduce( - (total, payload) => total + number(payload.sqlFailuresDelta, 'Relay SQL failures'), 0 - )) - const waitingMax = split((list) => Math.max(...list.map( - (payload) => number(payload.databasePoolWaiting, 'database pool waiting') - ))) - const waitersMax = split((list) => Math.max(...list.map( - (payload) => number(payload.databasePoolWaitersMax, 'database pool waiters') - ))) - const waitMsMax = split((list) => Math.max(...list.map( - (payload) => number(payload.databasePoolWaitMsMax, 'database pool wait time') - ))) - // Ungated director values stay reported so the artifact still shows them. - const gated = gateDirectorDatabase - ? { - relaySqlFailures: sqlFailures.director + sqlFailures.cell, - databasePoolWaitingMax: Math.max(waitingMax.director, waitingMax.cell), - databasePoolWaitersMax: Math.max(waitersMax.director, waitersMax.cell), - databasePoolWaitMsMax: Math.max(waitMsMax.director, waitMsMax.cell) - } - : { - relaySqlFailures: sqlFailures.cell, - directorSqlFailures: sqlFailures.director, - databasePoolWaitingMax: waitingMax.cell, - databasePoolWaitersMax: waitersMax.cell, - databasePoolWaitMsMax: waitMsMax.cell, - directorDatabasePoolWaitingMax: waitingMax.director, - directorDatabasePoolWaitersMax: waitersMax.director, - directorDatabasePoolWaitMsMax: waitMsMax.director - } - return { - ...placement, - ...gated, - targetControlsMax: Math.max(...cellPayloads.map( - (payload) => number(payload.controls, 'target controls') - )), - targetSplicesMax: Math.max(...cellPayloads.map( - (payload) => number(payload.splices, 'target splices') - )) - } -} - -// Fleet-level proof that director metrics flowed; the placement check proves the cell itself. -function assertPassingRuntimeMetrics(metrics, label, region = ASIA_REGION) { - if (region === US_REGION) { - if (number(metrics.usSelections, 'US selections') < 1) { - throw new Error(`${label} observed no US selections`) - } - } else if (number(metrics.asiaSelections, 'Asia selections') < 1) { - throw new Error(`${label} observed no Asia selections`) - } - for (const key of [ - 'unavailableRegions', 'relaySqlFailures', 'databasePoolWaitingMax' - ]) { - if (number(metrics[key], key) !== 0) throw new Error(`${label} ${key} must be zero`) - } - if ( - number(metrics.databasePoolWaitersMax, 'databasePoolWaitersMax') > - DATABASE_POOL_TRANSIENT_WAITERS_MAX || - number(metrics.databasePoolWaitMsMax, 'databasePoolWaitMsMax') > - DATABASE_POOL_TRANSIENT_WAIT_MS_MAX - ) throw new Error(`${label} transient database pool pressure exceeded its bound`) -} - -function assertPassingCanary(metrics, cellId) { - const { label } = productionCanary(cellId) - if ( - number(metrics.targetControlsMax, `${label} controls`) < 1 || - number(metrics.targetSplicesMax, `${label} splices`) < 1 - ) throw new Error(`${label} canary traffic did not reach ${label}`) - if (number(metrics.cloudSqlBackendsMax, 'Cloud SQL backends') >= CLOUD_SQL_LIMIT) { - throw new Error(`Cloud SQL backends must remain below ${CLOUD_SQL_LIMIT}`) - } -} - -function assertProvenance(evidence, run, expected) { - const evidenceSource = object(evidence.source, 'evidence source') - if ( - run.id !== evidenceSource.runId || - run.run_attempt !== evidenceSource.runAttempt || - run.conclusion !== 'success' || - run.event !== 'workflow_dispatch' || - run.head_branch !== 'main' || - run.head_sha !== evidenceSource.commitSha || - run.repository?.full_name !== evidenceSource.repository || - run.path?.split('@')[0] !== expected.workflow || - evidenceSource.workflow !== expected.workflow || - evidenceSource.repository !== REPOSITORY || - expected.repository !== REPOSITORY || - evidenceSource.environment !== expected.environment || - evidenceSource.commitSha !== expected.commitSha - ) throw new Error('evidence workflow provenance does not match') -} - -export function verifyRolloutEvidence(evidence, run, expected) { - object(evidence, 'evidence') - object(run, 'workflow run') - if (evidence.version !== 1 || evidence.kind !== expected.kind) { - throw new Error('evidence kind is invalid') - } - assertProvenance(evidence, run, expected) - if (evidence.imageDigest !== expected.imageDigest) throw new Error('evidence image digest does not match') - if (!exactCells(evidence.topology?.cellIds, expected.cellIds)) { - throw new Error('evidence topology does not match') - } - positiveInteger(evidence.topology?.selectorGeneration, 'evidence selector generation') - if ( - expected.selectorGeneration !== undefined && - evidence.topology.selectorGeneration !== expected.selectorGeneration - ) throw new Error('evidence selector generation does not match') - const now = instant(expected.now, 'verification time') - const proofTime = instant(evidence.window?.endedAt, 'evidence time') - const maxAgeMs = expected.kind === 'staging-asia-readiness' ? 24 * 60 * 60_000 : 6 * 60 * 60_000 - if (proofTime > now || now.valueOf() - proofTime.valueOf() > maxAgeMs) { - throw new Error('rollout evidence is stale') - } - const canaryCell = Object.keys(PRODUCTION_CANARIES) - .find((cellId) => PRODUCTION_CANARIES[cellId].kind === expected.kind) - if (canaryCell) { - if (!exactCells(expected.cellIds, [canaryCell])) throw new Error('evidence topology does not match') - const start = instant(evidence.window?.startedAt, 'canary start') - if (proofTime.valueOf() - start.valueOf() < CANARY_MINIMUM_MS) { - throw new Error(`${productionCanary(canaryCell).label} canary window is shorter than 5 minutes`) - } - assertCanaryLoad(object(evidence.load, 'canary load'), canaryCell) - assertPassingCanary(object(evidence.metrics, 'canary metrics'), canaryCell) - } - return evidence -} - -function argumentsMap(argv) { - const values = new Map() - for (let index = 1; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined || values.has(key.slice(2))) { - throw new Error('invalid rollout evidence arguments') - } - values.set(key.slice(2), value) - } - return { command: argv[0], values } -} - -function required(values, key) { - const value = values.get(key) - if (!value) throw new Error(`missing --${key}`) - return value -} - -function commonInput(values) { - return { - repository: required(values, 'repository'), runId: required(values, 'run-id'), - runAttempt: required(values, 'run-attempt'), commitSha: required(values, 'commit-sha'), - imageDigest: required(values, 'image-digest'), - selectorGeneration: required(values, 'selector-generation') - } -} - -async function main(argv) { - const { command, values } = argumentsMap(argv) - const output = required(values, 'output') - if (command === 'create-staging') { - writeFileSync(output, `${JSON.stringify(buildStagingEvidence({ - ...commonInput(values), startedAt: required(values, 'started-at'), - endedAt: required(values, 'ended-at'), - launchReport: JSON.parse(readFileSync(required(values, 'launch-report'), 'utf8')), - logs: JSON.parse(readFileSync(required(values, 'logs-json'), 'utf8')), - cloudSql: await readCloudSqlBackends( - 'staging', required(values, 'started-at'), required(values, 'ended-at') - ) - }), null, 2)}\n`) - return - } - if (command === 'create-canary') { - const startedAt = required(values, 'started-at') - const endedAt = required(values, 'ended-at') - writeFileSync(output, `${JSON.stringify(buildProductionCanaryEvidence({ - ...commonInput(values), cellId: required(values, 'cell-id'), startedAt, endedAt, - loadReport: JSON.parse(readFileSync(required(values, 'load-report'), 'utf8')), - logs: JSON.parse(readFileSync(required(values, 'logs-json'), 'utf8')), - cloudSql: await readCloudSqlBackends('production', startedAt, endedAt) - }), null, 2)}\n`) - return - } - if (!['verify-staging', 'verify-c27'].includes(command)) throw new Error('invalid evidence command') - const kind = command === 'verify-staging' ? 'staging-asia-readiness' : 'production-c27-canary' - verifyRolloutEvidence( - JSON.parse(readFileSync(required(values, 'evidence'), 'utf8')), - JSON.parse(readFileSync(required(values, 'run-json'), 'utf8')), - { - kind, repository: REPOSITORY, - workflow: kind === 'staging-asia-readiness' ? STAGING_WORKFLOW : ADMISSION_WORKFLOW, - environment: kind === 'staging-asia-readiness' ? 'staging' : 'production', - commitSha: required(values, 'commit-sha'), imageDigest: required(values, 'image-digest'), - cellIds: [kind === 'staging-asia-readiness' ? STAGING_CELL : C27], - selectorGeneration: values.has('selector-generation') - ? positiveInteger(values.get('selector-generation'), 'selector generation') : undefined, - now: required(values, 'now') - } - ) - writeFileSync(output, 'verified\n') -} - -if (process.argv[1] === fileURLToPath(import.meta.url)) await main(process.argv.slice(2)) diff --git a/cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs b/cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs deleted file mode 100644 index 77e88a81452..00000000000 --- a/cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs +++ /dev/null @@ -1,592 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { RELAY_GITHUB_REPOSITORY, relayWorkflowPath } from './relay-repository.mjs' -import { - buildProductionCanaryEvidence, - buildStagingEvidence, - verifyRolloutEvidence -} from './relay-asia-rollout-evidence.mjs' - -const digest = `sha256:${'a'.repeat(64)}` -const commitSha = 'b'.repeat(40) -const repository = RELAY_GITHUB_REPOSITORY -const start = new Date('2026-08-13T12:00:00.000Z') -const end = new Date(start.valueOf() + 15 * 60_000) -const canaryEnd = new Date(start.valueOf() + 5 * 60_000) - -function sourceInput(overrides = {}) { - return { - repository, runId: 123, runAttempt: 2, commitSha, imageDigest: digest, - selectorGeneration: 9, ...overrides - } -} - -function metricLog(timestamp, role, cellId, overrides = {}) { - return { - timestamp: timestamp.toISOString(), - resource: { labels: role === 'director' ? { instance_id: 'director-1' } : {} }, - jsonPayload: { - role, cellId, region: role === 'cell' ? 'asia-east2' : 'us-central1', - controls: role === 'cell' ? 2_840 : 0, - splices: role === 'cell' ? 120 : 0, - selectedRegionsDelta: role === 'director' ? { 'asia-east2': 1 } : {}, - regionFallbacksDelta: {}, unavailableRegionsDelta: {}, sqlFailuresDelta: 0, - databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0, - ...overrides - } - } -} - -function completeLogs(cellId = 'production-gce-c27', windowEnd = end) { - const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000 + 1 - return Array.from({ length: samples }, (_, minute) => { - const timestamp = new Date(start.valueOf() + minute * 60_000) - return [ - metricLog(timestamp, 'director', 'production-director'), - metricLog(timestamp, 'cell', cellId) - ] - }).flat() -} - -function cloudSql(max = 319, windowEnd = end) { - const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000 - return { - timeSeries: [{ - points: Array.from({ length: samples }, (_, minute) => ({ - interval: { endTime: new Date(start.valueOf() + (minute + 1) * 60_000).toISOString() }, - value: { int64Value: String(minute === samples - 1 ? max : 300) } - })) - }] - } -} - -function loadReport({ - controls, shardIndex, splices = 0, slow = 0, wedged = 0, - launch = false -}) { - return { - event: 'relay_load_complete', controls, shardCount: 4, shardIndex, - configuredRampSeconds: 180, configuredSteadySeconds: 210, requiredLeaseHorizons: 2, - configuredSpliceHoldSeconds: 210, - configuredSplices: splices, configuredSlowReaderSplices: slow, - configuredWedgedReaderSplices: wedged, - peakActive: controls, steadyMinimumActive: controls, - connectionFailures: 0, rampConnectionFailures: 0, steadyConnectionFailures: 0, - transitionConnectionFailures: 0, unexpectedCloses: 0, protocolErrors: 0, - refreshErrors: 0, socketErrors: 0, failedSplices: 0, - regionalFallbacksProved: 0, - oldClientUsFirstProved: launch ? 1 : 0, - stickyAssignmentProved: launch ? 1 : 0, - requestUnitInvitesOpened: 0, - requestUnitPrincipalCount: 0, - relayAsiaLoadPrincipalCount: 32, - requestUnitOverflowReason: null, - requestUnitCleanupProved: 0, - phaseBarrierPassed: true, - rebindProbesOpened: launch ? 2 : 0, - rebindOverflowReason: null, - peakActiveSplices: splices, completedSplices: splices - wedged, - slowReaderSplicesCompleted: slow, wedgedReaderSplicesClosed: wedged, - readerQueuedBytesPeak: slow > 0 ? 1_024 : 0, - generatorCpuPercent: 25, generatorEventLoopP99Ms: 20, generatorRssGrowthMiB: 10, - readerQueueEvidence: slow > 0 ? [{ - origin: 'https://c4.relay-staging.onorca.dev', - baselineBytes: 128, - peakBytes: 1_152, - increaseBytes: 1_024 - }] : [], - shutdownEvidence: { - peerShutdowns: controls, activeControls: 0, activeSplices: 0, reconnectTimers: 0 - } - } -} - -function stagingInput(overrides = {}) { - const logs = completeLogs('staging-gce-c4') - logs[0].jsonPayload.regionFallbacksDelta = { 'us-central1': 1 } - return { - ...sourceInput({ selectorGeneration: 4 }), - startedAt: start.toISOString(), endedAt: end.toISOString(), - launchReport: Array.from({ length: 4 }, (_, shardIndex) => loadReport({ - controls: 5, shardIndex, splices: 5, launch: shardIndex === 0, - slow: shardIndex === 0 ? 4 : 0, wedged: shardIndex === 0 ? 1 : 0 - })), - logs, cloudSql: cloudSql(), ...overrides - } -} - -function canaryInput(overrides = {}, cellId = 'production-gce-c27') { - const load = loadReport({ controls: 1, shardIndex: 0, splices: 1 }) - Object.assign(load, { - shardCount: 1, - configuredSteadySeconds: 300, - configuredSpliceHoldSeconds: 60, - relayAsiaLoadPrincipalCount: 1, - assignedCellOrigins: [`https://${cellId.split('-').at(-1)}.relay.onorca.dev`] - }) - return { - ...sourceInput(), cellId, startedAt: start.toISOString(), endedAt: canaryEnd.toISOString(), - loadReport: load, - logs: completeLogs(cellId, canaryEnd), cloudSql: cloudSql(319, canaryEnd), - ...overrides - } -} - -function workflowRun(evidence, overrides = {}) { - return { - id: evidence.source.runId, run_attempt: evidence.source.runAttempt, - conclusion: 'success', event: 'workflow_dispatch', head_branch: 'main', - head_sha: evidence.source.commitSha, - repository: { full_name: evidence.source.repository }, path: evidence.source.workflow, - ...overrides - } -} - -function verifyExpected(kind, overrides = {}) { - const staging = kind === 'staging-asia-readiness' - return { - kind, repository, - workflow: staging - ? relayWorkflowPath('prove-relay-asia-staging.yml') - : relayWorkflowPath('operate-relay-asia-admission.yml'), - environment: staging ? 'staging' : 'production', commitSha, imageDigest: digest, - cellIds: [staging ? 'staging-gce-c4' : 'production-gce-c27'], - now: new Date(end.valueOf() + 60_000).toISOString(), ...overrides - } -} - -test('accepts the exact sharded staging load, telemetry, and provenance proof', () => { - const evidence = buildStagingEvidence(stagingInput()) - assert.equal(evidence.load.launch.controls, 20) - assert.equal(evidence.load.launch.peakActiveSplices, 20) - assert.equal(evidence.load.launch.readerQueueEvidence.length, 1) - assert.equal(verifyRolloutEvidence( - evidence, workflowRun(evidence), verifyExpected('staging-asia-readiness') - ), evidence) -}) - -test('ignores unrelated legacy cell metrics outside the Asia proof', () => { - const input = stagingInput() - const legacy = metricLog(start, 'cell', 'staging-gce-c1', { - region: undefined, - sqlFailuresDelta: 1 - }) - delete legacy.jsonPayload.databasePoolWaiting - delete legacy.jsonPayload.databasePoolWaitersMax - delete legacy.jsonPayload.databasePoolWaitMsMax - input.logs.push(legacy) - - assert.equal(buildStagingEvidence(input).metrics.relaySqlFailures, 0) -}) - -test('accepts bounded transient pool waits without a sampled queue', () => { - const input = stagingInput() - input.logs[0].jsonPayload.databasePoolWaitersMax = 4 - input.logs[0].jsonPayload.databasePoolWaitMsMax = 50 - - const metrics = buildStagingEvidence(input).metrics - assert.equal(metrics.databasePoolWaitingMax, 0) - assert.equal(metrics.databasePoolWaitersMax, 4) - assert.equal(metrics.databasePoolWaitMsMax, 50) -}) - -test('requires exactly the intentional sticky-assignment fallback in staging', () => { - const missing = stagingInput() - missing.logs[0].jsonPayload.regionFallbacksDelta = {} - assert.throws(() => buildStagingEvidence(missing), /intentional probes/) - - const extra = stagingInput() - extra.logs[2].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 } - assert.throws(() => buildStagingEvidence(extra), /intentional probes/) - - const substituted = stagingInput() - substituted.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 } - assert.throws(() => buildStagingEvidence(substituted), /intentional probes/) -}) - -test('accepts the wedged splice outside the sustained non-wedged peak', () => { - const input = stagingInput() - input.launchReport[0].peakActiveSplices = 4 - input.logs.filter((entry) => entry.jsonPayload.role === 'cell') - .forEach((entry) => { entry.jsonPayload.splices = 19 }) - const evidence = buildStagingEvidence(input) - assert.equal(evidence.load.launch.peakActiveSplices, 19) - - input.launchReport[0].peakActiveSplices = 3 - assert.throws(() => buildStagingEvidence(input), /mixed load evidence/) -}) - -test('rejects staging evidence with incomplete load or cleanup', () => { - const input = stagingInput() - input.launchReport[0].steadyMinimumActive-- - assert.throws(() => buildStagingEvidence(input), /required controls/) - const cleanup = stagingInput() - cleanup.launchReport[0].shutdownEvidence.activeControls = 1 - assert.throws(() => buildStagingEvidence(cleanup), /cleanup/) - const nonCausal = stagingInput() - nonCausal.launchReport[0].readerQueueEvidence[0].increaseBytes = 0 - assert.throws(() => buildStagingEvidence(nonCausal), /not causal/) - const multipleOwners = stagingInput() - multipleOwners.launchReport[0].configuredSlowReaderSplices-- - multipleOwners.launchReport[0].slowReaderSplicesCompleted-- - multipleOwners.launchReport[1] = loadReport({ - controls: 5, shardIndex: 1, splices: 5, slow: 1 - }) - assert.throws(() => buildStagingEvidence(multipleOwners), /one causal owner/) - const overloaded = stagingInput() - overloaded.launchReport[0].generatorCpuPercent = 80 - assert.throws(() => buildStagingEvidence(overloaded), /insufficient headroom/) - const missingLaunchProof = stagingInput() - missingLaunchProof.launchReport[0].rebindProbesOpened = 0 - assert.throws(() => buildStagingEvidence(missingLaunchProof), /launch-path/) - const offTarget = stagingInput() - offTarget.logs.filter((entry) => entry.jsonPayload.role === 'cell') - .forEach((entry) => { entry.jsonPayload.controls = 19 }) - assert.throws(() => buildStagingEvidence(offTarget), /did not reach C4/) -}) - -test('rejects staging evidence with a shortened splice hold', () => { - const input = stagingInput() - input.launchReport[0].configuredSpliceHoldSeconds = 60 - assert.throws(() => buildStagingEvidence(input), /load profile does not match/) -}) - -for (const [label, value] of [['missing', undefined], ['malformed', 'invalid']]) { - test(`rejects staging evidence with a ${label} splice hold`, () => { - const input = stagingInput() - input.launchReport[0].configuredSpliceHoldSeconds = value - assert.throws(() => buildStagingEvidence(input), /staging splice hold seconds is invalid/) - }) -} - -for (const [label, mutate, message] of [ - ['phase barrier', (input) => { input.launchReport[0].phaseBarrierPassed = false }, /profile/], - ['old-client routing', (input) => { input.launchReport[0].oldClientUsFirstProved = 0 }, /launch-path/], - ['sticky routing', (input) => { input.launchReport[0].stickyAssignmentProved = 0 }, /launch-path/] -]) { - test(`rejects staging evidence without ${label} proof`, () => { - const input = stagingInput() - mutate(input) - assert.throws(() => buildStagingEvidence(input), message) - }) -} - -test('rejects staging evidence from a non-canonical repository', () => { - assert.throws(() => buildStagingEvidence(stagingInput({ repository: 'fork/orca-cloud' })), /repository/) -}) - -test('rejects mismatched staging provenance, digest, topology, or age', () => { - const evidence = buildStagingEvidence(stagingInput()) - const expected = verifyExpected('staging-asia-readiness') - assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence, { - head_sha: 'c'.repeat(40) - }), expected), /provenance/) - assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), { - ...expected, commitSha: 'c'.repeat(40) - }), /provenance/) - assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), { - ...expected, imageDigest: `sha256:${'c'.repeat(64)}` - }), /image digest/) - assert.throws(() => verifyRolloutEvidence({ - ...evidence, topology: { ...evidence.topology, cellIds: ['staging-gce-c3'] } - }, workflowRun(evidence), expected), /topology/) - assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), { - ...expected, now: new Date(end.valueOf() + 25 * 60 * 60_000).toISOString() - }), /stale/) -}) - -test('builds and verifies a passing continuous 5-minute C27 canary', () => { - const evidence = buildProductionCanaryEvidence(canaryInput()) - assert.equal(evidence.load.completedSplices, 1) - assert.equal(evidence.metrics.asiaSelections, 6) - assert.equal(evidence.metrics.cloudSqlBackendsMax, 319) - assert.equal(verifyRolloutEvidence( - evidence, workflowRun(evidence), - verifyExpected('production-c27-canary', { selectorGeneration: 9 }) - ), evidence) -}) - -test('rejects short, sparse, or unrelated-cell-only C27 coverage', () => { - assert.throws(() => buildProductionCanaryEvidence(canaryInput({ - endedAt: new Date(canaryEnd.valueOf() - 1).toISOString() - })), /shorter than 5 minutes/) - const sparse = completeLogs('production-gce-c27', canaryEnd).filter((entry) => - entry.timestamp === start.toISOString() || entry.timestamp === canaryEnd.toISOString() - ) - assert.throws(() => buildProductionCanaryEvidence(canaryInput({ logs: sparse })), /sampling gap/) - assert.throws(() => buildProductionCanaryEvidence(canaryInput({ - logs: completeLogs('production-gce-c26', canaryEnd) - })), /production-gce-c27 metrics has no samples/) -}) - -test('rejects a C27 canary without a real control and splice', () => { - const input = canaryInput() - input.loadReport.completedSplices = 0 - assert.throws(() => buildProductionCanaryEvidence(input), /did not match/) - const shortHold = canaryInput() - shortHold.loadReport.configuredSpliceHoldSeconds = 59 - assert.throws(() => buildProductionCanaryEvidence(shortHold), /did not match/) -}) - -for (const [label, mutation, message] of [ - ['Asia selections', (input) => input.logs.forEach((entry) => { entry.jsonPayload.selectedRegionsDelta = {} }), /no Asia selections/], - ['Asia-targeted region fallbacks', (input) => { input.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 } }, /C27 canary asiaRegionFallbacks must be zero/], - ['unavailable regions', (input) => { input.logs[0].jsonPayload.unavailableRegionsDelta = { 'asia-east2': 1 } }, /unavailableRegions/], - ['C27 SQL failures', (input) => { input.logs[1].jsonPayload.sqlFailuresDelta = 1 }, /C27 canary relaySqlFailures must be zero/], - ['C27 pool waiting', (input) => { input.logs[1].jsonPayload.databasePoolWaiting = 1 }, /databasePoolWaitingMax/], - ['C27 pool waiters', (input) => { input.logs[1].jsonPayload.databasePoolWaitersMax = 5 }, /transient database pool pressure/], - ['C27 pool wait time', (input) => { input.logs[1].jsonPayload.databasePoolWaitMsMax = 51 }, /transient database pool pressure/], - ['C27 controls', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell') - .forEach((entry) => { entry.jsonPayload.controls = 0 }), /did not reach C27/], - ['C27 splices', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell') - .forEach((entry) => { entry.jsonPayload.splices = 0 }), /did not reach C27/], - ['Cloud SQL headroom', (input) => { input.cloudSql = cloudSql(320, canaryEnd) }, /below 320/] -]) { - test(`rejects C27 evidence with ${label}`, () => { - const input = canaryInput() - mutation(input) - assert.throws(() => buildProductionCanaryEvidence(input), message) - }) -} - -test('rejects C27 evidence from a different selector generation', () => { - const evidence = buildProductionCanaryEvidence(canaryInput()) - assert.throws(() => verifyRolloutEvidence( - evidence, workflowRun(evidence), - verifyExpected('production-c27-canary', { selectorGeneration: 10 }) - ), /selector generation/) -}) - -test('rejects a C27 canary whose control was placed on another cell', () => { - for (const origins of [ - ['https://c28.relay.onorca.dev'], - ['https://c27.relay.onorca.dev', 'https://c28.relay.onorca.dev'], - [], - undefined - ]) { - const input = canaryInput() - input.loadReport.assignedCellOrigins = origins - assert.throws(() => buildProductionCanaryEvidence(input), /C27 canary load was not placed only on C27/) - } -}) - -const c30 = 'production-gce-c30' - -test('builds and verifies a C30 canary from C30 runtime metrics and placement', () => { - const evidence = buildProductionCanaryEvidence(canaryInput({}, c30)) - assert.equal(evidence.kind, 'production-c30-canary') - assert.deepEqual(evidence.topology, { cellIds: [c30], selectorGeneration: 9 }) - assert.equal(verifyRolloutEvidence( - evidence, workflowRun(evidence), - verifyExpected('production-c30-canary', { cellIds: [c30], selectorGeneration: 9 }) - ), evidence) - assert.throws(() => verifyRolloutEvidence( - evidence, workflowRun(evidence), - verifyExpected('production-c30-canary', { cellIds: [c30], selectorGeneration: 10 }) - ), /selector generation/) - assert.throws(() => verifyRolloutEvidence( - evidence, workflowRun(evidence), verifyExpected('production-c27-canary', { selectorGeneration: 9 }) - ), /evidence kind is invalid/) -}) - -test('builds a C31 canary that only C31 placement satisfies', () => { - const c31 = 'production-gce-c31' - const evidence = buildProductionCanaryEvidence(canaryInput({}, c31)) - assert.equal(evidence.kind, 'production-c31-canary') - assert.equal(verifyRolloutEvidence( - evidence, workflowRun(evidence), - verifyExpected('production-c31-canary', { cellIds: [c31], selectorGeneration: 9 }) - ), evidence) - const onC30 = canaryInput({}, c31) - onC30.loadReport.assignedCellOrigins = ['https://c30.relay.onorca.dev'] - assert.throws(() => buildProductionCanaryEvidence(onC30), /C31 canary load was not placed only on C31/) -}) - -test('builds a C34 canary that only C34 placement satisfies', () => { - const c34 = 'production-gce-c34' - const evidence = buildProductionCanaryEvidence(canaryInput({}, c34)) - assert.equal(evidence.kind, 'production-c34-canary') - assert.equal(verifyRolloutEvidence( - evidence, workflowRun(evidence), - verifyExpected('production-c34-canary', { cellIds: [c34], selectorGeneration: 9 }) - ), evidence) - const onC31 = canaryInput({}, c34) - onC31.loadReport.assignedCellOrigins = ['https://c31.relay.onorca.dev'] - assert.throws(() => buildProductionCanaryEvidence(onC31), /C34 canary load was not placed only on C34/) -}) - -test('builds a C35 canary that only C35 placement satisfies', () => { - const c35 = 'production-gce-c35' - const evidence = buildProductionCanaryEvidence(canaryInput({}, c35)) - assert.equal(evidence.kind, 'production-c35-canary') - assert.equal(verifyRolloutEvidence( - evidence, workflowRun(evidence), - verifyExpected('production-c35-canary', { cellIds: [c35], selectorGeneration: 9 }) - ), evidence) - const onC34 = canaryInput({}, c35) - onC34.loadReport.assignedCellOrigins = ['https://c34.relay.onorca.dev'] - assert.throws(() => buildProductionCanaryEvidence(onC34), /C35 canary load was not placed only on C35/) -}) - -test('records but does not gate organic US-targeted fallbacks during a canary', () => { - // Mirrors C31's 2026-10-01 canary: US-targeted fallbacks, none targeting Asia. - const input = canaryInput({}, 'production-gce-c31') - input.logs.filter((entry) => entry.jsonPayload.role === 'director').forEach((entry) => { - entry.jsonPayload.regionFallbacksDelta = { 'us-central1': 24 } - }) - const evidence = buildProductionCanaryEvidence(input) - assert.equal(evidence.metrics.regionFallbacks, 144) - assert.equal(evidence.metrics.usRegionFallbacks, 144) - assert.equal(evidence.metrics.asiaRegionFallbacks, 0) - input.logs[0].jsonPayload.regionFallbacksDelta = { 'us-central1': 24, 'asia-east2': 1 } - assert.throws(() => buildProductionCanaryEvidence(input), /C31 canary asiaRegionFallbacks must be zero/) -}) - -// A root-region cell logs region us-central1, and the directors place US hosts too. -function usCanaryInput(cellId) { - const input = canaryInput({}, cellId) - for (const entry of input.logs) { - if (entry.jsonPayload.role === 'cell') entry.jsonPayload.region = 'us-central1' - else entry.jsonPayload.selectedRegionsDelta = { 'asia-east2': 1, 'us-central1': 3 } - } - return input -} - -test('builds and verifies C32 and C33 canaries from their own us-central1 metrics', () => { - for (const cellId of ['production-gce-c32', 'production-gce-c33']) { - const label = cellId.split('-').at(-1).toUpperCase() - const kind = `production-${label.toLowerCase()}-canary` - const evidence = buildProductionCanaryEvidence(usCanaryInput(cellId)) - assert.equal(evidence.kind, kind) - assert.equal(evidence.metrics.usSelections, 18) - assert.equal(verifyRolloutEvidence( - evidence, workflowRun(evidence), verifyExpected(kind, { cellIds: [cellId], selectorGeneration: 9 }) - ), evidence) - // The Asia-region filter would find none of a US cell's samples. - assert.throws( - () => buildProductionCanaryEvidence(canaryInput({}, cellId)), - new RegExp(`${cellId} metrics`) - ) - const elsewhere = usCanaryInput(cellId) - elsewhere.loadReport.assignedCellOrigins = ['https://c26.relay.onorca.dev'] - assert.throws( - () => buildProductionCanaryEvidence(elsewhere), - new RegExp(`${label} canary load was not placed only on ${label}`) - ) - const noUs = usCanaryInput(cellId) - for (const entry of noUs.logs) { - if (entry.jsonPayload.role === 'director') entry.jsonPayload.selectedRegionsDelta = { 'asia-east2': 1 } - } - assert.throws(() => buildProductionCanaryEvidence(noUs), /observed no US selections/) - } -}) - -test('records but does not gate either region\'s fallbacks during a US canary', () => { - const input = usCanaryInput('production-gce-c32') - for (const entry of input.logs.filter(({ jsonPayload }) => jsonPayload.role === 'director')) { - entry.jsonPayload.regionFallbacksDelta = { 'us-central1': 24, 'asia-east2': 1 } - } - const evidence = buildProductionCanaryEvidence(input) - assert.equal(evidence.metrics.usRegionFallbacks, 144) - assert.equal(evidence.metrics.asiaRegionFallbacks, 6) - // Every other gate still binds a US canary. - input.logs.find(({ jsonPayload }) => jsonPayload.role === 'cell').jsonPayload.sqlFailuresDelta = 1 - assert.throws(() => buildProductionCanaryEvidence(input), /relaySqlFailures must be zero/) -}) - -test('keeps the Asia canary evidence field set unchanged', () => { - const evidence = buildProductionCanaryEvidence(canaryInput({}, 'production-gce-c31')) - assert.equal('usSelections' in evidence.metrics, false) -}) - -test('rejects a C30 canary that C30 did not serve', () => { - const onLaunchCell = canaryInput({}, c30) - onLaunchCell.loadReport.assignedCellOrigins = ['https://c27.relay.onorca.dev'] - assert.throws(() => buildProductionCanaryEvidence(onLaunchCell), /C30 canary load was not placed only on C30/) - assert.throws(() => buildProductionCanaryEvidence(canaryInput({ - logs: completeLogs('production-gce-c27', canaryEnd) - }, c30)), /production-gce-c30 metrics has no samples/) - const idle = canaryInput({}, c30) - idle.logs.filter((entry) => entry.jsonPayload.role === 'cell') - .forEach((entry) => { entry.jsonPayload.splices = 0 }) - assert.throws(() => buildProductionCanaryEvidence(idle), /C30 canary traffic did not reach C30/) - const poolPressure = canaryInput({}, c30) - poolPressure.logs.at(-1).jsonPayload.databasePoolWaiting = 1 - assert.throws(() => buildProductionCanaryEvidence(poolPressure), /C30 canary databasePoolWaitingMax/) - assert.throws(() => buildProductionCanaryEvidence(canaryInput({ - endedAt: new Date(canaryEnd.valueOf() - 1).toISOString() - }, c30)), /C30 canary window is shorter than 5 minutes/) -}) - -test('gates a C30 canary on C30 pool pressure and only reports the director baseline', () => { - const directorPressure = canaryInput({}, c30) - const directorLogs = directorPressure.logs.filter((entry) => entry.jsonPayload.role === 'director') - Object.assign(directorLogs[0].jsonPayload, { databasePoolWaiting: 3 }) - Object.assign(directorLogs[1].jsonPayload, { databasePoolWaitersMax: 9, databasePoolWaitMsMax: 240 }) - const evidence = buildProductionCanaryEvidence(directorPressure) - assert.deepEqual([ - evidence.metrics.databasePoolWaitingMax, - evidence.metrics.databasePoolWaitersMax, - evidence.metrics.databasePoolWaitMsMax - ], [0, 0, 0]) - assert.deepEqual([ - evidence.metrics.directorDatabasePoolWaitingMax, - evidence.metrics.directorDatabasePoolWaitersMax, - evidence.metrics.directorDatabasePoolWaitMsMax - ], [3, 9, 240]) - - for (const [field, value, message] of [ - ['databasePoolWaiting', 1, /C30 canary databasePoolWaitingMax must be zero/], - ['databasePoolWaitersMax', 5, /C30 canary transient database pool pressure/], - ['databasePoolWaitMsMax', 51, /C30 canary transient database pool pressure/] - ]) { - const cellPressure = canaryInput({}, c30) - cellPressure.logs.find((entry) => entry.jsonPayload.role === 'cell').jsonPayload[field] = value - assert.throws(() => buildProductionCanaryEvidence(cellPressure), message) - } -}) - -test('gates a C30 canary on C30 SQL failures and only reports the director baseline', () => { - const directorNoise = canaryInput({}, c30) - directorNoise.logs.filter((entry) => entry.jsonPayload.role === 'director') - .slice(0, 3).forEach((entry) => { entry.jsonPayload.sqlFailuresDelta = 2 }) - const evidence = buildProductionCanaryEvidence(directorNoise) - assert.equal(evidence.metrics.relaySqlFailures, 0) - assert.equal(evidence.metrics.directorSqlFailures, 6) - assert.equal(verifyRolloutEvidence( - evidence, workflowRun(evidence), - verifyExpected('production-c30-canary', { cellIds: [c30], selectorGeneration: 9 }) - ), evidence) - - const cellFailure = canaryInput({}, c30) - cellFailure.logs.find((entry) => entry.jsonPayload.role === 'cell').jsonPayload.sqlFailuresDelta = 1 - assert.throws(() => buildProductionCanaryEvidence(cellFailure), /C30 canary relaySqlFailures must be zero/) -}) - -test('keeps staging gated on director database metrics without director-only fields', () => { - const metrics = buildStagingEvidence(stagingInput()).metrics - assert.equal(metrics.relaySqlFailures, 0) - assert.equal(Object.keys(metrics).some((key) => key.startsWith('director')), false) - for (const [field, value, message] of [ - ['databasePoolWaiting', 1, /staging proof databasePoolWaitingMax must be zero/], - ['databasePoolWaitersMax', 5, /staging proof transient database pool pressure/], - ['databasePoolWaitMsMax', 51, /staging proof transient database pool pressure/] - ]) { - const directorPressure = stagingInput() - directorPressure.logs[0].jsonPayload[field] = value - assert.throws(() => buildStagingEvidence(directorPressure), message) - } - - const directorFailure = stagingInput() - directorFailure.logs[0].jsonPayload.sqlFailuresDelta = 1 - assert.throws(() => buildStagingEvidence(directorFailure), /staging proof relaySqlFailures must be zero/) -}) - -test('accepts canaries only for the reviewed production Asia cells', () => { - for (const cellId of ['production-gce-c28', 'production-gce-c29', 'staging-gce-c4', undefined]) { - assert.throws( - () => buildProductionCanaryEvidence({ ...canaryInput(), cellId }), - /not a reviewed production Asia canary/ - ) - } -}) diff --git a/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs b/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs deleted file mode 100644 index 3af25a21852..00000000000 --- a/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs +++ /dev/null @@ -1,244 +0,0 @@ -import assert from 'node:assert/strict' -import { spawnSync } from 'node:child_process' -import { mkdtempSync, readFileSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { test } from 'node:test' -import { relayWorkflowUrl } from './relay-repository.mjs' - -const workflow = readFileSync( - relayWorkflowUrl('deploy-relay-asia-topology.yml'), - 'utf8' -) -const iam = readFileSync( - new URL('../../infra/terraform/relay-asia-topology-iam.tf', import.meta.url), - 'utf8' -) -const cells = readFileSync( - new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url), - 'utf8' -) -const variables = readFileSync( - new URL('../../infra/terraform/variables.tf', import.meta.url), - 'utf8' -) - -test('uses only its exact workflow-bound topology identity', () => { - assert.match(workflow, /production-cloud-sql-rollout/) - assert.match(workflow, /relay-staging-mutation/) - assert.match(workflow, /RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER/) - assert.match(workflow, /RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT/) - assert.doesNotMatch(workflow, /GCP_DEPLOY_SERVICE_ACCOUNT/) - assert.match( - iam, - /assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_relay_asia_topology_workflow_file\}@refs\/heads\/main'/ - ) - assert.match(iam, /assertion\.ref == 'refs\/heads\/main'/) - assert.match(iam, /assertion\.event_name == 'workflow_dispatch'/) - assert.match(iam, /assertion\.environment == '\$\{var\.environment\}'/) -}) - -test('accepts only the reviewed Asia topology waves', () => { - const cases = /case "\$\{TARGET_ENVIRONMENT\}:\$\{TARGET_CELL_IDS\}" in\n([\s\S]*?)\n\s*esac/ - .exec(workflow)?.[1] - assert.ok(cases) - assert.deepEqual( - [...cases.matchAll(/^\s*([a-z]+:[a-z0-9,-]+)\)(?: target_region=[a-z0-9-]+)? ;;$/gm)] - .map((match) => match[1]), - [ - 'staging:staging-gce-c4', - 'production:production-gce-c27,production-gce-c28,production-gce-c29', - 'production:production-gce-c30', - 'production:production-gce-c31', - 'production:production-gce-c32,production-gce-c33', - 'production:production-gce-c34', - 'production:production-gce-c35' - ] - ) -}) - -// Runs the workflow's own region and target blocks, so an Asia wave's targets stay byte-identical. -function waveTargets(environment, cellIds) { - const block = (first, last) => { - const start = workflow.indexOf(first) - const end = workflow.indexOf(last, start) - assert.ok(start !== -1 && end !== -1, first) - return workflow.slice(start, end + last.length).replace(/^ {10}/gm, '') - } - const region = block('target_region=asia-east2\n', '\n esac') - const targets = block('file="${RUNNER_TEMP}/relay-asia-targets"', '\n done') - const temp = mkdtempSync(join(tmpdir(), 'relay-topology-targets-')) - try { - const result = spawnSync('bash', ['-euo', 'pipefail', '-c', - `${region}\nTARGET_REGION="\${target_region}"\n${targets}\n` + - 'echo "${TARGET_REGION}"; cat "${file}"'], { - env: { ...process.env, TARGET_ENVIRONMENT: environment, TARGET_CELL_IDS: cellIds, RUNNER_TEMP: temp }, - encoding: 'utf8' - }) - assert.equal(result.status, 0, result.stderr) - const [waveRegion, ...lines] = result.stdout.trim().split('\n') - return { region: waveRegion, targets: lines } - } finally { - rmSync(temp, { recursive: true, force: true }) - } -} - -const cellTargets = (cellId) => ['instance_template', 'instance_group_manager', 'backend_service'] - .map((kind) => `-target=google_compute_${kind}.relay_gce_cell["${cellId}"]`) - -test('targets the asia-east2 network for Asia waves and only the cell and URL map for US waves', () => { - const asiaNetwork = ['subnetwork', 'router', 'router_nat'] - .map((kind) => `-target=google_compute_${kind}.relay_gce_additional["asia-east2"]`) - const urlMap = '-target=google_compute_url_map.relay_gce[0]' - for (const [environment, cellIds] of [ - ['staging', 'staging-gce-c4'], - ['production', 'production-gce-c27,production-gce-c28,production-gce-c29'], - ['production', 'production-gce-c30'], - ['production', 'production-gce-c31'] - ]) { - assert.deepEqual(waveTargets(environment, cellIds), { - region: 'asia-east2', - targets: [...asiaNetwork, urlMap, ...cellIds.split(',').flatMap(cellTargets)] - }, cellIds) - } - const usWave = 'production-gce-c32,production-gce-c33' - assert.deepEqual(waveTargets('production', usWave), { - region: 'us-central1', - targets: [urlMap, ...usWave.split(',').flatMap(cellTargets)] - }) -}) - -// The live-image overlay refuses a declared non-target cell with no template, so the two US cells -// declared together must also plan together; a lone C32 or C33 plan is not a reviewed wave. -test('plans the two declared US cells as one wave', () => { - const cases = /case "\$\{TARGET_ENVIRONMENT\}:\$\{TARGET_CELL_IDS\}" in\n([\s\S]*?)\n\s*esac/ - .exec(workflow)?.[1] - assert.doesNotMatch(cases, /production:production-gce-c3[23]\)/) -}) - -test('plans only additive Asia topology and applies the saved plan', () => { - assert.doesNotMatch(workflow, /manage_artifact_dns/) - for (const target of [ - 'relay_gce_additional', - 'google_compute_instance_template.relay_gce_cell', - 'google_compute_instance_group_manager.relay_gce_cell', - 'google_compute_backend_service.relay_gce_cell', - 'google_compute_url_map.relay_gce' - ]) assert.match(workflow, new RegExp(target.replaceAll('.', '\\.'))) - assert.match(workflow, /apply -input=false -auto-approve "\$\{\{ steps\.plan\.outputs\.plan \}\}"/) - assert.match(workflow, /prepare-relay-asia-topology-input\.mjs/) - assert.doesNotMatch(workflow, /steps\.variables\.outputs\.file/) - assert.match(workflow, /\.variables\.relay_gce_cells\.value/) - assert.match( - workflow, - /\.variables\.relay_gce_additional_region_subnetwork_cidrs\.value/ - ) - // Console evaluates every output against state and fails while a declared cell has no MIG. - assert.doesNotMatch(workflow, /terraform[^\n]*console/) - assert.equal( - (workflow.match(/-var-file="\$\{TF_VARS\}" -var-file="\$\{\{ steps\.live-images\.outputs\.file \}\}"/g) ?? []).length, - 2 - ) - assert.equal((workflow.match(/-var-file=/g) ?? []).length, 5) - assert.doesNotMatch(workflow, /terraform[^\n]*apply[^\n]*-target/) - assert.doesNotMatch(workflow, /google_(?:sql|cloudflare|dns|certificate_manager)/) -}) - -test('validates before apply and proves convergence afterward', () => { - assert.equal((workflow.match(/validate-relay-asia-topology-plan\.mjs/g) ?? []).length, 2) - assert.match(workflow, /APPLY_RELAY_ASIA_TOPOLOGY/) - assert.match(workflow, /test "\$\(jq -er '\.changes'/) - assert.match(workflow, /Register the exact new cells atomically as migration-only/) -}) - -test('checks the connection budget and production live ceiling before planning', () => { - assert.match(workflow, /relay-cloud-sql-connection-budget\.mjs/) - assert.match(workflow, /gcloud sql instances describe "\$\{CLOUD_SQL_INSTANCE\}"/) - assert.match(workflow, /select\(\.name == "max_connections"\)/) - assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360/) - assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17/) - assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS: '500'/) - assert.match(workflow, /live_max="\$\{VERIFIED_DEFAULT_MAX_CONNECTIONS\}"/) - assert.doesNotMatch(workflow, /live_max=\d/) - assert.match(workflow, /live_source=verified-shape-default/) - assert.match(workflow, /test "\$\(jq -er '\.settings\.tier'/) - assert.match(workflow, /test "\$\(jq -er '\.databaseVersion'/) - assert.match(workflow, /test "\$\{live_max\}" = "\$\{checked_max\}"/) - assert.ok( - workflow.indexOf('relay-cloud-sql-connection-budget.mjs') < - workflow.indexOf('terraform -chdir=infra/terraform plan') - ) -}) - -test('binds computed Asia references to the matching Terraform cell resources', () => { - assert.match(cells, /instance_template = google_compute_instance_template\.relay_gce_cell\[each\.key\]\.self_link/) - assert.match(cells, /group\s+= google_compute_instance_group_manager\.relay_gce_cell\[each\.key\]\.instance_group/) - assert.match(cells, /default_service = google_compute_backend_service\.relay_gce_cell\[cell\.key\]\.id/) - assert.match(cells, /subnetwork = local\.relay_gce_subnetworks\[each\.value\.region\]/) -}) - -test('keeps cross-variable region constraints in Terraform 1.5 check blocks', () => { - assert.doesNotMatch(variables, /region != var\.region/) - assert.doesNotMatch(variables, /cell\.region == var\.region/) - assert.match(cells, /check "relay_gce_fixed_one_topology"[\s\S]*?region != var\.region/) - assert.match(cells, /cell\.region == var\.region[\s\S]*?configured subnetwork/) -}) - -test('the custom role cannot delete topology or mutate SQL and DNS', () => { - assert.doesNotMatch(iam, /compute\.[A-Za-z]+\.delete/) - assert.doesNotMatch( - iam, - /roles\/viewer|cloudsql\.instances\.(?:update|delete)|dns\.|certificatemanager|cloudflare/i - ) - assert.match(iam, /resource "google_project_iam_custom_role" "github_relay_asia_topology_read"/) - assert.match(iam, /"cloudsql\.instances\.get"/) - assert.match(iam, /"run\.revisions\.get"/) - assert.match(iam, /"run\.services\.get"/) - assert.match(iam, /"serviceusage\.services\.list"/) - assert.match(iam, /"compute\.networks\.updatePolicy"/) - assert.match(iam, /"compute\.healthChecks\.useReadOnly"/) - assert.match(iam, /"compute\.instanceGroups\.create"/) - assert.match(iam, /"compute\.instances\.use"/) - assert.match(iam, /roles\/storage\.objectAdmin/) - assert.match(iam, /default\.tfstate/) - assert.match(iam, /default\.tflock/) - assert.match( - iam, - /resource "google_project_iam_custom_role" "github_relay_asia_topology_state_list"[\s\S]*?permissions = \["storage\.objects\.list"\]/ - ) - assert.match( - iam, - /resource "google_storage_bucket_iam_member" "github_relay_asia_topology_state_list"[\s\S]*?role\s+= google_project_iam_custom_role\.github_relay_asia_topology_state_list\[0\]\.id/ - ) -}) - -test('plans every non-target cell at its served image, read from state templates only', () => { - const step = /- id: live-images\n[\s\S]*?\n\n/.exec(workflow)?.[0] - assert.ok(step) - assert.match(step, /terraform -chdir=infra\/terraform show -json \| jq -ce '\[/) - assert.match(step, /\.type == "google_compute_instance_template" and \.name == "relay_gce_cell"/) - assert.match(step, /\{ index, metadata_startup_script: \.values\.metadata_startup_script \}/) - assert.match(step, /relay-live-cell-image-overlay\.mjs/) - assert.match(step, /--cell-ids "\$\{TARGET_CELL_IDS\}"/) - // The committed map comes from a read-only plan over the same targets, never from console. - assert.match(step, /mapfile -t targets < "\$\{\{ steps\.targets\.outputs\.file \}\}"/) - assert.match( - step, - /terraform -chdir=infra\/terraform plan -input=false -refresh=false -lock=false \\\n\s+-var-file="\$\{TF_VARS\}" "\$\{targets\[@\]\}" -out="\$\{committed_plan\}" > \/dev\/null/ - ) - assert.match(step, /show -json "\$\{committed_plan\}" \\\n\s+\| jq -ce '\.variables\.relay_gce_cells\.value \| objects' > "\$\{cells\}"/) - assert.equal((step.match(/terraform -chdir=infra\/terraform (?:plan|apply)/g) ?? []).length, 1) - assert.ok(workflow.indexOf('- id: targets') < workflow.indexOf('- id: live-images')) - assert.ok(workflow.indexOf('- id: live-images') < workflow.indexOf('- name: Create and validate the saved topology plan')) -}) - -test('the deployments output tolerates a cell declared before its topology apply', () => { - const outputs = readFileSync(new URL('../../infra/terraform/outputs.tf', import.meta.url), 'utf8') - const start = outputs.indexOf('output "relay_gce_cell_deployments" {') - const block = outputs.slice(start, outputs.indexOf('\n}\n', start)) - const lookups = [...block.matchAll(/^\s+\w+\s+=\s+(.*\.relay_gce_cell\[cell_id\].*)$/gm)].map((match) => match[1]) - assert.equal(lookups.length, 6) - for (const lookup of lookups) { - assert.match(lookup, /^try\(google_compute_\w+\.relay_gce_cell\[cell_id\]\.\w+, null\)$/) - } -}) diff --git a/cloud/dev/scripts/relay-clock-skew-monitor-workflow.test.mjs b/cloud/dev/scripts/relay-clock-skew-monitor-workflow.test.mjs deleted file mode 100644 index cdd40a23c82..00000000000 --- a/cloud/dev/scripts/relay-clock-skew-monitor-workflow.test.mjs +++ /dev/null @@ -1,73 +0,0 @@ -import assert from 'node:assert/strict' -import { spawnSync } from 'node:child_process' -import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { delimiter, join } from 'node:path' -import { test } from 'node:test' -import { readRelayWorkflow, relayTreePath } from './relay-repository.mjs' - -const workflow = readRelayWorkflow('monitor-relay-clock-skew.yml') -const productionTfvars = readFileSync( - new URL('../../infra/terraform/environments/production.tfvars', import.meta.url), - 'utf8' -) - -// The measuring step's `run: |` block, unindented as the runner receives it. -function measureScript() { - const lines = workflow.split('\n') - const step = lines.findIndex((line) => line.includes('name: Measure Date-header skew')) - const start = lines.findIndex((line, index) => index > step && line.trim() === 'run: |') + 1 - assert.ok(step !== -1 && start > 0, 'measuring step not found') - const end = lines.findIndex( - (line, index) => index >= start && line.trim() !== '' && !line.startsWith(' '.repeat(10)) - ) - return lines.slice(start, end === -1 ? undefined : end).map((line) => line.slice(10)).join('\n') -} - -// Runs the step in a checkout holding `tfvars`, with every cell answering like a fenced one so -// each probe is named without a clock to judge. -function runMonitor(tfvars) { - const root = mkdtempSync(join(tmpdir(), 'relay-clock-skew-')) - try { - const bin = join(root, 'bin') - mkdirSync(bin) - writeFileSync(join(bin, 'curl'), '#!/usr/bin/env bash\nprintf "HTTP/2 404 \\r\\n\\r\\n"\n') - chmodSync(join(bin, 'curl'), 0o755) - const environments = join(root, 'cloud', 'infra', 'terraform', 'environments') - mkdirSync(environments, { recursive: true }) - if (tfvars !== undefined) writeFileSync(join(environments, 'production.tfvars'), tfvars) - return spawnSync('bash', ['-e', '-c', measureScript()], { - cwd: join(root, 'cloud'), - env: { ...process.env, PATH: `${bin}${delimiter}${process.env.PATH}` }, - encoding: 'utf8' - }) - } finally { - rmSync(root, { recursive: true, force: true }) - } -} - -test('checks out the tfvars the measuring step reads', () => { - const tfvars = relayTreePath('infra/terraform/environments/production.tfvars') - assert.ok(workflow.includes(`sparse-checkout: ${tfvars}\n`)) - assert.ok(workflow.includes('sparse-checkout-cone-mode: false\n')) -}) - -test('probes every cell production.tfvars declares', () => { - const declared = [ - ...productionTfvars.matchAll(/"production-gce-c\d+"\s*=\s*\{([\s\S]*?)\n {2}\}/g) - ].map(([, body]) => /\bhostname\s*=\s*"([^"]+)"/.exec(body)?.[1]) - assert.ok(declared.length > 0) - const result = runMonitor(productionTfvars) - assert.deepEqual( - [...result.stdout.matchAll(/^(\S+): not serving/gm)].map(([, cell]) => cell), - declared - ) -}) - -test('fails instead of probing nothing when no cell can be read', () => { - for (const tfvars of [undefined, 'relay_gce_cells = {}\n']) { - const result = runMonitor(tfvars) - assert.equal(result.status, 1) - assert.match(result.stdout, /::error::read no relay cells/) - } -}) diff --git a/cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs b/cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs deleted file mode 100644 index 79036918f23..00000000000 --- a/cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs +++ /dev/null @@ -1,148 +0,0 @@ -import { readFileSync } from 'node:fs' -import { fileURLToPath } from 'node:url' - -const DEFAULT_PATHS = { - productionTfvars: new URL('../../infra/terraform/environments/production.tfvars', import.meta.url), - terraformVariables: new URL('../../infra/terraform/variables.tf', import.meta.url), - relayConfig: new URL('../../apps/relay/src/config.ts', import.meta.url) -} - -// Auth and API live outside the Relay tree, so their consumption is published as a contract -// rather than parsed from their source. production-cloud-sql-app-consumers.test.mjs binds it back. -const APP_CONSUMERS_CONTRACT = new URL( - '../contracts/production-cloud-sql-app-consumers.json', - import.meta.url -) - -const APP_CONSUMER_FIELDS = ['authInstances', 'authPoolMax', 'apiInstances', 'apiPoolMax', 'maxConnections'] - -export function readProductionCloudSqlAppConsumers(contract) { - const parsed = contract ?? JSON.parse(readFileSync(APP_CONSUMERS_CONTRACT, 'utf8')) - for (const field of APP_CONSUMER_FIELDS) { - const value = parsed[field] - if (!Number.isSafeInteger(value) || value <= 0) { - throw new Error(`could not read ${field} from the app consumer contract`) - } - } - return parsed -} - -function requiredInteger(source, pattern, label) { - const value = Number(source.match(pattern)?.[1]) - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`could not read ${label}`) - return value -} - -function productionCells(source, defaultPoolMax) { - const fencedMatch = source.match(/relay_gce_fenced_cells\s*=\s*\[([^\]]*)\]/) - if (!fencedMatch) throw new Error('could not read fenced Relay cells') - const fenced = new Set([...fencedMatch[1].matchAll(/"([^"]+)"/g)].map((match) => match[1])) - const cells = [...source.matchAll(/"(production-gce-[^"]+)"\s*=\s*\{([\s\S]*?)\n\s*\}/g)].map( - ([, id, body]) => ({ - id, - fenced: fenced.has(id), - region: body.match(/\bregion\s*=\s*"([^"]+)"/)?.[1] ?? 'us-central1', - poolMax: body.match(/\bdatabase_pool_max\s*=\s*(\d+)/) - ? Number(body.match(/\bdatabase_pool_max\s*=\s*(\d+)/)[1]) - : defaultPoolMax - }) - ) - if (cells.length === 0) throw new Error('could not read production Relay cells') - return cells -} - -export function calculateRelayCloudSqlConnectionBudget(inputs) { - const consumers = { - cells: inputs.cellPoolTotal + inputs.asiaCellCount * inputs.asiaPoolMax, - directors: inputs.directorInstances * inputs.directorPoolMax, - auth: inputs.authInstances * inputs.authPoolMax, - api: inputs.apiInstances * inputs.apiPoolMax - } - const configuredMaximum = Object.values(consumers).reduce((total, value) => total + value, 0) - const retainedDirectorRollback = inputs.directorInstances * inputs.directorPoolMax - const candidateOverlap = { - relayDirectorCandidate: retainedDirectorRollback * 2, - apiCandidate: retainedDirectorRollback + inputs.apiInstances * inputs.apiPoolMax, - authCandidate: retainedDirectorRollback + inputs.authInstances * inputs.authPoolMax, - relayCells: retainedDirectorRollback - } - const rolloutOverlap = Math.max(...Object.values(candidateOverlap)) - const operatingMaximum = configuredMaximum + rolloutOverlap + inputs.maintenanceAdminAllowance - const usableCeiling = inputs.maxConnections - inputs.explicitReserve - const budgetedTotal = operatingMaximum + inputs.explicitReserve - return { - maxConnections: inputs.maxConnections, - consumers, - asia: { cells: inputs.asiaCellCount, poolMax: inputs.asiaPoolMax }, - configuredMaximum, - rolloutOverlap: { - ...candidateOverlap, - retainedDirectorRollback, - maximum: rolloutOverlap, - reason: 'serialized rollouts include directly addressable tagged revisions outside service-level caps' - }, - maintenanceAdminAllowance: inputs.maintenanceAdminAllowance, - maintenanceAdminAllowanceReason: 'covers bounded work outside configured services', - explicitReserve: inputs.explicitReserve, - explicitReserveReason: 'remains unavailable to configured services and planned rollouts', - usableCeiling, - operatingMaximum, - remainingWithinUsableCeiling: usableCeiling - operatingMaximum, - budgetedTotal, - unallocated: inputs.maxConnections - budgetedTotal, - withinBudget: operatingMaximum <= usableCeiling && budgetedTotal < inputs.maxConnections - } -} - -export function readRelayCloudSqlConnectionBudget({ - sources, - appConsumers, - proposedAsiaCellCount = 3, - asiaPoolMax = 10, - maxConnections, - maintenanceAdminAllowance = 5, - explicitReserve = 10 -} = {}) { - const read = (name) => sources?.[name] ?? readFileSync(DEFAULT_PATHS[name], 'utf8') - const apps = readProductionCloudSqlAppConsumers(appConsumers) - const productionTfvars = read('productionTfvars') - const terraformVariables = read('terraformVariables') - const relayConfig = read('relayConfig') - const cells = productionCells( - productionTfvars, - requiredInteger(relayConfig, /RELAY_DATABASE_POOL_MAX\s*=\s*(\d+)/, 'Relay pool maximum') - ) - const poweredCells = cells.filter(({ fenced }) => !fenced) - const configuredAsiaCells = poweredCells.filter(({ region }) => region === 'asia-east2') - const nonAsiaCells = poweredCells.filter(({ region }) => region !== 'asia-east2') - const cellPoolTotal = nonAsiaCells.reduce((total, cell) => total + cell.poolMax, 0) - const asiaCellCount = configuredAsiaCells.length || proposedAsiaCellCount - const configuredAsiaPoolMax = configuredAsiaCells[0]?.poolMax ?? asiaPoolMax - if (configuredAsiaCells.some(({ poolMax }) => poolMax !== configuredAsiaPoolMax)) { - throw new Error('Asia Relay cells must use one checked pool maximum') - } - return calculateRelayCloudSqlConnectionBudget({ - cellPoolTotal, - asiaCellCount, - asiaPoolMax: configuredAsiaPoolMax, - directorInstances: requiredInteger(productionTfvars, /relay_max_instances\s*=\s*(\d+)/, 'director instances'), - directorPoolMax: requiredInteger( - terraformVariables, - /variable\s+"relay_director_database_pool_max"[\s\S]*?default\s*=\s*(\d+)/, - 'director pool maximum' - ), - authInstances: apps.authInstances, - authPoolMax: apps.authPoolMax, - apiInstances: apps.apiInstances, - apiPoolMax: apps.apiPoolMax, - maxConnections: maxConnections ?? apps.maxConnections, - maintenanceAdminAllowance, - explicitReserve - }) -} - -if (process.argv[1] === fileURLToPath(import.meta.url)) { - const report = readRelayCloudSqlConnectionBudget() - console.log(JSON.stringify({ event: 'relay_cloud_sql_connection_budget', ...report }, null, 2)) - if (!report.withinBudget) process.exitCode = 1 -} diff --git a/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs b/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs deleted file mode 100644 index 0ea73086add..00000000000 --- a/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs +++ /dev/null @@ -1,173 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { - calculateRelayCloudSqlConnectionBudget, - readRelayCloudSqlConnectionBudget -} from './relay-cloud-sql-connection-budget.mjs' - -test('production shared consumers keep allowance and reserve below the ceiling', () => { - // cells: 22 pools at 10 (220, C32/C33 included) + the seven asia-east2 pools at 16 (112). - const report = readRelayCloudSqlConnectionBudget() - - assert.deepEqual(report.consumers, { cells: 332, directors: 15, auth: 20, api: 50 }) - assert.deepEqual(report.asia, { cells: 7, poolMax: 16 }) - assert.equal(report.configuredMaximum, 417) - assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30) - assert.equal(report.rolloutOverlap.apiCandidate, 65) - assert.equal(report.rolloutOverlap.authCandidate, 35) - assert.equal(report.rolloutOverlap.relayCells, 15) - assert.equal(report.rolloutOverlap.retainedDirectorRollback, 15) - assert.equal(report.rolloutOverlap.maximum, 65) - assert.equal(report.maintenanceAdminAllowance, 5) - assert.equal(report.explicitReserve, 10) - assert.equal(report.usableCeiling, 490) - assert.equal(report.operatingMaximum, 487) - assert.equal(report.remainingWithinUsableCeiling, 3) - assert.equal(report.budgetedTotal, 497) - assert.equal(report.unallocated, 3) - assert.equal(report.withinBudget, true) -}) - -test('fails closed when pool growth consumes the explicit reserve', () => { - const report = calculateRelayCloudSqlConnectionBudget({ - cellPoolTotal: 200, - asiaCellCount: 3, - asiaPoolMax: 20, - directorInstances: 5, - directorPoolMax: 3, - authInstances: 2, - authPoolMax: 10, - apiInstances: 20, - apiPoolMax: 5, - maxConnections: 400, - maintenanceAdminAllowance: 5, - explicitReserve: 10 - }) - - assert.equal(report.operatingMaximum, 515) - assert.equal(report.withinBudget, false) -}) - -test('excludes fenced cell pools and reads per-cell pool overrides', () => { - const report = readRelayCloudSqlConnectionBudget({ - proposedAsiaCellCount: 1, - appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 }, - sources: { - productionTfvars: ` - relay_max_instances = 1 - relay_gce_fenced_cells = ["production-gce-c1"] - relay_gce_cells = { - "production-gce-c1" = { database_pool_max = 99 - } - "production-gce-c2" = { database_pool_max = 4 - } - } - `, - terraformVariables: [ - 'variable "relay_director_database_pool_max" { default = 3 }', - 'variable "push_max_instances" { default = 1 }', - 'variable "push_database_pool_max" { default = 2 }' - ].join('\n'), - relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10' - }, - maxConnections: 100, - maintenanceAdminAllowance: 1, - explicitReserve: 1 - }) - - assert.equal(report.consumers.cells, 14) - assert.equal(report.operatingMaximum, 46) - assert.equal(report.budgetedTotal, 47) -}) - -test('refuses an Asia cell whose pool differs from its siblings', () => { - const budget = (c30PoolMax) => readRelayCloudSqlConnectionBudget({ - appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 500 }, - sources: { - productionTfvars: ` - relay_max_instances = 1 - relay_gce_fenced_cells = [] - relay_gce_cells = { -${['c27', 'c28', 'c29', 'c30'].map((hostname) => ` "production-gce-${hostname}" = { - region = "asia-east2" - database_pool_max = ${hostname === 'c30' ? c30PoolMax : 16} - }`).join('\n')} - } - `, - terraformVariables: [ - 'variable "relay_director_database_pool_max" { default = 3 }', - 'variable "push_max_instances" { default = 1 }', - 'variable "push_database_pool_max" { default = 2 }' - ].join('\n'), - relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10' - }, - maxConnections: 500 - }) - assert.deepEqual(budget(16).asia, { cells: 4, poolMax: 16 }) - assert.throws(() => budget(10), /Asia Relay cells must use one checked pool maximum/) -}) - -test('dedicated push scaling does not consume shared capacity', () => { - const report = readRelayCloudSqlConnectionBudget({ - proposedAsiaCellCount: 1, - appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 }, - sources: { - productionTfvars: ` - relay_max_instances = 1 - push_max_instances = 3 - relay_gce_fenced_cells = [] - relay_gce_cells = { - "production-gce-c2" = { database_pool_max = 4 - } - } - `, - terraformVariables: [ - 'variable "relay_director_database_pool_max" { default = 3 }', - 'variable "push_max_instances" { default = 1 }', - 'variable "push_database_pool_max" { default = 2 }' - ].join('\n'), - relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10' - }, - maxConnections: 100, - maintenanceAdminAllowance: 1, - explicitReserve: 1 - }) - - assert.equal(report.consumers.push, undefined) - assert.equal(report.rolloutOverlap.pushCandidate, undefined) - assert.equal(report.operatingMaximum, 46) -}) - -test('requires strict headroom below the physical ceiling', () => { - const report = calculateRelayCloudSqlConnectionBudget({ - cellPoolTotal: 20, - asiaCellCount: 0, - asiaPoolMax: 10, - directorInstances: 1, - directorPoolMax: 3, - authInstances: 1, - authPoolMax: 10, - apiInstances: 1, - apiPoolMax: 5, - maxConnections: 50, - maintenanceAdminAllowance: 9, - explicitReserve: 3 - }) - - assert.equal(report.budgetedTotal, 63) - assert.equal(report.withinBudget, false) -}) - -test('pages Relay channels when Cloud SQL backends consume headroom', () => { - const terraform = readFileSync( - new URL('../../infra/terraform/relay-observability.tf', import.meta.url), - 'utf8' - ) - const policy = terraform.match( - /resource "google_monitoring_alert_policy" "relay_cloud_sql_backends" \{([\s\S]*?)\n\}/ - )?.[1] - - assert.ok(policy) - assert.match(policy, /notification_channels\s*=\s*var\.relay_alert_notification_channels/) -}) diff --git a/cloud/dev/scripts/relay-control-flag-alerts.test.mjs b/cloud/dev/scripts/relay-control-flag-alerts.test.mjs deleted file mode 100644 index 95604f29d6e..00000000000 --- a/cloud/dev/scripts/relay-control-flag-alerts.test.mjs +++ /dev/null @@ -1,37 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { fileURLToPath } from 'node:url' - -// Why: the metric keys on the event name and fields the switch-file reader writes. A rename on -// either side silences the alert without failing anything. - -const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8') -const terraform = read('../../infra/terraform/relay-observability.tf') -const channel = read('../../apps/relay/src/relay-control-flag-channel.ts') -const cellFlags = read('../../apps/relay/src/cell-flags.ts') - -const block = (kind, name) => { - const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1] - assert.ok(body, `${kind}.${name} not found in relay-observability.tf`) - return body -} - -test('the metric counts the event and fields the reader writes', () => { - const unreadableLine = /event: 'orca_relay_control_flags_unreadable',\s*object: input\.objectName,\s*failure,/ - assert.match(channel, unreadableLine) - assert.match(cellFlags, /return `cells\/\$\{cellId\}\.json`/) - const metric = block('google_logging_metric', 'relay_control_flags_unreadable') - assert.ok(metric.includes('jsonPayload.event=\\"orca_relay_control_flags_unreadable\\"')) - assert.ok(metric.includes('REGEXP_EXTRACT(jsonPayload.object, \\"cells/([^.]+)\\")')) - assert.ok(metric.includes('failure = "EXTRACT(jsonPayload.failure)"')) -}) - -test('the alert fires on one line per cell and stays in the console', () => { - const alert = block('google_monitoring_alert_policy', 'relay_control_flags_unreadable') - // The reader logs once per change of cause, so a stuck cell writes one line; a higher bar misses it. - assert.match(alert, /threshold_value = 0\n/) - assert.match(alert, /notification_channels = \[\]/) - assert.ok(alert.includes('metric.type=\\"logging.googleapis.com/user/orca_relay_control_flags_unreadable\\"')) - assert.ok(alert.includes('group_by_fields = ["metric.label.\\"cell_id\\""]')) -}) diff --git a/cloud/dev/scripts/relay-db-fault-alerts.test.mjs b/cloud/dev/scripts/relay-db-fault-alerts.test.mjs deleted file mode 100644 index 4f802b0b843..00000000000 --- a/cloud/dev/scripts/relay-db-fault-alerts.test.mjs +++ /dev/null @@ -1,76 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { fileURLToPath } from 'node:url' - -// Why: these alerts key on event names and field paths the relay writes (the rejection fence and -// the lease shadow). A rename on either side silences the alert without failing anything. - -const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8') -const terraform = read('../../infra/terraform/relay-observability.tf') - -const block = (kind, name) => { - const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1] - assert.ok(body, `${kind}.${name} not found in relay-observability.tf`) - return body -} - -const incidentMetric = (key) => { - const body = new RegExp(`\\n ${key} = \\{([\\s\\S]*?)\\n \\}`).exec(terraform)?.[1] - assert.ok(body, `relay_incident_metrics.${key} not found`) - return body -} - -test('the fence metrics count the exact events on directors and cells', () => { - for (const [key, event] of [ - ['database_rejection_fenced', 'orca_relay_database_rejection_fenced'], - ['process_fatal', 'orca_relay_process_fatal'] - ]) { - const metric = incidentMetric(key) - assert.ok(metric.includes(`jsonPayload.event=\\"${event}\\"`), key) - assert.ok(metric.includes('resource.type=\\"cloud_run_revision\\"'), key) - assert.ok(metric.includes('resource.type=\\"gce_instance\\"'), key) - } -}) - -test('the fenced alert is one fleet-wide series, and the fatal alert leaves cells to the exit alert', () => { - const fenced = block('google_monitoring_alert_policy', 'relay_database_rejection_fenced') - assert.ok( - fenced.includes( - 'query = "sum(increase(logging_googleapis_com:user_orca_relay_database_rejection_fenced[5m])) > 0"' - ) - ) - const fatal = block('google_monitoring_alert_policy', 'relay_process_fatal') - assert.ok(fatal.includes('resource.type=\\"cloud_run_revision\\" AND metric.type=\\"logging.googleapis.com/user/orca_relay_process_fatal\\"')) - assert.doesNotMatch(fatal, /gce_instance/) - // The cell side of a fatal rejection still pages, once, through the exit alert. - block('google_monitoring_alert_policy', 'relay_cell_process_exit') -}) - -test('the bad-signature alert stays in the console', () => { - const alert = block('google_monitoring_alert_policy', 'relay_assignment_lease_bad_signature') - assert.match(alert, /notification_channels = \[\]/) -}) - -test('the lease shadow metrics extract every class the cell writes', () => { - const classes = [ - 'agree', - 'disagree', - 'absent', - 'expired', - 'badSignature', - 'wrongHost', - 'wrongCell', - 'epochBehind', - 'epochAhead' - ] - for (const field of classes) { - assert.match(terraform, new RegExp(`field = "classes\\.${field}"`), field) - } - assert.match(terraform, /field = "dbReadMs\.p99"/) - assert.match(terraform, /field = "dbRefused"/) - const metric = block('google_logging_metric', 'relay_assignment_lease_shadow') - assert.ok(metric.includes('jsonPayload.event=\\"orca_relay_assignment_lease_shadow\\"')) - const alert = block('google_monitoring_alert_policy', 'relay_assignment_lease_bad_signature') - assert.ok(alert.includes('user_orca_relay_assignment_lease_shadow_bad_signature_sum')) -}) diff --git a/cloud/dev/scripts/relay-deploy-workflow-argv.test.mjs b/cloud/dev/scripts/relay-deploy-workflow-argv.test.mjs deleted file mode 100644 index 8dea82fe6e9..00000000000 --- a/cloud/dev/scripts/relay-deploy-workflow-argv.test.mjs +++ /dev/null @@ -1,106 +0,0 @@ -import assert from 'node:assert/strict' -import { readdirSync, readFileSync } from 'node:fs' -import test from 'node:test' -import { parseArguments as parseBlueGreen } from './deploy-relay-blue-green.mjs' -import { parseArguments as parseGceCandidate } from './deploy-relay-gce-candidate.mjs' -import { RELAY_WORKFLOW_DIRECTORY, readRelayWorkflow } from './relay-repository.mjs' - -// Every workflow's real argv, literals as written: a flag combination the parser refuses -// (staging's lone --admin-audience once was one) fails here instead of on a release. - -const PROJECT = 'orca-test-project' -const DIGEST = `sha256:${'a'.repeat(64)}` - -// Shell-expanded values only; anything a workflow writes literally is parsed as written. -const SAMPLES = { - project: PROJECT, - image: `relay@${DIGEST}`, - 'predecessor-image-digest': DIGEST, - 'director-cells-json': JSON.stringify([ - { id: 'cell-a', url: 'https://cell-a.example.test', region: 'us-central1', capacityRequests: 10, initiallyEnabled: true } - ]), - 'regional-placement-secret-version': '3', - 'min-instances': '1', - 'max-instances': '5', - 'prune-revisions': 'false', - 'bootstrap-runtime-identity': 'false', - 'expected-rehome-generation': '4', - 'region-correction-cohort-percent': '10', - 'rehome-audience': 'https://relay.example.test/v1/admin/host-drain', - 'director-origin': 'https://relay.example.test', - 'admin-audience': 'https://relay.example.test/v1/admin/drain', - 'reserve-placement': 'preserve', - 'shadow-seat-feed-cells': 'preserve', - mode: 'audit' -} - -function sample(flag) { - if (flag in SAMPLES) return SAMPLES[flag] - if (flag.endsWith('service-account')) return `${flag.slice(0, 20).replace(/-$/, '')}@${PROJECT}.iam.gserviceaccount.com` - return `sample-${flag}` -} - -// Flags a workflow passes through a bash array (`"${name[@]}"`), from every `name=(...)` it -// assigns; parsing them all at once is the widest set that workflow can send. -function arrayArguments(source, name) { - const flags = [] - for (const match of source.matchAll(new RegExp(`\\b${name}=\\(([^)]*)\\)`, 'g'))) { - for (const entry of match[1].matchAll(/(--[a-z0-9-]+)\s+("[^"]*"|\S+)/g)) { - flags.push([entry[1], entry[2].replace(/^"(.*)"$/, '$1')]) - } - } - if (flags.length === 0) throw new Error(`no flags assigned to ${name}`) - return flags -} - -function invocations(source, script) { - const lines = source.split('\n') - const found = [] - for (let index = 0; index < lines.length; index++) { - if (!lines[index].includes(`dev/scripts/${script}`)) continue - const argv = [] - let line = lines[index] - while (line.trimEnd().endsWith('\\')) { - line = lines[++index] - const array = /^\s*"\$\{([a-z_]+)\[@\]\}"\s*\\?\s*$/.exec(line) - if (array) { - for (const [flag, value] of arrayArguments(source, array[1])) { - argv.push(flag, value.includes('$') ? sample(flag.slice(2)) : value) - } - continue - } - const match = /^\s*(--[a-z0-9-]+)\s+(.+?)\s*\\?\s*$/.exec(line) - if (!match) throw new Error(`unparsed ${script} argument line: ${line.trim()}`) - const [, flag, raw] = match - // A trailing `)"` closes the command substitution the invocation sits in. - const value = raw.replace(/\)"$/, '').replace(/^"(.*)"$/, '$1') - argv.push(flag, value.includes('$') ? sample(flag.slice(2)) : value) - } - found.push(argv) - } - return found -} - -const SCRIPTS = [ - ['deploy-relay-blue-green.mjs', parseBlueGreen], - ['deploy-relay-gce-candidate.mjs', parseGceCandidate] -] - -for (const [script, parse] of SCRIPTS) { - test(`every workflow's ${script} argv parses`, () => { - let count = 0 - for (const file of readdirSync(RELAY_WORKFLOW_DIRECTORY).filter((name) => name.endsWith('.yml'))) { - for (const argv of invocations(readFileSync(new URL(file, RELAY_WORKFLOW_DIRECTORY), 'utf8'), script)) { - count++ - assert.doesNotThrow(() => parse(argv), `${file}: ${argv.join(' ')}`) - } - } - assert.ok(count > 0, `no workflow runs ${script}`) - }) -} - -test('the production and staging director workflows are among the parsed', () => { - for (const name of ['deploy-relay-production-director.yml', 'deploy-relay-staging.yml']) { - assert.equal(invocations(readRelayWorkflow(name), 'deploy-relay-blue-green.mjs').length, 1, name) - } -}) diff --git a/cloud/dev/scripts/relay-director-deploy-plan.mjs b/cloud/dev/scripts/relay-director-deploy-plan.mjs deleted file mode 100644 index 5f45de5afb5..00000000000 --- a/cloud/dev/scripts/relay-director-deploy-plan.mjs +++ /dev/null @@ -1,335 +0,0 @@ -// Pure pieces of the director deploy driver: the exact inputs each existing workflow receives, -// and the parsers that read a run's result back. No process, network, or clock access here. - -import { - RELAY_GITHUB_REPOSITORY, - RELAY_WORKFLOW_FILE_PREFIX, - relayWorkflowFile -} from './relay-repository.mjs' - -export const REPOSITORY = RELAY_GITHUB_REPOSITORY -export const WORKFLOW_REF = 'main' -export const PROJECT = 'onorca-cloud' -export const REGION = 'us-central1' -export const DIRECTOR_SERVICE = 'orca-cloud-relay' -export const ROLLBACK_TAG = 'selector-rollback' -export const IMAGE_REPOSITORY = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay' -// Any reviewed registration wave is accepted by admission inspect; the launch wave never changes. -const ADMISSION_INSPECT_CELLS = 'production-gce-c27,production-gce-c28,production-gce-c29' - -const workflow = (name, title) => ({ file: relayWorkflowFile(name), name: title }) -export const WORKFLOWS = { - publish: workflow('publish-relay-production.yml', 'Publish Relay Production Image'), - director: workflow('deploy-relay-production-director.yml', 'Deploy Relay Production Director'), - rehome: workflow('operate-relay-production-rehome.yml', 'Operate Relay Production Rehome'), - admission: workflow('operate-relay-asia-admission.yml', 'Operate Relay Asia Admission'), - monitor: workflow('monitor-relay-production.yml', 'Monitor Relay Production') -} - -// Read-only or unrelated to the production rollout lane, so they never block a deploy. -const NON_BLOCKING_WORKFLOWS = new Set( - ['verify.yml', 'monitor-relay-clock-skew.yml'].map(relayWorkflowFile) -) - -const DIGEST = /^sha256:[a-f0-9]{64}$/ -const COMMIT = /^[a-f0-9]{40}$/ -const PRODUCTION_CELL = /^production-gce-c[1-9][0-9]?$/ -const MEMBERSHIP_KEYS = ['existingOnly', 'migrationOnly', 'general'] - -export function requireDigest(value, label) { - if (typeof value !== 'string' || !DIGEST.test(value)) - throw new Error(`${label} is not an immutable sha256 digest`) - return value -} - -export function requireCommit(value, label) { - if (typeof value !== 'string' || !COMMIT.test(value)) - throw new Error(`${label} is not a full commit SHA`) - return value -} - -function requireGeneration(value, label) { - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${label} is invalid`) - return value -} - -export function blocksDeploy(path) { - // A `@ref` suffix never appears on run paths today; stripping it keeps the check fail-closed. - const file = String(path ?? '') - .split('@')[0] - .split('/') - .at(-1) - return ( - file.startsWith(RELAY_WORKFLOW_FILE_PREFIX) && - file.endsWith('.yml') && - !NON_BLOCKING_WORKFLOWS.has(file) - ) -} - -export function membershipInput(cells) { - return cells.length === 0 ? 'none' : [...cells].sort().join(',') -} - -export function parseSelector(value, label) { - const selector = { - generation: requireGeneration(value?.generation, `${label} generation`), - membership: {} - } - for (const key of MEMBERSHIP_KEYS) { - const cells = value?.membership?.[key] - if (!Array.isArray(cells) || cells.some((cell) => !PRODUCTION_CELL.test(cell))) { - throw new Error(`${label} ${key} membership is invalid`) - } - selector.membership[key] = [...cells].sort() - } - const all = MEMBERSHIP_KEYS.flatMap((key) => selector.membership[key]) - if (new Set(all).size !== all.length) throw new Error(`${label} membership has duplicates`) - return selector -} - -function selectorInputs(selector) { - return { - 'expected-selector-generation': String(selector.generation), - 'expected-existing-only-cells': membershipInput(selector.membership.existingOnly), - 'expected-migration-only-cells': membershipInput(selector.membership.migrationOnly), - 'expected-general-cells': membershipInput(selector.membership.general) - } -} - -export function parseControl(value, label) { - const control = value ?? {} - const integers = [ - 'generation', - 'notBefore', - 'ratePerMinute', - 'preferenceMaxAgeMs', - 'drainGraceMs' - ] - if ( - typeof control.enabled !== 'boolean' || - integers.some((key) => !Number.isSafeInteger(control[key])) - ) { - throw new Error(`${label} is not a complete regional rehome control`) - } - if (control.hostCooldownMs !== undefined && !Number.isSafeInteger(control.hostCooldownMs)) { - throw new Error(`${label} host cooldown is invalid`) - } - return Object.fromEntries( - [...integers, 'enabled', 'hostCooldownMs'].map((key) => [key, control[key]]) - ) -} - -const INTEGER_INPUT = /^(0|[1-9][0-9]*)$/ - -/** - * The last check before `gh workflow run`. Builders also render dry-run placeholders such as - * ``; this guarantees none of them, or a malformed digest, is ever dispatched. - */ -export function validateDispatchInputs(inputs) { - for (const [key, value] of Object.entries(inputs)) { - if (typeof value !== 'string' || value.startsWith('<')) - throw new Error(`input ${key} is not resolved`) - if (key.endsWith('digest') && !DIGEST.test(value)) - throw new Error(`input ${key} is not a sha256 digest`) - if ( - (key.includes('generation') || - ['not-before', 'monitor-run-id', 'monitor-run-attempt'].includes(key)) && - !INTEGER_INPUT.test(value) - ) { - throw new Error(`input ${key} is not an integer`) - } - } - return inputs -} - -export function admissionInspectInputs(servingDigest) { - return { - environment: 'production', - mode: 'inspect', - 'cell-ids': ADMISSION_INSPECT_CELLS, - 'image-digest': servingDigest - } -} - -function rehomeInputs(mode, { director, selector, controlGeneration }) { - return { - mode, - 'director-image-digest': director.servingDigest, - 'rollback-image-digest': director.rollbackDigest, - ...selectorInputs(selector), - 'expected-control-generation': String(controlGeneration) - } -} - -export function rehomeInspectInputs(context) { - return rehomeInputs('inspect', context) -} - -// Pause keeps every durable field as inspected; only `enabled` and the generation change. -// Every `confirmation` is the phrase the operator typed, never filled in here. -export function rehomePauseInputs({ director, selector, control, confirmation }) { - return { - ...rehomeInputs('pause', { director, selector, controlGeneration: control.generation }), - 'not-before': String(control.notBefore), - 'rate-per-minute': String(control.ratePerMinute), - 'preference-max-age-ms': String(control.preferenceMaxAgeMs), - 'host-cooldown-ms': String(control.hostCooldownMs ?? 604_800_000), - 'drain-grace-ms': String(control.drainGraceMs), - confirmation - } -} - -export function rehomeEnableInputs({ - director, - selector, - control, - controlGeneration, - notBefore, - monitor, - confirmation -}) { - return { - ...rehomeInputs('enable', { director, selector, controlGeneration }), - 'not-before': String(notBefore), - // The enable job accepts exactly 10 per minute. - 'rate-per-minute': '10', - 'preference-max-age-ms': String(control.preferenceMaxAgeMs), - 'host-cooldown-ms': String(control.hostCooldownMs), - 'drain-grace-ms': String(control.drainGraceMs), - 'monitor-run-id': String(monitor.runId), - 'monitor-run-attempt': String(monitor.attempt), - confirmation - } -} - -export function publishInputs() { - return { mode: 'publish' } -} - -export function directorDeployInputs({ imageDigest, predecessorDigest, rehomeGeneration }) { - return { - 'image-digest': imageDigest, - 'regional-placement-mode': 'preserve', - 'region-correction-cohort-percent': 'preserve', - 'reserve-placement': 'preserve', - 'shadow-seat-feed-cells': 'preserve', - 'prune-incompatible-revisions': 'false', - 'expected-rehome-generation': String(rehomeGeneration), - 'bootstrap-runtime-identity': 'false', - // Required by the form even without the bootstrap; it is only format-checked then. - 'predecessor-image-digest': predecessorDigest - } -} - -export function configureInputs({ - cells, - cellImageDigest, - directorDigest, - selectorGeneration, - confirmation -}) { - return { - environment: 'production', - mode: 'configure', - 'cell-ids': cells.join(','), - 'selector-generation': String(selectorGeneration), - 'image-digest': cellImageDigest, - 'director-image-digest': directorDigest, - confirmation - } -} - -export function monitorDryRunInputs(selector) { - return { - mode: 'dry-run', - ...selectorInputs(selector), - 'migration-policy': 'strict', - 'recovery-source-cell-id': 'none', - 'capacity-cell-id': 'none' - } -} - -export function parseConfigureWave(value) { - const separator = String(value).lastIndexOf('=') - const cells = String(value) - .slice(0, separator) - .split(',') - .map((cell) => cell.trim()) - .filter(Boolean) - const cellImageDigest = String(value).slice(separator + 1) - if ( - separator < 1 || - cells.length === 0 || - new Set(cells).size !== cells.length || - cells.some((cell) => !PRODUCTION_CELL.test(cell)) - ) { - throw new Error('--configure must be [,...]=sha256:') - } - return { cells, cellImageDigest: requireDigest(cellImageDigest, '--configure cell image digest') } -} - -/** The single 100% revision and the selector-rollback revision of `gcloud run services describe`. */ -export function directorRevisions(service) { - const traffic = Array.isArray(service?.status?.traffic) ? service.status.traffic : [] - const serving = traffic.filter((entry) => (entry.percent ?? 0) > 0) - const rollback = traffic.filter((entry) => entry.tag === ROLLBACK_TAG) - if (serving.length !== 1 || serving[0].percent !== 100 || !serving[0].revisionName) { - throw new Error('director does not serve one revision at 100%') - } - if (rollback.length !== 1 || !rollback[0].revisionName) { - throw new Error(`director has no single ${ROLLBACK_TAG} revision`) - } - return { servingRevision: serving[0].revisionName, rollbackRevision: rollback[0].revisionName } -} - -export function revisionDigest(revision, label) { - const image = revision?.spec?.containers?.[0]?.image - const [repository, digest] = String(image ?? '').split('@') - if (repository !== IMAGE_REPOSITORY) - throw new Error(`${label} does not run the production relay image`) - return requireDigest(digest, `${label} image`) -} - -/** The relay push line, `sha-: digest: sha256:… size: …`, must name the registry digest. */ -export function logConfirmsPublishedDigest(log, commit, digest) { - return String(log) - .split('\n') - .some((line) => line.includes(`sha-${commit}: digest: ${digest} size:`)) -} - -/** The last `relay_regional_rehome_control` JSON line a rehome run printed, of any mode. */ -export function rehomeResultFromLog(log) { - let found - for (const line of String(log).split('\n')) { - const start = line.indexOf('{"event":"relay_regional_rehome_control"') - if (start < 0) continue - try { - found = JSON.parse(line.slice(start).trim()) - } catch { - // A truncated or echoed line is not the result. - } - } - if (!found) throw new Error('the rehome run printed no control result') - return { - mode: found.mode, - recovered: found.recovered, - control: parseControl(found.control, `rehome ${found.mode} control`) - } -} - -/** - * The generation a run paused rehome at, or undefined. Only two lines prove a run paused it: a - * `pause`, and a failed enable's `recover-enable` that itself disabled rehome (`recovered: true`). - * `recovered: false` means rehome was already disabled, possibly by a director safety pause. - */ -export function pausedGeneration(result) { - const paused = - !result.control.enabled && - (result.mode === 'pause' || (result.mode === 'recover-enable' && result.recovered === true)) - return paused ? result.control.generation : undefined -} - -export function admissionInspectResult(result) { - if (result?.mode !== 'inspect') throw new Error('admission result is not an inspect') - return parseSelector(result, 'admission selector') -} diff --git a/cloud/dev/scripts/relay-evidence-code-provenance.mjs b/cloud/dev/scripts/relay-evidence-code-provenance.mjs deleted file mode 100644 index dba11559e6f..00000000000 --- a/cloud/dev/scripts/relay-evidence-code-provenance.mjs +++ /dev/null @@ -1,101 +0,0 @@ -import { spawnSync } from 'node:child_process' -import { fileURLToPath } from 'node:url' -import { - RELAY_REPOSITORY_ROOT, - relayTreePath, - relayWorkflowPath -} from './relay-repository.mjs' - -const SHA = /^[a-f0-9]{40}$/ - -// Every file that decides how relay evidence is produced, sealed, verified, and then spent against -// production; identical content across two commits is what makes the older commit's verdict binding. -export const TRUSTED_EVIDENCE_CODE_PATHS = [ - // Produces and seals the 15-minute dry-run evidence. - relayWorkflowPath('monitor-relay-production.yml'), - relayWorkflowPath('monitor-relay-production-job.yml'), - // Rehome enable downloads it, verifies its authority, and mutates production on it. - relayWorkflowPath('operate-relay-production-rehome.yml'), - relayWorkflowPath('operate-relay-production-rehome-job.yml'), - // Same-cap seals and spends canary authority, which a later batch accepts only from this code. - relayWorkflowPath('deploy-relay-production-same-cap.yml'), - relayWorkflowPath('deploy-relay-production-same-cap-job.yml'), - // Sealing, verification, the wave/canary authority, and the path constants below. - relayTreePath('dev/scripts/relay-evidence-code-provenance.mjs'), - relayTreePath('dev/scripts/relay-monitor-evidence.mjs'), - relayTreePath('dev/scripts/relay-production-same-cap-wave.mjs'), - relayTreePath('dev/scripts/relay-repository.mjs'), - // Every other script those jobs run against live production. - relayTreePath('dev/scripts/check-relay-same-cap-headroom.mjs'), - relayTreePath('dev/scripts/infra.mjs'), - relayTreePath('dev/scripts/operate-relay-regional-rehome.mjs'), - relayTreePath('dev/scripts/prepare-relay-production-capacity-canary.mjs'), - relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'), - relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'), - relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'), - relayTreePath('dev/scripts/relay-admin-transient-retry.mjs'), - relayTreePath('dev/scripts/relay-reserve-cell-refusal.mjs'), - // The exit metric filter the same-cap pre-drain sample's crash rule reads. - relayTreePath('infra/terraform/relay-observability.tf'), - // The monitor, the live preflight recheck, and the same-cap pre-drain sample, plus anything that - // changes their behaviour. - relayTreePath('apps/relay-ops'), - relayTreePath('package.json'), - relayTreePath('pnpm-lock.yaml'), - relayTreePath('pnpm-workspace.yaml'), - // The Cloud SQL rollout lease every mutation job takes and releases. - '.github/actions/cloud-sql-rollout-lease' -] - -function git(root, args) { - const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' }) - if (result.error) throw new Error('relay evidence provenance cannot run git') - return result -} - -/** - * Accepts evidence sealed at a different commit only when the current commit descends from it and - * every trusted path is byte-identical, so the verdict provably came from this exact code. Anything - * git cannot answer (no checkout, unknown commit, shallow clone) fails closed. - */ -export function requireSameEvidenceCode({ - sealedSha, - currentSha, - label, - repositoryRoot = fileURLToPath(RELAY_REPOSITORY_ROOT) -}) { - if (!SHA.test(sealedSha ?? '') || !SHA.test(currentSha ?? '')) { - throw new Error(`${label} commit is invalid`) - } - if (sealedSha === currentSha) return - if (git(repositoryRoot, ['rev-parse', '--git-dir']).status !== 0) { - throw new Error(`${label} commit cannot be compared without a git checkout`) - } - for (const sha of [sealedSha, currentSha]) { - if (git(repositoryRoot, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`]).status !== 0) { - throw new Error( - `${label} commit ${sha} is unknown to this checkout; check out with fetch-depth: 0` - ) - } - } - const ancestry = git(repositoryRoot, ['merge-base', '--is-ancestor', sealedSha, currentSha]) - if (ancestry.status === 1) { - throw new Error(`${label} commit ${sealedSha} is not an ancestor of ${currentSha}`) - } - if (ancestry.status !== 0) { - throw new Error(`${label} commit ancestry could not be determined`) - } - const diff = git(repositoryRoot, [ - 'diff', - '--name-only', - sealedSha, - currentSha, - '--', - ...TRUSTED_EVIDENCE_CODE_PATHS - ]) - if (diff.status !== 0) throw new Error(`${label} commit comparison failed`) - const changed = diff.stdout.split('\n').filter(Boolean) - if (changed.length > 0) { - throw new Error(`${label} code changed after it was sealed: ${changed.join(',')}`) - } -} diff --git a/cloud/dev/scripts/relay-gce-terraform-fence.mjs b/cloud/dev/scripts/relay-gce-terraform-fence.mjs deleted file mode 100644 index 82b9dcbff6c..00000000000 --- a/cloud/dev/scripts/relay-gce-terraform-fence.mjs +++ /dev/null @@ -1,1387 +0,0 @@ -import { execFileSync } from 'node:child_process' -import { createHash, randomUUID } from 'node:crypto' -import { - chmodSync, - mkdtempSync, - readFileSync, - rmSync, - statSync -} from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' - -const MIG_ADDRESS_PREFIX = 'google_compute_instance_group_manager.relay_gce_cell' -const PLAN_OBJECT_PREFIX = 'terraform/state/relay-fence-plans' -const TERRAFORM_STATE_LINEAGE = /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i -const GENERATED_MIG_VERSION_NAME = - /^0\/[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?\+00:00$/ - -function changedActions(change) { - return change.change.actions.filter((action) => action !== 'no-op' && action !== 'read') -} - -export function validateTerraformFencePlan(plan, expected) { - const changes = (plan.resource_changes ?? []).filter( - (change) => changedActions(change).length > 0 - ) - if (changes.length !== 1) throw new Error('fence plan must contain exactly one mutation') - const [change] = changes - if (change.address !== `${MIG_ADDRESS_PREFIX}["${expected.cellId}"]`) { - throw new Error('fence plan mutates an unexpected resource') - } - if ( - JSON.stringify(change.change.actions) !== JSON.stringify(['update']) || - Number(change.change.before?.target_size) !== 1 || - Number(change.change.after?.target_size) !== 0 - ) { - throw new Error('fence plan must update only the requested MIG from one to zero') - } - for (const field of ['name', 'zone', 'instance_group']) { - if ( - change.change.before?.[field] !== change.change.after?.[field] || - change.change.after?.[field] !== expected[field] - ) { - throw new Error(`fence plan changed or mismatched MIG ${field}`) - } - } - const beforeGeneration = change.change.before?.version?.[0]?.instance_template - const afterGeneration = change.change.after?.version?.[0]?.instance_template - if ( - beforeGeneration !== afterGeneration || - afterGeneration !== expected.generationIdentity - ) { - throw new Error('fence plan changed or mismatched the MIG generation') - } - return change -} - -export function validateTerraformFenceCompletionPlan(plan, expected) { - const changes = (plan.resource_changes ?? []).filter( - (change) => changedActions(change).length > 0 - ) - if (changes.length === 0) return - if (changes.length !== 1) { - throw new Error('completed fence plan contains an unexpected mutation') - } - const [change] = changes - const before = change.change.before - const after = change.change.after - const beforeVersion = before?.version?.[0] - const afterVersion = after?.version?.[0] - if ( - change.address !== `${MIG_ADDRESS_PREFIX}["${expected.cellId}"]` || - JSON.stringify(change.change.actions) !== JSON.stringify(['update']) || - Number(before?.target_size) !== 0 || - Number(after?.target_size) !== 0 || - before?.name !== expected.name || - after?.name !== expected.name || - before?.zone !== expected.zone || - after?.zone !== expected.zone || - before?.instance_group !== expected.instance_group || - after?.instance_group !== expected.instance_group || - before?.version?.length !== 1 || - after?.version?.length !== 1 || - beforeVersion?.instance_template !== expected.generationIdentity || - afterVersion?.instance_template !== expected.generationIdentity || - !GENERATED_MIG_VERSION_NAME.test(beforeVersion?.name ?? '') || - afterVersion?.name !== 'primary' - ) { - throw new Error('completed fence plan is not a safe provider normalization') - } - const normalizedBefore = structuredClone(before) - normalizedBefore.version[0].name = afterVersion.name - if (JSON.stringify(normalizedBefore) !== JSON.stringify(after)) { - throw new Error('completed fence plan changes more than the provider version label') - } - return change -} - -export function terraformFenceState(state, expected) { - const resources = state.values?.root_module?.resources ?? [] - const matches = resources.filter( - (resource) => resource.address === `${MIG_ADDRESS_PREFIX}["${expected.cellId}"]` - ) - if (matches.length !== 1) throw new Error('Terraform state has no unique requested MIG') - const values = matches[0].values - if ( - values.name !== expected.name || - values.zone !== expected.zone || - values.instance_group !== expected.instance_group || - values.version?.[0]?.instance_template !== expected.generationIdentity - ) { - throw new Error('Terraform state MIG identity does not match the reviewed topology') - } - const targetSize = Number(values.target_size) - if (![0, 1].includes(targetSize)) throw new Error('Terraform state MIG size is unsafe') - return targetSize -} - -export function classifyTerraformFenceProgress({ - stateTargetSize, - liveTargetSize, - instanceCount, - operationStatus, - operationError = false, - operationAuditBound = false -}) { - if (operationError) throw new Error('Terraform fence GCE operation failed') - if (operationStatus === 'DONE' && !operationAuditBound) { - throw new Error('Terraform fence GCE operation lacks exact audit binding') - } - if ( - stateTargetSize === 0 && - liveTargetSize === 0 && - instanceCount === 0 && - operationStatus === 'DONE' - ) { - return 'complete' - } - if ( - [0, 1].includes(stateTargetSize) && - [0, 1].includes(liveTargetSize) && - ['PENDING', 'RUNNING'].includes(operationStatus) - ) { - return 'in-progress' - } - if ( - stateTargetSize === 1 && - liveTargetSize === 0 && - instanceCount === 0 && - operationStatus === 'DONE' - ) { - return 'reconcile-state' - } - if ( - stateTargetSize === 1 && - liveTargetSize === 1 && - instanceCount === 1 && - operationStatus === 'ABSENT' - ) { - return 'not-started' - } - throw new Error('Terraform fence progress is ambiguous or unsafe') -} - -function sha256(path, readFile) { - return createHash('sha256').update(readFile(path)).digest('hex') -} - -function terraformJson(deps, args) { - return JSON.parse(deps.terraform(args, { encoding: 'utf8' })) -} - -export function terraformProcessStdio(options = {}) { - if (!options.encoding) return 'inherit' - return [options.input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'] -} - -function defaultTerraform(args, options = {}) { - return execFileSync(process.env.IAC_TOOL || 'terraform', args, { - ...options, - stdio: terraformProcessStdio(options) - }) -} - -function defaultGit(args, options = {}) { - return execFileSync('git', args, { - ...options, - stdio: options.encoding ? ['ignore', 'pipe', 'pipe'] : 'inherit' - }) -} - -function privatePlanDirectory(deps) { - const previousMask = process.umask(0o077) - try { - const directory = deps.mkdtemp(join(deps.tmpdir(), 'orca-relay-fence-')) - deps.chmod(directory, 0o700) - return directory - } finally { - process.umask(previousMask) - } -} - -function exactMigExpected(cell) { - return { - cellId: cell.cellId, - name: cell.migName, - zone: cell.zone, - instance_group: cell.instanceGroup, - generationIdentity: cell.generationIdentity - } -} - -function assertFenceIdentity(cell) { - if (!cell.generationIdentity) throw new Error('requested cell has no generation identity') -} - -function assertAttemptMatches(config, attempt, requirePlanGeneration = true) { - for (const [field, expected] of Object.entries({ - environment: config.environment, - cellId: config.cell.cellId, - cellIncarnation: config.cellIncarnation, - migName: config.cell.migName, - instanceGroup: config.cell.instanceGroup, - generationIdentity: config.cell.generationIdentity, - fenceCommit: config.fenceCommit - })) { - if (attempt[field] !== expected) throw new Error(`fence attempt ${field} mismatch`) - } - if (!/^[a-f0-9]{64}$/.test(attempt.planSha256 ?? '')) { - throw new Error('fence attempt has no valid saved-plan digest') - } - if ( - attempt.planObjectName !== - `${PLAN_OBJECT_PREFIX}/${config.environment}/${attempt.attemptId}.tfplan` - ) { - throw new Error('fence attempt saved-plan object mismatch') - } - if ( - requirePlanGeneration && - !/^[1-9][0-9]{0,30}$/.test(attempt.planObjectGeneration ?? '') - ) { - throw new Error('fence attempt has no valid saved-plan generation') - } - if (!/^[a-f0-9]{64}$/.test(attempt.varFileSha256 ?? '')) { - throw new Error('fence attempt has no valid variable-file digest') - } - if ( - !TERRAFORM_STATE_LINEAGE.test(attempt.terraformStateLineage ?? '') || - !Number.isSafeInteger(attempt.terraformStateSerial) || - attempt.terraformStateSerial < 0 - ) { - throw new Error('fence attempt has no valid Terraform state identity') - } - if ( - !/^[1-9][0-9]{0,30}$/.test( - attempt.terraformStateObjectGeneration ?? '' - ) || - !/^[a-f0-9]{64}$/.test(attempt.terraformStateObjectSha256 ?? '') - ) { - throw new Error('fence attempt has no valid Terraform state object binding') - } - if (attempt.requestReason !== `orca-relay-fence/${attempt.attemptId}`) { - throw new Error('fence attempt request-reason mismatch') - } -} - -export function terraformFenceStateIdentity(config, deps = {}) { - const terraform = deps.terraform ?? defaultTerraform - const state = terraformJson({ terraform }, [ - `-chdir=${config.terraformDir}`, - 'state', - 'pull' - ]) - if ( - !TERRAFORM_STATE_LINEAGE.test(state.lineage ?? '') || - !Number.isSafeInteger(state.serial) || - state.serial < 0 - ) { - throw new Error('Terraform state has no valid lineage or serial') - } - return { lineage: state.lineage, serial: state.serial } -} - -export function assertReviewedFenceCheckout(config, deps = {}) { - const environment = deps.environment ?? process.env - const git = deps.git ?? defaultGit - const readFile = deps.readFile ?? readFileSync - const varPath = join(config.terraformDir, config.varFile) - const imageCommit = environment.ORCA_RELAY_FENCE_IMAGE_COMMIT - if (imageCommit !== undefined) { - if (!/^[a-f0-9]{40}$/.test(imageCommit) || imageCommit !== config.fenceCommit) { - throw new Error('fence commit does not match immutable broker image') - } - return sha256(varPath, readFile) - } - const head = git(['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() - if (head !== config.fenceCommit) throw new Error('fence commit does not match checked-out HEAD') - const status = git(['status', '--porcelain=v1', '--untracked-files=no'], { - encoding: 'utf8' - }).trim() - if (status) throw new Error('Terraform fence requires a clean checkout') - const terraformStatus = git( - ['status', '--porcelain=v1', '--untracked-files=all', '--', config.terraformDir], - { encoding: 'utf8' } - ).trim() - if (terraformStatus) throw new Error('Terraform fence directory contains unreviewed files') - git(['ls-files', '--error-unmatch', '--', varPath], { encoding: 'utf8' }) - return sha256(varPath, readFile) -} - -function assertAttemptCheckoutBinding(config, attempt, deps) { - const digest = assertReviewedFenceCheckout(config, deps) - if (digest !== attempt.varFileSha256) { - throw new Error('reviewed Terraform variable-file digest changed') - } -} - -function assertReplayStateIdentity(progress, attempt) { - if ( - progress.stateLineage !== attempt.terraformStateLineage || - progress.stateSerial !== attempt.terraformStateSerial - ) { - throw new Error('Terraform state lineage or serial changed before saved-plan replay') - } -} - -function assertStateObjectBinding(binding, attempt) { - if ( - binding.generation !== attempt.terraformStateObjectGeneration || - binding.sha256 !== attempt.terraformStateObjectSha256 || - binding.lineage !== attempt.terraformStateLineage || - binding.serial !== attempt.terraformStateSerial - ) { - throw new Error('Terraform pre-state object generation or digest changed') - } -} - -function completedStateBranch(progress, attempt) { - if (progress.stateLineage !== attempt.terraformStateLineage) { - throw new Error('completed Terraform fence state identity is unexpected') - } - if (progress.stateSerial === attempt.terraformStateSerial) return 'replay' - if (progress.stateSerial === attempt.terraformStateSerial + 1) return 'complete' - throw new Error('completed Terraform fence state identity is unexpected') -} - -async function persistInvocationOperations(attempt, progress, markOperation) { - let updated = attempt - for (const observed of progress.invocationOperations ?? []) { - const recorded = (updated.applyInvocations ?? []).find( - (value) => value.invocationId === observed.invocationId - ) - if (!observed.gceOperation || recorded?.gceOperation) continue - const marked = await markOperation( - { ...updated, gceOperation: observed.gceOperation }, - observed - ) - updated = { - ...marked.attempt, - applyInvocations: (updated.applyInvocations ?? []).map((value) => - value.invocationId === marked.invocation.invocationId - ? marked.invocation - : value - ) - } - } - return updated -} - -export function assertTerraformFenceZeroDiff(config, deps = {}) { - const terraform = deps.terraform ?? defaultTerraform - const directory = privatePlanDirectory({ - mkdtemp: deps.mkdtemp ?? mkdtempSync, - chmod: deps.chmod ?? chmodSync, - tmpdir: deps.tmpdir ?? tmpdir - }) - const planPath = join(directory, 'completion.tfplan') - try { - terraform( - [ - `-chdir=${config.terraformDir}`, - 'plan', - '-input=false', - '-refresh=false', - `-lock-timeout=${config.lockTimeout}`, - `-var-file=${config.varFile}`, - `-target=${MIG_ADDRESS_PREFIX}["${config.cell.cellId}"]`, - `-out=${planPath}`, - '-no-color' - ], - { encoding: 'utf8' } - ) - const plan = terraformJson({ terraform }, [ - `-chdir=${config.terraformDir}`, - 'show', - '-json', - planPath - ]) - validateTerraformFenceCompletionPlan(plan, exactMigExpected(config.cell)) - } catch { - throw new Error('completed Terraform fence has an unsafe reviewed diff') - } finally { - ;(deps.remove ?? rmSync)(directory, { recursive: true, force: true }) - } -} - -export function assertTerraformFenceStateFenced(config, deps = {}) { - const terraform = deps.terraform ?? defaultTerraform - const state = terraformJson({ terraform }, [ - `-chdir=${config.terraformDir}`, - 'show', - '-json' - ]) - if (terraformFenceState(state, exactMigExpected(config.cell)) !== 0) { - throw new Error('Terraform state does not record the requested cell fence') - } -} - -export async function adoptLegacyTerraformFence(config, deps) { - for (const dependency of [ - 'loadAttempt', - 'assertCommittedFenceSet', - 'assertStateFenced', - 'preApplyGuard', - 'postApplyGuard', - 'attest', - 'commitAdoption' - ]) { - if (typeof deps[dependency] !== 'function') throw new Error(`missing ${dependency} dependency`) - } - assertFenceIdentity(config.cell) - assertReviewedFenceCheckout(config, deps) - if (await deps.loadAttempt()) { - throw new Error('legacy Terraform fence adoption requires no durable attempt') - } - await deps.assertCommittedFenceSet() - await deps.preApplyGuard() - await deps.assertStateFenced() - await deps.postApplyGuard(config.cellIncarnation) - if (await deps.loadAttempt()) { - throw new Error('durable fence attempt appeared during legacy adoption') - } - await deps.attest(config.cellIncarnation) - await deps.postApplyGuard(config.cellIncarnation) - await deps.commitAdoption(config.cellIncarnation) - deps.emit?.({ - event: 'terraform_cell_fence_legacy_adopted', - cellId: config.cell.cellId - }) -} - -function planObjectUri(config, attempt) { - return `gs://${config.project}-terraform-state/${attempt.planObjectName}` -} - -export async function readTerraformStateObjectBinding( - config, - deps, - statePath -) { - const uri = `gs://${config.project}-terraform-state/terraform/state/default.tfstate` - const metadata = deps.commandJson([ - 'storage', - 'objects', - 'describe', - uri, - '--format=json' - ]) - const generation = String(metadata.generation ?? '') - if (!/^[1-9][0-9]{0,30}$/.test(generation)) { - throw new Error('Terraform state object has no valid generation') - } - deps.command([ - 'storage', - 'cp', - `${uri}#${generation}`, - statePath, - `--if-generation-match=${generation}`, - '--quiet' - ]) - ;(deps.chmod ?? chmodSync)(statePath, 0o600) - const contents = (deps.readFile ?? readFileSync)(statePath) - const state = JSON.parse(contents.toString()) - if ( - !TERRAFORM_STATE_LINEAGE.test(state.lineage ?? '') || - !Number.isSafeInteger(state.serial) || - state.serial < 0 - ) { - throw new Error('Terraform state object identity is invalid') - } - return { - generation, - sha256: createHash('sha256').update(contents).digest('hex'), - lineage: state.lineage, - serial: state.serial - } -} - -export async function uploadTerraformFencePlan(config, deps, planPath, attempt) { - const uri = planObjectUri(config, attempt) - deps.command([ - 'storage', - 'cp', - planPath, - uri, - '--if-generation-match=0', - '--quiet' - ]) - const metadata = deps.commandJson([ - 'storage', - 'objects', - 'describe', - uri, - '--format=json' - ]) - const generation = String(metadata.generation ?? '') - if (!/^[1-9][0-9]{0,30}$/.test(generation)) { - throw new Error('uploaded fence plan has no valid object generation') - } - return { generation } -} - -export async function resolveTerraformFencePlanGeneration(config, deps, attempt) { - const result = deps.commandResult([ - 'storage', - 'objects', - 'describe', - planObjectUri(config, attempt), - '--format=value(generation)' - ]) - if (result.status !== 0) { - if (/(?:404|not found|no urls matched)/i.test(result.stderr ?? '')) { - return { generation: null } - } - throw new Error('durable fence plan object could not be inspected') - } - const generation = String(result.stdout ?? '').trim() - if (!/^[1-9][0-9]{0,30}$/.test(generation)) { - throw new Error('durable fence plan has no valid object generation') - } - return { generation } -} - -export async function downloadTerraformFencePlan(config, deps, attempt, planPath) { - const uri = `${planObjectUri(config, attempt)}#${attempt.planObjectGeneration}` - deps.command([ - 'storage', - 'cp', - uri, - planPath, - `--if-generation-match=${attempt.planObjectGeneration}`, - '--quiet' - ]) - ;(deps.chmod ?? chmodSync)(planPath, 0o600) -} - -export async function deleteTerraformFencePlan(config, deps, attempt) { - const result = deps.commandResult([ - 'storage', - 'rm', - `${planObjectUri(config, attempt)}#${attempt.planObjectGeneration}`, - `--if-generation-match=${attempt.planObjectGeneration}`, - '--quiet' - ]) - if (result.status === 0) return - if (/(?:404|not found|no urls matched)/i.test(result.stderr ?? '')) return - throw new Error('exact Terraform fence plan generation could not be deleted') -} - -export async function runTerraformFenceApply(config, overrides = {}) { - const terraform = overrides.terraform ?? defaultTerraform - const deps = { - terraform, - git: overrides.git ?? defaultGit, - mkdtemp: overrides.mkdtemp ?? mkdtempSync, - chmod: overrides.chmod ?? chmodSync, - tmpdir: overrides.tmpdir ?? tmpdir, - readFile: overrides.readFile ?? readFileSync, - remove: overrides.remove ?? rmSync, - stat: overrides.stat ?? statSync, - randomUUID: overrides.randomUUID ?? randomUUID, - inspectProgress: overrides.inspectProgress, - prepareAttempt: overrides.prepareAttempt, - bindPlan: overrides.bindPlan, - markApplyStarted: overrides.markApplyStarted, - markOperation: overrides.markOperation, - attest: overrides.attest, - uploadPlan: overrides.uploadPlan, - deletePlan: overrides.deletePlan, - stateObjectBinding: overrides.stateObjectBinding, - assertZeroDiff: - overrides.assertZeroDiff ?? - (async () => assertTerraformFenceZeroDiff(config, { terraform })), - preApplyGuard: overrides.preApplyGuard, - postApplyGuard: overrides.postApplyGuard, - assertCommittedFenceSet: - overrides.assertCommittedFenceSet ?? - (() => assertTerraformFenceSet(config, { terraform })), - emit: overrides.emit ?? (() => {}) - } - for (const dependency of [ - 'inspectProgress', - 'prepareAttempt', - 'bindPlan', - 'markApplyStarted', - 'markOperation', - 'attest', - 'uploadPlan', - 'deletePlan', - 'stateObjectBinding', - 'assertZeroDiff', - 'preApplyGuard', - 'postApplyGuard' - ]) { - if (typeof deps[dependency] !== 'function') throw new Error(`missing ${dependency} dependency`) - } - assertFenceIdentity(config.cell) - const varFileSha256 = assertReviewedFenceCheckout(config, deps) - await deps.assertCommittedFenceSet() - const expected = exactMigExpected(config.cell) - const directory = privatePlanDirectory(deps) - const planPath = join(directory, 'fence.tfplan') - try { - const initialProgress = await deps.inspectProgress(expected, null) - if (classifyTerraformFenceProgress(initialProgress) !== 'not-started') { - throw new Error('Terraform fence is not in the exact initial live state') - } - const stateBinding = await deps.stateObjectBinding( - join(directory, 'pre-state.tfstate') - ) - deps.terraform([ - `-chdir=${config.terraformDir}`, - 'plan', - '-input=false', - '-refresh=false', - `-lock-timeout=${config.lockTimeout}`, - `-var-file=${config.varFile}`, - `-target=${MIG_ADDRESS_PREFIX}["${config.cell.cellId}"]`, - `-out=${planPath}` - ]) - deps.chmod(planPath, 0o600) - const postPlanStateBinding = await deps.stateObjectBinding( - join(directory, 'post-plan-state.tfstate') - ) - if ( - JSON.stringify(postPlanStateBinding) !== JSON.stringify(stateBinding) - ) { - throw new Error('Terraform state object changed while creating the fence plan') - } - if ((deps.stat(planPath).mode & 0o077) !== 0) { - throw new Error('saved fence plan permissions are not private') - } - const plan = terraformJson(deps, [ - `-chdir=${config.terraformDir}`, - 'show', - '-json', - planPath - ]) - validateTerraformFencePlan(plan, expected) - const planSha256 = sha256(planPath, deps.readFile) - const attemptId = deps.randomUUID() - const planObjectName = - `${PLAN_OBJECT_PREFIX}/${config.environment}/${attemptId}.tfplan` - const attempt = { - attemptId, - environment: config.environment, - cellId: config.cell.cellId, - cellIncarnation: config.cellIncarnation, - migName: config.cell.migName, - instanceGroup: config.cell.instanceGroup, - generationIdentity: config.cell.generationIdentity, - fenceCommit: config.fenceCommit, - planSha256, - planObjectName, - varFileSha256, - terraformStateLineage: stateBinding.lineage, - terraformStateSerial: stateBinding.serial, - terraformStateObjectGeneration: stateBinding.generation, - terraformStateObjectSha256: stateBinding.sha256, - requestReason: `orca-relay-fence/${attemptId}` - } - const prepared = await deps.prepareAttempt(attempt) - let durableAttempt = prepared?.attempt ?? prepared - if ( - !durableAttempt || - !Number.isSafeInteger(durableAttempt.createdAt) || - !Number.isSafeInteger(durableAttempt.expiresAt) - ) { - throw new Error('durable fence attempt has no creation or expiry time') - } - assertAttemptMatches(config, durableAttempt, false) - const uploaded = await deps.uploadPlan(planPath, durableAttempt) - const bound = await deps.bindPlan({ - ...durableAttempt, - planObjectGeneration: uploaded.generation - }) - durableAttempt = bound?.attempt ?? bound - assertAttemptMatches(config, durableAttempt) - assertAttemptCheckoutBinding(config, durableAttempt, deps) - await deps.preApplyGuard() - validateTerraformFencePlan( - terraformJson(deps, [ - `-chdir=${config.terraformDir}`, - 'show', - '-json', - planPath - ]), - expected - ) - if (sha256(planPath, deps.readFile) !== planSha256) { - throw new Error('saved fence plan digest changed') - } - assertStateObjectBinding( - await deps.stateObjectBinding(join(directory, 'pre-apply-state.tfstate')), - durableAttempt - ) - const invocationId = deps.randomUUID() - const invocationRequestReason = - `${durableAttempt.requestReason}/${invocationId}` - const startedResult = await deps.markApplyStarted(durableAttempt, { - invocationId, - requestReason: invocationRequestReason - }) - const startedAttempt = { - ...startedResult.attempt, - applyInvocations: [ - ...(durableAttempt.applyInvocations ?? []), - startedResult.invocation - ] - } - if (!Number.isSafeInteger(startedAttempt?.applyStartedAt)) { - throw new Error('durable fence attempt has no apply-start time') - } - let applyError - try { - deps.terraform( - [ - `-chdir=${config.terraformDir}`, - 'apply', - '-input=false', - `-lock-timeout=${config.lockTimeout}`, - planPath - ], - { - env: { - ...process.env, - GOOGLE_REQUEST_REASON: invocationRequestReason - } - } - ) - } catch (error) { - applyError = error - } - const progress = await deps.inspectProgress(expected, startedAttempt) - const classification = classifyTerraformFenceProgress(progress) - const observedAttempt = await persistInvocationOperations( - startedAttempt, - progress, - deps.markOperation - ) - if (classification !== 'complete') { - const reason = applyError ? 'apply response failed' : 'apply did not converge' - throw new Error(`${reason}; recover-forward required`) - } - if (!progress.gceOperation) throw new Error('completed fence has no GCE operation evidence') - if (completedStateBranch(progress, startedAttempt) !== 'complete') { - throw new Error('Terraform state did not persist the completed fence') - } - await deps.assertZeroDiff() - await deps.postApplyGuard(startedAttempt.cellIncarnation) - const completedAttempt = { - ...observedAttempt, - gceOperation: progress.gceOperation - } - await deps.attest(completedAttempt) - await deps.deletePlan(completedAttempt) - deps.emit({ - event: 'terraform_cell_fenced', - cellId: config.cell.cellId, - attemptId: startedAttempt.attemptId, - planSha256 - }) - return durableAttempt - } finally { - deps.remove(directory, { recursive: true, force: true }) - } -} - -export async function inspectTerraformFenceProgress(config, deps, attempt) { - const terraform = deps.terraform ?? defaultTerraform - const expected = exactMigExpected(config.cell) - const stateIdentity = terraformFenceStateIdentity(config, { terraform }) - const state = terraformJson({ terraform }, [ - `-chdir=${config.terraformDir}`, - 'show', - '-json' - ]) - const stateTargetSize = terraformFenceState(state, expected) - const live = deps.gcloudJson([ - 'compute', - 'instance-groups', - 'managed', - 'describe', - config.cell.migName, - '--project', - config.project, - '--zone', - config.cell.zone, - '--format=json' - ]) - const instances = deps.gcloudJson([ - 'compute', - 'instance-groups', - 'managed', - 'list-instances', - config.cell.migName, - '--project', - config.project, - '--zone', - config.cell.zone, - '--format=json' - ]) - const operations = deps.gcloudJson([ - 'compute', - 'operations', - 'list', - '--project', - config.project, - `--filter=zone:(${config.cell.zone}) AND targetLink:${config.cell.migName}`, - '--sort-by=~insertTime', - '--limit=20', - '--format=json' - ]) - const operationCandidates = operations.filter((operation) => { - const insertedAt = Date.parse(operation.insertTime) - return ( - typeof operation.name === 'string' && - operation.targetLink === - `https://www.googleapis.com/compute/v1/projects/${config.project}/zones/${config.cell.zone}/instanceGroupManagers/${config.cell.migName}` && - operation.operationType === 'compute.instanceGroupManagers.resize' && - Number.isSafeInteger(attempt?.applyStartedAt) && - Number.isFinite(insertedAt) && - insertedAt >= attempt.applyStartedAt - ) - }) - const invocations = attempt?.applyInvocations ?? [] - if (attempt?.applyStartedAt && invocations.length === 0) { - throw new Error('Terraform fence apply has no durable invocation ledger') - } - const auditEntries = invocations.length > 0 - ? deps.gcloudJson([ - 'logging', - 'read', - `protoPayload.requestMetadata.requestAttributes.reason:"${attempt.requestReason}/" AND protoPayload.resourceName="projects/${config.project}/zones/${config.cell.zone}/instanceGroupManagers/${config.cell.migName}"`, - '--project', - config.project, - '--limit=20', - '--format=json' - ]) - : [] - const invocationOperations = invocations.map((invocation) => { - const matchingAudits = (Array.isArray(auditEntries) ? auditEntries : []).filter((entry) => { - const payload = entry.protoPayload ?? {} - return ( - payload.requestMetadata?.requestAttributes?.reason === invocation.requestReason && - payload.resourceName === - `projects/${config.project}/zones/${config.cell.zone}/instanceGroupManagers/${config.cell.migName}` && - String(payload.methodName ?? '').endsWith('instanceGroupManagers.resize') && - Number(payload.request?.size ?? payload.request?.targetSize) === 0 - ) - }) - if (matchingAudits.length > 1) { - throw new Error('Terraform fence invocation has ambiguous audit operations') - } - const responseName = String( - matchingAudits[0]?.protoPayload?.response?.name ?? '' - ) - const operationName = responseName.includes('/operations/') - ? responseName.slice(responseName.lastIndexOf('/') + 1) - : responseName - const expectedName = invocation.gceOperation ?? operationName - const operation = expectedName - ? operationCandidates.find((candidate) => candidate.name === expectedName) - : undefined - if ( - (invocation.gceOperation && operationName && invocation.gceOperation !== operationName) || - (expectedName && !operation) - ) { - throw new Error('Terraform fence invocation operation mismatch') - } - return { - ...invocation, - gceOperation: operation?.name, - operationStatus: operation?.status ?? 'ABSENT', - operationError: Boolean(operation?.error), - auditBound: Boolean(matchingAudits.length === 1 && operation) - } - }) - const boundOperations = invocationOperations.filter( - (invocation) => invocation.gceOperation - ) - const finalOperation = boundOperations.at(-1) - const operationStatus = invocationOperations.some((invocation) => - ['PENDING', 'RUNNING'].includes(invocation.operationStatus) - ) - ? 'RUNNING' - : (finalOperation?.operationStatus ?? 'ABSENT') - return { - stateTargetSize, - stateLineage: stateIdentity.lineage, - stateSerial: stateIdentity.serial, - liveTargetSize: Number(live.targetSize), - instanceCount: instances.length, - operationStatus, - operationError: invocationOperations.some( - (invocation) => invocation.operationError - ), - operationAuditBound: - boundOperations.length > 0 && - boundOperations.every((invocation) => invocation.auditBound), - gceOperation: finalOperation?.gceOperation, - invocationOperations - } -} - -function responseOperationName(entry) { - const name = String(entry?.protoPayload?.response?.name ?? '') - return name.includes('/operations/') - ? name.slice(name.lastIndexOf('/') + 1) - : name -} - -export async function inspectCompletedTerraformFenceProgress( - config, - deps, - attempt, - recovery -) { - if (!Number.isSafeInteger(attempt?.applyStartedAt)) { - throw new Error('completed fence recovery has no durable apply start') - } - const terraform = deps.terraform ?? defaultTerraform - const expected = exactMigExpected(config.cell) - const stateIdentity = terraformFenceStateIdentity(config, { terraform }) - const state = terraformJson({ terraform }, [ - `-chdir=${config.terraformDir}`, - 'show', - '-json' - ]) - const stateTargetSize = terraformFenceState(state, expected) - const live = deps.gcloudJson([ - 'compute', - 'instance-groups', - 'managed', - 'describe', - config.cell.migName, - '--project', - config.project, - '--zone', - config.cell.zone, - '--format=json' - ]) - const instances = deps.gcloudJson([ - 'compute', - 'instance-groups', - 'managed', - 'list-instances', - config.cell.migName, - '--project', - config.project, - '--zone', - config.cell.zone, - '--format=json' - ]) - const targetLink = - `https://www.googleapis.com/compute/v1/projects/${config.project}/zones/${config.cell.zone}/instanceGroupManagers/${config.cell.migName}` - const operations = deps.gcloudJson([ - 'compute', - 'operations', - 'list', - '--project', - config.project, - `--filter=zone:(${config.cell.zone}) AND targetLink:${config.cell.migName}`, - '--sort-by=~insertTime', - '--limit=20', - '--format=json' - ]) - const operationCandidates = operations.filter((operation) => { - const insertedAt = Date.parse(operation.insertTime) - return ( - typeof operation.name === 'string' && - operation.targetLink === targetLink && - operation.operationType === 'compute.instanceGroupManagers.resize' && - Number.isFinite(insertedAt) && - insertedAt >= attempt.applyStartedAt - ) - }) - if ( - operationCandidates.length !== 1 || - operationCandidates[0].name !== recovery.gceOperation - ) { - throw new Error('completed fence recovery has no unique Compute operation') - } - const resourceName = - `projects/${config.project}/zones/${config.cell.zone}/instanceGroupManagers/${config.cell.migName}` - const auditEntries = deps.gcloudJson([ - 'logging', - 'read', - `protoPayload.authenticationInfo.principalEmail="${recovery.principalEmail}" AND protoPayload.resourceName="${resourceName}" AND protoPayload.methodName:"instanceGroupManagers.resize" AND timestamp>="${new Date(attempt.applyStartedAt).toISOString()}"`, - '--project', - config.project, - '--limit=20', - '--format=json' - ]) - const matchingAudits = (Array.isArray(auditEntries) ? auditEntries : []).filter( - (entry) => { - const payload = entry.protoPayload ?? {} - const timestamp = Date.parse(entry.timestamp) - return ( - payload.authenticationInfo?.principalEmail === recovery.principalEmail && - payload.resourceName === resourceName && - String(payload.methodName ?? '').endsWith('instanceGroupManagers.resize') && - Number(payload.request?.size ?? payload.request?.targetSize) === 0 && - Number.isFinite(timestamp) && - timestamp >= attempt.applyStartedAt && - responseOperationName(entry) === recovery.gceOperation - ) - } - ) - if (matchingAudits.length !== 1) { - throw new Error('completed fence recovery has no unique Audit Log operation') - } - const [operation] = operationCandidates - return { - stateTargetSize, - stateLineage: stateIdentity.lineage, - stateSerial: stateIdentity.serial, - liveTargetSize: Number(live.targetSize), - instanceCount: instances.length, - liveStable: live.status?.isStable === true, - operationStatus: operation.status, - operationError: Boolean(operation.error), - gceOperation: operation.name - } -} - -export function assertTerraformFenceSet(config, deps = {}) { - const terraform = deps.terraform ?? defaultTerraform - const result = terraform( - [ - `-chdir=${config.terraformDir}`, - 'console', - `-var-file=${config.varFile}` - ], - { - encoding: 'utf8', - input: - `contains(var.relay_gce_fenced_cells, ${JSON.stringify(config.cell.cellId)}) && ` + - `try(local.relay_gce_cell_target_sizes[${JSON.stringify(config.cell.cellId)}], -1) == 0\n` - } - ) - if (result.trim() !== 'true') { - throw new Error('requested cell is not in the committed Terraform fence set') - } -} - -export async function recoverSupersededCompletedTerraformFence( - config, - deps, - recovery -) { - assertFenceIdentity(config.cell) - const varFileSha256 = assertReviewedFenceCheckout(config, deps) - await deps.assertCommittedFenceSet() - const attempt = await deps.loadAttempt(config.cell.cellId) - if ( - !attempt || - attempt.fenceCommit === config.fenceCommit || - attempt.attemptId !== recovery.attemptId || - attempt.fenceCommit !== recovery.fenceCommit || - attempt.terraformStateSerial !== recovery.terraformStateSerial || - attempt.planObjectGeneration !== recovery.planObjectGeneration - ) { - throw new Error('completed fence recovery does not match the pinned attempt') - } - assertAttemptMatches({ ...config, fenceCommit: attempt.fenceCommit }, attempt) - if ( - varFileSha256 !== attempt.varFileSha256 || - !Number.isSafeInteger(attempt.applyStartedAt) || - attempt.abortedAt || - (attempt.gceOperation && attempt.gceOperation !== recovery.gceOperation) - ) { - throw new Error('completed fence recovery attempt is not adoptable') - } - const invocations = attempt.applyInvocations ?? [] - if ( - invocations.length !== 1 || - (invocations[0].gceOperation && - invocations[0].gceOperation !== recovery.gceOperation) || - invocations[0].startedAt < attempt.applyStartedAt - ) { - throw new Error('completed fence recovery invocation ledger is unsafe') - } - const resolved = await deps.resolvePlan(attempt) - const planExists = resolved.generation === attempt.planObjectGeneration - if (!planExists && !(attempt.completedAt && resolved.generation === null)) { - throw new Error('completed fence recovery saved-plan generation changed') - } - const directory = privatePlanDirectory({ - mkdtemp: deps.mkdtemp ?? mkdtempSync, - chmod: deps.chmod ?? chmodSync, - tmpdir: deps.tmpdir ?? tmpdir - }) - try { - const stateBinding = await deps.stateObjectBinding( - join(directory, 'completed-state.tfstate') - ) - if ( - stateBinding.generation !== recovery.terraformStateObjectGeneration || - stateBinding.sha256 !== recovery.terraformStateObjectSha256 || - stateBinding.lineage !== attempt.terraformStateLineage || - stateBinding.serial !== attempt.terraformStateSerial + 1 - ) { - throw new Error('completed fence recovery state object changed') - } - if (planExists) { - const planPath = join(directory, 'fence.tfplan') - await deps.downloadPlan(attempt, planPath) - if ((deps.stat ?? statSync)(planPath).mode & 0o077) { - throw new Error('downloaded saved fence plan permissions are not private') - } - if (sha256(planPath, deps.readFile ?? readFileSync) !== attempt.planSha256) { - throw new Error('completed fence recovery saved-plan digest changed') - } - validateTerraformFencePlan( - terraformJson( - { terraform: deps.terraform ?? defaultTerraform }, - [`-chdir=${config.terraformDir}`, 'show', '-json', planPath] - ), - exactMigExpected(config.cell) - ) - } - const progress = await deps.inspectCompletedProgress( - exactMigExpected(config.cell), - attempt, - recovery - ) - if ( - progress.stateLineage !== attempt.terraformStateLineage || - progress.stateSerial !== attempt.terraformStateSerial + 1 || - progress.stateTargetSize !== 0 || - progress.liveTargetSize !== 0 || - progress.instanceCount !== 0 || - progress.liveStable !== true || - progress.operationStatus !== 'DONE' || - progress.operationError || - progress.gceOperation !== recovery.gceOperation - ) { - throw new Error('completed fence recovery production evidence is unsafe') - } - const invocation = { - ...invocations[0], - gceOperation: recovery.gceOperation - } - const marked = attempt.gceOperation - ? { attempt, invocation } - : await deps.markOperation( - { ...attempt, gceOperation: recovery.gceOperation }, - invocation - ) - await deps.assertZeroDiff() - await deps.postApplyGuard(attempt.cellIncarnation) - await deps.attest({ - ...marked.attempt, - applyInvocations: [marked.invocation], - gceOperation: recovery.gceOperation - }) - if (planExists) await deps.deletePlan(attempt) - deps.emit?.({ - event: 'terraform_cell_fence_completed_attempt_recovered', - cellId: config.cell.cellId, - attemptId: attempt.attemptId, - gceOperation: recovery.gceOperation - }) - } finally { - ;(deps.remove ?? rmSync)(directory, { recursive: true, force: true }) - } -} - -export async function resumeTerraformFence(config, deps) { - assertFenceIdentity(config.cell) - assertReviewedFenceCheckout(config, deps) - await deps.assertCommittedFenceSet() - let attempt = await deps.loadAttempt(config.cell.cellId) - assertAttemptMatches(config, attempt, false) - if (!attempt.planObjectGeneration) { - const resolved = await deps.resolvePlan(attempt) - if (!resolved.generation) throw new Error('durable fence plan object is missing') - const bound = await deps.bindPlan({ - ...attempt, - planObjectGeneration: resolved.generation - }) - attempt = bound?.attempt ?? bound - } - assertAttemptMatches(config, attempt) - assertAttemptCheckoutBinding(config, attempt, deps) - let progress = await deps.inspectProgress(exactMigExpected(config.cell), attempt) - let classification = classifyTerraformFenceProgress(progress) - if (classification === 'complete') { - if (completedStateBranch(progress, attempt) === 'replay') { - classification = 'reconcile-state' - } else { - await deps.assertZeroDiff() - } - } - if (classification !== 'complete') { - if (classification === 'in-progress' && Number.isSafeInteger(attempt.applyStartedAt)) { - throw new Error('Terraform fence operation is still running; recover-forward required') - } - attempt = await persistInvocationOperations( - attempt, - progress, - deps.markOperation - ) - assertReplayStateIdentity(progress, attempt) - await deps.preApplyGuard() - const directory = privatePlanDirectory({ - mkdtemp: deps.mkdtemp ?? mkdtempSync, - chmod: deps.chmod ?? chmodSync, - tmpdir: deps.tmpdir ?? tmpdir - }) - const planPath = join(directory, 'fence.tfplan') - try { - assertStateObjectBinding( - await deps.stateObjectBinding( - join(directory, 'replay-pre-state.tfstate') - ), - attempt - ) - await deps.downloadPlan(attempt, planPath) - const stat = (deps.stat ?? statSync)(planPath) - if ((stat.mode & 0o077) !== 0) { - throw new Error('downloaded saved fence plan permissions are not private') - } - if (sha256(planPath, deps.readFile ?? readFileSync) !== attempt.planSha256) { - throw new Error('downloaded saved fence plan digest mismatch') - } - validateTerraformFencePlan( - terraformJson( - { terraform: deps.terraform ?? defaultTerraform }, - [`-chdir=${config.terraformDir}`, 'show', '-json', planPath] - ), - exactMigExpected(config.cell) - ) - const invocationId = (deps.randomUUID ?? randomUUID)() - const invocationRequestReason = `${attempt.requestReason}/${invocationId}` - const started = await deps.markApplyStarted(attempt, { - invocationId, - requestReason: invocationRequestReason - }) - attempt = { - ...started.attempt, - applyInvocations: [ - ...(attempt.applyInvocations ?? []), - started.invocation - ] - } - let applyError - try { - ;(deps.terraform ?? defaultTerraform)( - [ - `-chdir=${config.terraformDir}`, - 'apply', - '-input=false', - `-lock-timeout=${config.lockTimeout}`, - planPath - ], - { - env: { - ...process.env, - GOOGLE_REQUEST_REASON: invocationRequestReason - } - } - ) - } catch (error) { - applyError = error - } - progress = await deps.inspectProgress(exactMigExpected(config.cell), attempt) - classification = classifyTerraformFenceProgress(progress) - attempt = await persistInvocationOperations( - attempt, - progress, - deps.markOperation - ) - if (classification !== 'complete') { - const reason = applyError ? 'saved-plan replay response failed' : 'saved-plan replay did not converge' - throw new Error(`${reason}; recover-forward required`) - } - if (completedStateBranch(progress, attempt) !== 'complete') { - throw new Error('Terraform state did not persist the replayed fence') - } - await deps.assertZeroDiff() - } finally { - ;(deps.remove ?? rmSync)(directory, { recursive: true, force: true }) - } - } - const gceOperation = progress.gceOperation ?? attempt.gceOperation - if (!gceOperation) throw new Error('completed fence has no GCE operation evidence') - await deps.postApplyGuard(attempt.cellIncarnation) - await deps.attest({ ...attempt, gceOperation }) - await deps.deletePlan(attempt) - deps.emit?.({ - event: 'terraform_cell_fence_resumed', - cellId: config.cell.cellId, - attemptId: attempt.attemptId - }) -} - -export async function abortTerraformFenceBeforeApply(config, deps) { - assertFenceIdentity(config.cell) - assertReviewedFenceCheckout(config, deps) - await deps.assertCommittedFenceSet() - let attempt = await deps.loadAttempt(config.cell.cellId) - assertAttemptMatches(config, attempt, false) - if (!attempt.planObjectGeneration) { - const resolved = await deps.resolvePlan(attempt) - if (resolved.generation) { - const bound = await deps.bindPlan({ - ...attempt, - planObjectGeneration: resolved.generation - }) - attempt = bound?.attempt ?? bound - } - } - assertAttemptMatches(config, attempt, Boolean(attempt.planObjectGeneration)) - assertAttemptCheckoutBinding(config, attempt, deps) - const progress = await deps.inspectProgress(exactMigExpected(config.cell), attempt) - if (classifyTerraformFenceProgress(progress) !== 'not-started') { - throw new Error('cannot abort after Terraform fence apply may have started') - } - if (attempt.applyStartedAt) throw new Error('cannot abort after Terraform fence apply was marked') - await deps.abortAttempt(attempt) - if (attempt.planObjectGeneration) await deps.deletePlan(attempt) - deps.emit?.({ event: 'terraform_fence_aborted_before_apply', cellId: config.cell.cellId }) -} - -export async function abortSupersededTerraformFenceBeforeUpload(config, deps) { - assertFenceIdentity(config.cell) - const varFileSha256 = assertReviewedFenceCheckout(config, deps) - await deps.assertCommittedFenceSet() - const attempt = await deps.loadAttempt(config.cell.cellId) - if ( - !attempt || - !/^[a-f0-9]{40}$/.test(attempt.fenceCommit ?? '') || - attempt.fenceCommit === config.fenceCommit - ) { - throw new Error('prepared Terraform fence attempt is not from a superseded commit') - } - assertAttemptMatches({ ...config, fenceCommit: attempt.fenceCommit }, attempt, false) - if (attempt.varFileSha256 !== varFileSha256) { - throw new Error('superseded Terraform fence variable-file digest changed') - } - if ( - attempt.planObjectGeneration || - attempt.applyStartedAt || - attempt.completedAt || - attempt.gceOperation || - (attempt.applyInvocations?.length ?? 0) > 0 - ) { - throw new Error('cannot supersede a Terraform fence attempt after plan upload') - } - const resolved = await deps.resolvePlan(attempt) - if (resolved?.generation !== null) { - throw new Error('cannot supersede a Terraform fence attempt with a saved plan') - } - const progress = await deps.inspectProgress(exactMigExpected(config.cell), attempt) - if (classifyTerraformFenceProgress(progress) !== 'not-started') { - throw new Error('cannot supersede after Terraform fence apply may have started') - } - await deps.abortAttempt(attempt) - deps.emit?.({ - event: 'terraform_fence_superseded_before_upload', - cellId: config.cell.cellId, - previousFenceCommit: attempt.fenceCommit, - fenceCommit: config.fenceCommit - }) -} diff --git a/cloud/dev/scripts/relay-gce-terraform-fence.test.mjs b/cloud/dev/scripts/relay-gce-terraform-fence.test.mjs deleted file mode 100644 index 306e28c887b..00000000000 --- a/cloud/dev/scripts/relay-gce-terraform-fence.test.mjs +++ /dev/null @@ -1,1522 +0,0 @@ -import assert from 'node:assert/strict' -import { createHash } from 'node:crypto' -import { - chmodSync, - existsSync, - mkdtempSync, - readFileSync, - rmSync, - writeFileSync -} from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { test } from 'node:test' -import { - abortSupersededTerraformFenceBeforeUpload, - abortTerraformFenceBeforeApply, - adoptLegacyTerraformFence, - assertReviewedFenceCheckout, - assertTerraformFenceSet, - assertTerraformFenceStateFenced, - assertTerraformFenceZeroDiff, - classifyTerraformFenceProgress, - deleteTerraformFencePlan, - inspectCompletedTerraformFenceProgress, - inspectTerraformFenceProgress, - recoverSupersededCompletedTerraformFence, - runTerraformFenceApply, - resumeTerraformFence, - terraformFenceState, - terraformProcessStdio, - validateTerraformFenceCompletionPlan, - validateTerraformFencePlan -} from './relay-gce-terraform-fence.mjs' - -const cell = { - cellId: 'production-gce-c1', - migName: 'orca-relay-c1', - zone: 'us-central1-a', - instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1', - generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc', - fenced: true, - desiredTargetSize: 0 -} - -const expected = { - cellId: cell.cellId, - name: cell.migName, - zone: cell.zone, - instance_group: cell.instanceGroup, - generationIdentity: cell.generationIdentity -} -const stateLineage = 'c739dab4-e6e1-e627-02a9-504b3dda1a2c' - -test('adopts a legacy fence only after repeated no-op and live guards', async () => { - const events = [] - const calls = [] - await adoptLegacyTerraformFence( - { - environment: 'production', - terraformDir: 'infra/terraform', - varFile: 'environments/production.tfvars', - fenceCommit: 'a'.repeat(40), - cellIncarnation: '11111111-1111-4111-8111-111111111111', - cell - }, - { - environment: { ORCA_RELAY_FENCE_IMAGE_COMMIT: 'a'.repeat(40) }, - readFile: () => Buffer.from('reviewed variables'), - loadAttempt: async () => { - calls.push('attempt') - return null - }, - assertCommittedFenceSet: async () => calls.push('fence-set'), - preApplyGuard: async () => calls.push('pre'), - assertStateFenced: async () => calls.push('state-fenced'), - postApplyGuard: async () => calls.push('post'), - attest: async () => calls.push('attest'), - commitAdoption: async () => calls.push('commit'), - emit: (event) => events.push(event) - } - ) - assert.deepEqual(calls, [ - 'attempt', - 'fence-set', - 'pre', - 'state-fenced', - 'post', - 'attempt', - 'attest', - 'post', - 'commit' - ]) - assert.deepEqual(events, [ - { event: 'terraform_cell_fence_legacy_adopted', cellId: cell.cellId } - ]) -}) - -test('refuses legacy adoption when a durable attempt appears during proof', async () => { - let reads = 0 - let attested = false - await assert.rejects( - adoptLegacyTerraformFence( - { - environment: 'production', - terraformDir: 'infra/terraform', - varFile: 'environments/production.tfvars', - fenceCommit: 'a'.repeat(40), - cellIncarnation: '11111111-1111-4111-8111-111111111111', - cell - }, - { - environment: { ORCA_RELAY_FENCE_IMAGE_COMMIT: 'a'.repeat(40) }, - readFile: () => Buffer.from('reviewed variables'), - loadAttempt: async () => (++reads === 1 ? null : { attemptId: 'new' }), - assertCommittedFenceSet: async () => {}, - preApplyGuard: async () => {}, - assertStateFenced: async () => {}, - postApplyGuard: async () => {}, - attest: async () => { - attested = true - }, - commitAdoption: async () => {} - } - ), - /durable fence attempt appeared/ - ) - assert.equal(attested, false) -}) - -test('does not commit legacy adoption when the final guard fails', async () => { - let postGuards = 0 - let committed = false - await assert.rejects( - adoptLegacyTerraformFence( - { - environment: 'production', - terraformDir: 'infra/terraform', - varFile: 'environments/production.tfvars', - fenceCommit: 'a'.repeat(40), - cellIncarnation: '11111111-1111-4111-8111-111111111111', - cell - }, - { - environment: { ORCA_RELAY_FENCE_IMAGE_COMMIT: 'a'.repeat(40) }, - readFile: () => Buffer.from('reviewed variables'), - loadAttempt: async () => null, - assertCommittedFenceSet: async () => {}, - preApplyGuard: async () => {}, - assertStateFenced: async () => {}, - postApplyGuard: async () => { - postGuards++ - if (postGuards === 2) throw new Error('final guard failed') - }, - attest: async () => {}, - commitAdoption: async () => { - committed = true - } - } - ), - /final guard failed/ - ) - assert.equal(committed, false) -}) - -test('pipes reviewed Terraform console expressions to stdin', () => { - assert.deepEqual( - terraformProcessStdio({ encoding: 'utf8', input: 'contains(...)\n' }), - ['pipe', 'pipe', 'pipe'] - ) - assert.deepEqual(terraformProcessStdio({ encoding: 'utf8' }), [ - 'ignore', - 'pipe', - 'pipe' - ]) - assert.equal(terraformProcessStdio(), 'inherit') -}) - -test('checks the exact committed fence cell through Terraform console', () => { - let invocation - assertTerraformFenceSet( - { - terraformDir: 'infra/terraform', - varFile: 'environments/production.tfvars', - cell - }, - { - terraform: (args, options) => { - invocation = { args, options } - return 'true\n' - } - } - ) - assert.deepEqual(invocation.args, [ - '-chdir=infra/terraform', - 'console', - '-var-file=environments/production.tfvars' - ]) - assert.equal( - invocation.options.input, - 'contains(var.relay_gce_fenced_cells, "production-gce-c1") && ' + - 'try(local.relay_gce_cell_target_sizes["production-gce-c1"], -1) == 0\n' - ) -}) - -test('binds a gitless broker checkout to its immutable image commit', () => { - const config = { - fenceCommit: 'a'.repeat(40), - terraformDir: 'infra/terraform', - varFile: 'environments/production.tfvars' - } - const contents = Buffer.from('reviewed production variables') - const digest = assertReviewedFenceCheckout(config, { - environment: { ORCA_RELAY_FENCE_IMAGE_COMMIT: config.fenceCommit }, - readFile: () => contents, - git: () => { - throw new Error('git must not run inside the immutable broker image') - } - }) - assert.equal(digest, createHash('sha256').update(contents).digest('hex')) -}) - -test('rejects a broker image built for a different fence commit', () => { - assert.throws( - () => - assertReviewedFenceCheckout( - { - fenceCommit: 'a'.repeat(40), - terraformDir: 'infra/terraform', - varFile: 'environments/production.tfvars' - }, - { - environment: { ORCA_RELAY_FENCE_IMAGE_COMMIT: 'b'.repeat(40) }, - readFile: () => Buffer.from('reviewed production variables') - } - ), - /immutable broker image/ - ) -}) - -function plan(actions = ['update'], address = undefined) { - return { - resource_changes: [ - { - address: - address ?? - `google_compute_instance_group_manager.relay_gce_cell["${cell.cellId}"]`, - change: { - actions, - before: { - name: cell.migName, - zone: cell.zone, - instance_group: cell.instanceGroup, - target_size: 1, - version: [{ instance_template: cell.generationIdentity }] - }, - after: { - name: cell.migName, - zone: cell.zone, - instance_group: cell.instanceGroup, - target_size: 0, - version: [{ instance_template: cell.generationIdentity }] - } - } - } - ] - } -} - -function state(targetSize = 0) { - return { - values: { - root_module: { - resources: [ - { - address: - `google_compute_instance_group_manager.relay_gce_cell["${cell.cellId}"]`, - values: { - name: cell.migName, - zone: cell.zone, - instance_group: cell.instanceGroup, - target_size: targetSize, - version: [{ instance_template: cell.generationIdentity }] - } - } - ] - } - } - } -} - -test('accepts only one exact in-place MIG resize from one to zero', () => { - assert.equal(validateTerraformFencePlan(plan(), expected).change.after.target_size, 0) - for (const actions of [['create'], ['delete'], ['delete', 'create']]) { - assert.throws(() => validateTerraformFencePlan(plan(actions), expected)) - } - assert.throws(() => - validateTerraformFencePlan( - plan(['update'], 'google_compute_backend_service.relay_gce_cell["production-gce-c1"]'), - expected - ) - ) - const unrelated = plan() - unrelated.resource_changes.push({ - address: 'google_compute_url_map.relay_gce[0]', - change: { actions: ['update'], before: {}, after: {} } - }) - assert.throws(() => validateTerraformFencePlan(unrelated, expected)) -}) - -function completionPlan() { - const result = plan() - const before = result.resource_changes[0].change.before - const after = result.resource_changes[0].change.after - before.target_size = 0 - before.version[0].name = '0/2026-07-31 03:52:56.639922+00:00' - after.version[0].name = 'primary' - return result -} - -test('accepts only the empty MIG provider version-label normalization', () => { - assert.equal(validateTerraformFenceCompletionPlan({ resource_changes: [] }, expected), undefined) - assert.equal( - validateTerraformFenceCompletionPlan(completionPlan(), expected).change.after.version[0] - .name, - 'primary' - ) - - const resized = completionPlan() - resized.resource_changes[0].change.after.target_size = 1 - assert.throws(() => validateTerraformFenceCompletionPlan(resized, expected)) - - const replaced = completionPlan() - replaced.resource_changes[0].change.after.version[0].instance_template += '-other' - assert.throws(() => validateTerraformFenceCompletionPlan(replaced, expected)) - - const extraChange = completionPlan() - extraChange.resource_changes[0].change.after.update_policy = { type: 'PROACTIVE' } - assert.throws(() => validateTerraformFenceCompletionPlan(extraChange, expected)) - - const arbitraryLabel = completionPlan() - arbitraryLabel.resource_changes[0].change.before.version[0].name = 'other' - assert.throws(() => validateTerraformFenceCompletionPlan(arbitraryLabel, expected)) -}) - -test('binds Terraform state to the exact MIG generation', () => { - assert.equal(terraformFenceState(state(0), expected), 0) - const replaced = state(0) - replaced.values.root_module.resources[0].values.version[0].instance_template += '-other' - assert.throws(() => terraformFenceState(replaced, expected)) -}) - -test('requires Terraform state to record the exact completed fence', () => { - let invocation - assertTerraformFenceStateFenced(applyConfig(), { - terraform: (args) => { - invocation = args - return JSON.stringify(state(0)) - } - }) - assert.deepEqual(invocation, ['-chdir=infra/terraform', 'show', '-json']) - - assert.throws( - () => - assertTerraformFenceStateFenced(applyConfig(), { - terraform: () => JSON.stringify(state(1)) - }), - /does not record the requested cell fence/ - ) - const replaced = state(0) - replaced.values.root_module.resources[0].values.version[0].instance_template += '-other' - assert.throws(() => - assertTerraformFenceStateFenced(applyConfig(), { - terraform: () => JSON.stringify(replaced) - }) - ) -}) - -test('builds and verifies fence plans without unrelated live refreshes', () => { - let zeroDiffArgs - assertTerraformFenceZeroDiff(applyConfig(), { - terraform: (args) => { - if (args.includes('plan')) zeroDiffArgs = args - if (args.includes('show')) return JSON.stringify({ resource_changes: [] }) - } - }) - assert.equal(zeroDiffArgs.includes('-refresh=false'), true) - assert.equal( - zeroDiffArgs.includes( - '-target=google_compute_instance_group_manager.relay_gce_cell["production-gce-c1"]' - ), - true - ) -}) - -test('classifies complete, in-progress, and conclusively not-started fences', () => { - assert.equal( - classifyTerraformFenceProgress({ - stateTargetSize: 0, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'DONE', - operationAuditBound: true - }), - 'complete' - ) - assert.equal( - classifyTerraformFenceProgress({ - stateTargetSize: 1, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'RUNNING' - }), - 'in-progress' - ) - assert.equal( - classifyTerraformFenceProgress({ - stateTargetSize: 1, - liveTargetSize: 1, - instanceCount: 1, - operationStatus: 'ABSENT' - }), - 'not-started' - ) - assert.equal( - classifyTerraformFenceProgress({ - stateTargetSize: 1, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'DONE', - operationAuditBound: true - }), - 'reconcile-state' - ) - assert.throws( - () => - classifyTerraformFenceProgress({ - stateTargetSize: 0, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'DONE', - operationAuditBound: false - }), - /audit binding/ - ) -}) - -function applyHarness({ - loseApplyResponse = false, - guardError = null, - postGuardError = null, - tamperPlan = false, - progress -} = {}) { - const root = mkdtempSync(join(tmpdir(), 'relay-fence-test-')) - const calls = [] - const applyEnvironments = [] - const events = [] - let planPath - let progressReads = 0 - const terraform = (args, options = {}) => { - calls.push(args) - if (args.includes('state') && args.includes('pull')) { - return JSON.stringify({ lineage: stateLineage, serial: 7 }) - } - if (args.includes('plan')) { - planPath = args.find((arg) => arg.startsWith('-out=')).slice(5) - writeFileSync(planPath, 'private saved plan', { mode: 0o644 }) - chmodSync(planPath, 0o644) - return - } - if (args.includes('show')) return JSON.stringify(plan()) - if (args.includes('apply')) { - applyEnvironments.push(options.env) - if (loseApplyResponse) throw new Error('lost response') - } - } - const evidence = [] - return { - root, - calls, - events, - evidence, - applyEnvironments, - planPath: () => planPath, - overrides: { - terraform, - git: (args) => { - if (args.includes('rev-parse')) return `${applyConfig().fenceCommit}\n` - return '' - }, - assertCommittedFenceSet: async () => {}, - tmpdir: () => root, - randomUUID: () => '11111111-1111-4111-8111-111111111111', - inspectProgress: async () => { - if (progressReads++ === 0) { - return { - stateTargetSize: 1, - liveTargetSize: 1, - instanceCount: 1, - operationStatus: 'ABSENT', - operationError: false, - operationAuditBound: false, - stateLineage, - stateSerial: 7, - invocationOperations: [] - } - } - return progress ?? { - stateTargetSize: 0, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'DONE', - operationError: false, - operationAuditBound: true, - stateLineage, - stateSerial: 8, - gceOperation: 'operation-1', - invocationOperations: [ - { - invocationId: '11111111-1111-4111-8111-111111111111', - requestReason: - 'orca-relay-fence/11111111-1111-4111-8111-111111111111/11111111-1111-4111-8111-111111111111', - startedAt: 101, - gceOperation: 'operation-1', - operationStatus: 'DONE', - operationError: false, - auditBound: true - } - ] - } - }, - uploadPlan: async () => ({ generation: '123456789' }), - stateObjectBinding: async () => ({ - generation: '987654321', - sha256: createHash('sha256').update('pre-state object').digest('hex'), - lineage: stateLineage, - serial: 7 - }), - bindPlan: async (value) => { - evidence.push(['bind', value]) - return { attempt: value } - }, - deletePlan: async (value) => evidence.push(['delete', value]), - assertZeroDiff: async () => {}, - prepareAttempt: async (value) => { - evidence.push(['prepare', value]) - return { attempt: { ...value, createdAt: 100, expiresAt: 3_600_100 } } - }, - markApplyStarted: async (value, invocation) => { - const started = { ...value, applyStartedAt: 101 } - const durableInvocation = { ...invocation, startedAt: 101 } - evidence.push(['started', started]) - return { attempt: started, invocation: durableInvocation } - }, - markOperation: async (value, invocation) => { - const durableInvocation = { - ...invocation, - gceOperation: value.gceOperation - } - evidence.push(['operation', value]) - return { attempt: value, invocation: durableInvocation } - }, - attest: async (value) => evidence.push(['attest', value]), - preApplyGuard: async () => { - if (guardError) throw guardError - if (tamperPlan) writeFileSync(planPath, 'tampered plan', { mode: 0o600 }) - }, - postApplyGuard: async () => { - if (postGuardError) throw postGuardError - }, - emit: (event) => events.push(event) - }, - cleanup: () => rmSync(root, { recursive: true, force: true }) - } -} - -function applyConfig() { - return { - project: 'project', - environment: 'production', - terraformDir: 'infra/terraform', - varFile: 'environments/production.tfvars', - lockTimeout: '5m', - fenceCommit: 'a'.repeat(40), - cellIncarnation: '22222222-2222-4222-8222-222222222222', - cell - } -} - -function durableAttempt(config = applyConfig(), overrides = {}) { - const attemptId = '11111111-1111-4111-8111-111111111111' - const varFile = join(config.terraformDir, config.varFile) - const result = { - attemptId, - environment: config.environment, - cellId: cell.cellId, - cellIncarnation: config.cellIncarnation, - migName: cell.migName, - instanceGroup: cell.instanceGroup, - generationIdentity: cell.generationIdentity, - fenceCommit: config.fenceCommit, - planSha256: createHash('sha256').update('private saved plan').digest('hex'), - planObjectName: `terraform/state/relay-fence-plans/${config.environment}/${attemptId}.tfplan`, - planObjectGeneration: '123456789', - varFileSha256: createHash('sha256').update(readFileSync(varFile)).digest('hex'), - terraformStateLineage: stateLineage, - terraformStateSerial: 7, - terraformStateObjectGeneration: '987654321', - terraformStateObjectSha256: createHash('sha256') - .update('pre-state object') - .digest('hex'), - requestReason: `orca-relay-fence/${attemptId}`, - createdAt: 100, - expiresAt: 3_600_100, - ...overrides - } - if (result.applyStartedAt && result.applyInvocations === undefined) { - const invocationId = '66666666-6666-4666-8666-666666666666' - result.applyInvocations = [ - { - invocationId, - requestReason: `${result.requestReason}/${invocationId}`, - startedAt: result.applyStartedAt, - gceOperation: result.gceOperation - } - ] - } - return result -} - -test('applies and attests the exact private saved plan', async () => { - const harness = applyHarness() - try { - await runTerraformFenceApply(applyConfig(), harness.overrides) - assert.deepEqual( - harness.evidence.map(([event]) => event), - ['prepare', 'bind', 'started', 'operation', 'attest', 'delete'] - ) - assert.equal(harness.calls.filter((args) => args.includes('apply')).length, 1) - const planArgs = harness.calls.find((args) => args.includes('plan')) - assert.equal(planArgs.includes('-refresh=false'), true) - assert.equal( - harness.applyEnvironments[0].GOOGLE_REQUEST_REASON, - 'orca-relay-fence/11111111-1111-4111-8111-111111111111/11111111-1111-4111-8111-111111111111' - ) - assert.equal(harness.events[0].event, 'terraform_cell_fenced') - assert.equal(existsSync(harness.planPath()), false) - } finally { - harness.cleanup() - } -}) - -test('rejects a malformed Terraform lineage before plan upload', async () => { - const harness = applyHarness() - const prepareAttempt = harness.overrides.prepareAttempt - harness.overrides.prepareAttempt = async (value) => { - const prepared = await prepareAttempt(value) - return { - attempt: { - ...prepared.attempt, - terraformStateLineage: 'not-a-terraform-lineage' - } - } - } - try { - await assert.rejects( - runTerraformFenceApply(applyConfig(), harness.overrides), - /valid Terraform state identity/ - ) - assert.deepEqual( - harness.evidence.map(([event]) => event), - ['prepare'] - ) - assert.equal(harness.calls.filter((args) => args.includes('apply')).length, 0) - } finally { - harness.cleanup() - } -}) - -test('recovers a successful fence after losing the apply response', async () => { - const harness = applyHarness({ loseApplyResponse: true }) - try { - await runTerraformFenceApply(applyConfig(), harness.overrides) - assert.equal(harness.evidence.some(([event]) => event === 'attest'), true) - } finally { - harness.cleanup() - } -}) - -test('does not start apply after a final pre-apply guard failure', async () => { - const harness = applyHarness({ guardError: new Error('guard failed') }) - try { - await assert.rejects( - runTerraformFenceApply(applyConfig(), harness.overrides), - /guard failed/ - ) - assert.equal(harness.calls.some((args) => args.includes('apply')), false) - assert.deepEqual(harness.evidence.map(([event]) => event), ['prepare', 'bind']) - } finally { - harness.cleanup() - } -}) - -test('rejects a saved plan whose digest changes before apply', async () => { - const harness = applyHarness({ tamperPlan: true }) - try { - await assert.rejects( - runTerraformFenceApply(applyConfig(), harness.overrides), - /digest changed/ - ) - assert.equal(harness.calls.some((args) => args.includes('apply')), false) - } finally { - harness.cleanup() - } -}) - -test('requires recover-forward when an apply remains in progress', async () => { - const harness = applyHarness({ - loseApplyResponse: true, - progress: { - stateTargetSize: 1, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'RUNNING', - operationError: false, - stateLineage, - stateSerial: 7, - gceOperation: 'operation-1' - } - }) - try { - await assert.rejects( - runTerraformFenceApply(applyConfig(), harness.overrides), - /recover-forward required/ - ) - assert.notEqual(harness.evidence.at(-1)[0], 'attest') - } finally { - harness.cleanup() - } -}) - -test('does not attest before retained topology and heartbeat guards pass', async () => { - const harness = applyHarness({ postGuardError: new Error('topology mismatch') }) - try { - await assert.rejects( - runTerraformFenceApply(applyConfig(), harness.overrides), - /topology mismatch/ - ) - assert.notEqual(harness.evidence.at(-1)[0], 'attest') - } finally { - harness.cleanup() - } -}) - -test('aborts only when state and live GCE prove apply never began', async () => { - let aborted = false - let deleted = false - const config = applyConfig() - const attempt = durableAttempt(config) - const common = { - git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), - loadAttempt: async () => attempt, - deletePlan: async () => { - deleted = true - } - } - await abortTerraformFenceBeforeApply(config, { - ...common, - assertCommittedFenceSet: async () => {}, - inspectProgress: async () => ({ - stateTargetSize: 1, - liveTargetSize: 1, - instanceCount: 1, - operationStatus: 'ABSENT', - stateLineage, - stateSerial: 7 - }), - abortAttempt: async () => { - aborted = true - } - }) - assert.equal(aborted, true) - assert.equal(deleted, true) - await assert.rejects( - abortTerraformFenceBeforeApply(config, { - ...common, - assertCommittedFenceSet: async () => {}, - inspectProgress: async () => ({ - stateTargetSize: 1, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'RUNNING', - stateLineage, - stateSerial: 7 - }), - abortAttempt: async () => {} - }), - /cannot abort/ - ) -}) - -test('supersedes only an older unuploaded fence attempt proven not started', async () => { - const config = applyConfig() - const attempt = durableAttempt(config, { - fenceCommit: 'b'.repeat(40), - planObjectGeneration: undefined - }) - const events = [] - let aborted = false - const common = { - git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), - assertCommittedFenceSet: async () => {}, - loadAttempt: async () => attempt, - resolvePlan: async () => ({ generation: null }), - inspectProgress: async () => ({ - stateTargetSize: 1, - liveTargetSize: 1, - instanceCount: 1, - operationStatus: 'ABSENT', - stateLineage, - stateSerial: 7 - }), - abortAttempt: async () => { - aborted = true - }, - emit: (event) => events.push(event) - } - await abortSupersededTerraformFenceBeforeUpload(config, common) - assert.equal(aborted, true) - assert.deepEqual(events, [ - { - event: 'terraform_fence_superseded_before_upload', - cellId: cell.cellId, - previousFenceCommit: 'b'.repeat(40), - fenceCommit: config.fenceCommit - } - ]) - - await assert.rejects( - abortSupersededTerraformFenceBeforeUpload(config, { - ...common, - loadAttempt: async () => ({ ...attempt, planObjectGeneration: '123456789' }) - }), - /after plan upload/ - ) - await assert.rejects( - abortSupersededTerraformFenceBeforeUpload(config, { - ...common, - resolvePlan: async () => ({ generation: '123456789' }) - }), - /with a saved plan/ - ) -}) - -test('resumes and attests when state and live GCE are already zero', async () => { - let attested - const config = applyConfig() - const attempt = durableAttempt(config, { - applyStartedAt: 101, - gceOperation: 'operation-1', - completedAt: 120 - }) - let deleted = false - await resumeTerraformFence(config, { - git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), - assertCommittedFenceSet: async () => {}, - loadAttempt: async () => attempt, - inspectProgress: async () => ({ - stateTargetSize: 0, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'DONE', - operationError: false, - operationAuditBound: true, - stateLineage, - stateSerial: 8, - gceOperation: 'operation-1' - }), - preApplyGuard: async () => {}, - postApplyGuard: async () => {}, - assertZeroDiff: async () => {}, - deletePlan: async () => { - deleted = true - }, - attest: async (value) => { - attested = value - } - }) - assert.equal(attested.attemptId, attempt.attemptId) - assert.equal(deleted, true) -}) - -function replayHarness({ - initialProgress, - finalProgress, - applyError = null, - zeroDiffError = null, - downloadedPlan = 'private saved plan', - attemptOverrides = {} -} = {}) { - const config = applyConfig() - const root = mkdtempSync(join(tmpdir(), 'relay-fence-replay-test-')) - const attempt = durableAttempt(config, { - applyStartedAt: 101, - ...attemptOverrides - }) - const progress = [ - initialProgress ?? { - stateTargetSize: 1, - liveTargetSize: 1, - instanceCount: 1, - operationStatus: 'ABSENT', - operationError: false, - stateLineage, - stateSerial: 7 - }, - finalProgress ?? { - stateTargetSize: 0, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'DONE', - operationError: false, - operationAuditBound: true, - stateLineage, - stateSerial: 8, - gceOperation: 'operation-1' - } - ] - let progressIndex = 0 - let applies = 0 - let downloads = 0 - let deleted = false - let attested = false - let zeroDiffChecks = 0 - return { - config, - attempt, - applies: () => applies, - downloads: () => downloads, - deleted: () => deleted, - attested: () => attested, - zeroDiffChecks: () => zeroDiffChecks, - deps: { - git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), - tmpdir: () => root, - assertCommittedFenceSet: async () => {}, - loadAttempt: async () => attempt, - resolvePlan: async () => ({ generation: '123456789' }), - bindPlan: async (value) => ({ attempt: value }), - inspectProgress: async (_expected, currentAttempt) => { - const value = progress[Math.min(progressIndex++, progress.length - 1)] - if (!value.gceOperation || value.invocationOperations) return value - const invocation = currentAttempt.applyInvocations?.at(-1) - return { - ...value, - invocationOperations: invocation - ? [ - { - ...invocation, - gceOperation: value.gceOperation, - operationStatus: value.operationStatus, - operationError: value.operationError, - auditBound: value.operationAuditBound - } - ] - : [] - } - }, - preApplyGuard: async () => {}, - postApplyGuard: async () => {}, - stateObjectBinding: async () => ({ - generation: attempt.terraformStateObjectGeneration, - sha256: attempt.terraformStateObjectSha256, - lineage: stateLineage, - serial: 7 - }), - assertZeroDiff: async () => { - zeroDiffChecks++ - if (zeroDiffError) throw zeroDiffError - }, - downloadPlan: async (value, path) => { - assert.equal( - value.planObjectGeneration, - attempt.planObjectGeneration ?? '123456789' - ) - downloads++ - writeFileSync(path, downloadedPlan, { mode: 0o600 }) - }, - terraform: (args) => { - if (args.includes('show')) return JSON.stringify(plan()) - if (args.includes('apply')) { - applies++ - if (applyError) throw applyError - } - }, - markOperation: async (value, invocation) => ({ - attempt: value, - invocation: { ...invocation, gceOperation: value.gceOperation } - }), - markApplyStarted: async (value, invocation) => ({ - attempt: { ...value, applyStartedAt: value.applyStartedAt ?? 101 }, - invocation: { ...invocation, startedAt: 101 } - }), - attest: async () => { - attested = true - }, - deletePlan: async () => { - deleted = true - } - }, - cleanup: () => rmSync(root, { recursive: true, force: true }) - } -} - -test('replays the exact durable plan after crashing immediately after apply-start', async () => { - const harness = replayHarness() - try { - await resumeTerraformFence(harness.config, harness.deps) - assert.equal(harness.downloads(), 1) - assert.equal(harness.applies(), 1) - assert.equal(harness.attested(), true) - assert.equal(harness.deleted(), true) - } finally { - harness.cleanup() - } -}) - -test('recovers an uploaded plan whose generation was not bound before runner loss', async () => { - const harness = replayHarness({ - attemptOverrides: { - planObjectGeneration: undefined, - applyStartedAt: undefined - } - }) - try { - await resumeTerraformFence(harness.config, harness.deps) - assert.equal(harness.downloads(), 1) - assert.equal(harness.applies(), 1) - assert.equal(harness.attested(), true) - } finally { - harness.cleanup() - } -}) - -test('replays the exact durable plan to reconcile live zero with stale state', async () => { - const harness = replayHarness({ - initialProgress: { - stateTargetSize: 1, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'DONE', - operationError: false, - operationAuditBound: true, - stateLineage, - stateSerial: 7, - gceOperation: 'operation-1' - } - }) - try { - await resumeTerraformFence(harness.config, harness.deps) - assert.equal(harness.applies(), 1) - assert.equal(harness.attested(), true) - } finally { - harness.cleanup() - } -}) - -test('does not replay a stale saved plan after the first apply already persisted state', async () => { - const harness = replayHarness({ - initialProgress: { - stateTargetSize: 0, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'DONE', - operationError: false, - operationAuditBound: true, - stateLineage, - stateSerial: 8, - gceOperation: 'operation-1' - } - }) - try { - await resumeTerraformFence(harness.config, harness.deps) - assert.equal(harness.downloads(), 0) - assert.equal(harness.applies(), 0) - assert.equal(harness.deleted(), true) - assert.equal(harness.zeroDiffChecks(), 1) - } finally { - harness.cleanup() - } -}) - -test('replays when the recorded Terraform serial is unchanged even if refresh sees zero', async () => { - const harness = replayHarness({ - initialProgress: { - stateTargetSize: 0, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'DONE', - operationError: false, - operationAuditBound: true, - stateLineage, - stateSerial: 7, - gceOperation: 'operation-1' - } - }) - try { - await resumeTerraformFence(harness.config, harness.deps) - assert.equal(harness.applies(), 1) - assert.equal(harness.zeroDiffChecks(), 1) - } finally { - harness.cleanup() - } -}) - -test('freezes a serial-plus-one completion unless the reviewed targeted plan is zero diff', async () => { - const harness = replayHarness({ - initialProgress: { - stateTargetSize: 0, - liveTargetSize: 0, - instanceCount: 0, - operationStatus: 'DONE', - operationError: false, - operationAuditBound: true, - stateLineage, - stateSerial: 8, - gceOperation: 'operation-1' - }, - zeroDiffError: new Error('not zero diff') - }) - try { - await assert.rejects( - resumeTerraformFence(harness.config, harness.deps), - /not zero diff/ - ) - assert.equal(harness.attested(), false) - assert.equal(harness.deleted(), false) - } finally { - harness.cleanup() - } -}) - -test('rejects saved-plan object generation and hash mismatches', async () => { - const generation = replayHarness({ - attemptOverrides: { planObjectGeneration: '0' } - }) - try { - await assert.rejects( - resumeTerraformFence(generation.config, generation.deps), - /saved-plan generation/ - ) - } finally { - generation.cleanup() - } - const digest = replayHarness({ downloadedPlan: 'tampered plan' }) - try { - await assert.rejects( - resumeTerraformFence(digest.config, digest.deps), - /saved fence plan digest mismatch/ - ) - assert.equal(digest.applies(), 0) - assert.equal(digest.deleted(), false) - } finally { - digest.cleanup() - } -}) - -test('rejects Terraform state lineage or serial drift before replay', async () => { - const harness = replayHarness({ - initialProgress: { - stateTargetSize: 1, - liveTargetSize: 1, - instanceCount: 1, - operationStatus: 'ABSENT', - operationError: false, - stateLineage, - stateSerial: 8 - } - }) - try { - await assert.rejects( - resumeTerraformFence(harness.config, harness.deps), - /lineage or serial changed/ - ) - assert.equal(harness.downloads(), 0) - } finally { - harness.cleanup() - } -}) - -test('keeps the durable plan when replay cannot acquire the Terraform lock', async () => { - const notStarted = { - stateTargetSize: 1, - liveTargetSize: 1, - instanceCount: 1, - operationStatus: 'ABSENT', - operationError: false, - stateLineage, - stateSerial: 7 - } - const harness = replayHarness({ - initialProgress: notStarted, - finalProgress: notStarted, - applyError: new Error('state lock unavailable') - }) - try { - await assert.rejects( - resumeTerraformFence(harness.config, harness.deps), - /recover-forward required/ - ) - assert.equal(harness.deleted(), false) - assert.equal(harness.attested(), false) - } finally { - harness.cleanup() - } -}) - -test('deletes the exact object generation permanently and accepts confirmed absence', async () => { - const config = applyConfig() - const attempt = durableAttempt(config) - let args - await deleteTerraformFencePlan( - config, - { - commandResult: (value) => { - args = value - return { status: 0, stderr: '' } - } - }, - attempt - ) - assert.equal( - args[2], - `gs://project-terraform-state/${attempt.planObjectName}#${attempt.planObjectGeneration}` - ) - await assert.doesNotReject( - deleteTerraformFencePlan( - config, - { commandResult: () => ({ status: 1, stderr: '404 not found' }) }, - attempt - ) - ) - await assert.rejects( - deleteTerraformFencePlan( - config, - { commandResult: () => ({ status: 1, stderr: 'permission denied' }) }, - attempt - ), - /could not be deleted/ - ) -}) - -test('binds a DONE resize operation to the exact post-start audit request reason', async () => { - const config = applyConfig() - const attempt = durableAttempt(config, { applyStartedAt: 101 }) - const targetLink = - `https://www.googleapis.com/compute/v1/projects/${config.project}/zones/${cell.zone}/instanceGroupManagers/${cell.migName}` - const terraform = (args) => - args.includes('state') - ? JSON.stringify({ lineage: stateLineage, serial: 8 }) - : JSON.stringify(state(0)) - const gcloudJson = (args) => { - if (args.includes('list-instances')) return [] - if (args.includes('managed')) return { targetSize: 0 } - if (args[0] === 'compute') { - return [ - { - name: 'operation-1', - insertTime: new Date(102).toISOString(), - targetLink, - operationType: 'compute.instanceGroupManagers.resize', - status: 'DONE' - } - ] - } - return [ - { - protoPayload: { - requestMetadata: { - requestAttributes: { - reason: attempt.applyInvocations[0].requestReason - } - }, - resourceName: - `projects/${config.project}/zones/${cell.zone}/instanceGroupManagers/${cell.migName}`, - methodName: 'v1.compute.instanceGroupManagers.resize', - request: { size: 0 }, - response: { name: 'operation-1' } - } - } - ] - } - const progress = await inspectTerraformFenceProgress( - config, - { terraform, gcloudJson }, - attempt - ) - assert.equal(progress.operationAuditBound, true) - assert.equal(classifyTerraformFenceProgress(progress), 'complete') - const unbound = await inspectTerraformFenceProgress( - config, - { - terraform, - gcloudJson: (args) => (args[0] === 'logging' ? [] : gcloudJson(args)) - }, - attempt - ) - assert.throws( - () => classifyTerraformFenceProgress(unbound), - /ambiguous or unsafe/ - ) -}) - -test('inspects an older completed fence through exact principal and operation evidence', async () => { - const config = applyConfig() - const attempt = durableAttempt(config, { applyStartedAt: 101 }) - const gceOperation = 'operation-1' - const principalEmail = 'fence-broker@example.gserviceaccount.com' - const targetLink = - `https://www.googleapis.com/compute/v1/projects/${config.project}/zones/${cell.zone}/instanceGroupManagers/${cell.migName}` - const resourceName = - `projects/${config.project}/zones/${cell.zone}/instanceGroupManagers/${cell.migName}` - const terraform = (args) => - args.includes('state') - ? JSON.stringify({ lineage: stateLineage, serial: 8 }) - : JSON.stringify(state(0)) - const progress = await inspectCompletedTerraformFenceProgress( - config, - { - terraform, - gcloudJson: (args) => { - if (args.includes('list-instances')) return [] - if (args.includes('managed')) { - return { targetSize: 0, status: { isStable: true } } - } - if (args[0] === 'compute') { - return [ - { - name: gceOperation, - insertTime: new Date(102).toISOString(), - targetLink, - operationType: 'compute.instanceGroupManagers.resize', - status: 'DONE' - } - ] - } - return [ - { - timestamp: new Date(103).toISOString(), - protoPayload: { - authenticationInfo: { principalEmail }, - resourceName, - methodName: 'v1.compute.instanceGroupManagers.resize', - request: { size: '0' }, - response: { name: gceOperation } - } - } - ] - } - }, - attempt, - { gceOperation, principalEmail } - ) - assert.deepEqual(progress, { - stateTargetSize: 0, - stateLineage, - stateSerial: 8, - liveTargetSize: 0, - instanceCount: 0, - liveStable: true, - operationStatus: 'DONE', - operationError: false, - gceOperation - }) -}) - -test('adopts only the pinned completed older attempt without replaying Terraform', async () => { - const config = applyConfig() - const root = mkdtempSync(join(tmpdir(), 'relay-fence-completed-recovery-test-')) - const attempt = durableAttempt(config, { - fenceCommit: 'b'.repeat(40), - applyStartedAt: 101 - }) - const recovery = { - attemptId: attempt.attemptId, - fenceCommit: attempt.fenceCommit, - gceOperation: 'operation-1', - terraformStateSerial: 7, - planObjectGeneration: attempt.planObjectGeneration, - terraformStateObjectGeneration: '222222222', - terraformStateObjectSha256: 'c'.repeat(64), - principalEmail: 'fence-broker@example.gserviceaccount.com' - } - const events = [] - let applies = 0 - try { - await recoverSupersededCompletedTerraformFence( - config, - { - git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), - tmpdir: () => root, - assertCommittedFenceSet: async () => {}, - loadAttempt: async () => attempt, - resolvePlan: async () => ({ generation: attempt.planObjectGeneration }), - stateObjectBinding: async () => ({ - generation: recovery.terraformStateObjectGeneration, - sha256: recovery.terraformStateObjectSha256, - lineage: stateLineage, - serial: 8 - }), - downloadPlan: async (_value, path) => - writeFileSync(path, 'private saved plan', { mode: 0o600 }), - terraform: (args) => { - if (args.includes('apply')) applies++ - if (args.includes('show')) return JSON.stringify(plan()) - }, - inspectCompletedProgress: async () => ({ - stateTargetSize: 0, - stateLineage, - stateSerial: 8, - liveTargetSize: 0, - instanceCount: 0, - liveStable: true, - operationStatus: 'DONE', - operationError: false, - gceOperation: recovery.gceOperation - }), - markOperation: async (value, invocation) => { - events.push('operation') - return { attempt: value, invocation } - }, - assertZeroDiff: async () => events.push('zero-diff'), - postApplyGuard: async () => events.push('post-apply'), - attest: async () => events.push('attest'), - deletePlan: async () => events.push('delete'), - emit: () => events.push('emit') - }, - recovery - ) - assert.equal(applies, 0) - assert.deepEqual(events, [ - 'operation', - 'zero-diff', - 'post-apply', - 'attest', - 'delete', - 'emit' - ]) - } finally { - rmSync(root, { recursive: true, force: true }) - } -}) - -test('continues after an adopted fence was attested and its plan was deleted', async () => { - const config = applyConfig() - const root = mkdtempSync(join(tmpdir(), 'relay-fence-completed-retry-test-')) - const gceOperation = 'operation-1' - const attempt = durableAttempt(config, { - fenceCommit: 'b'.repeat(40), - applyStartedAt: 101, - completedAt: 200, - gceOperation - }) - const recovery = { - attemptId: attempt.attemptId, - fenceCommit: attempt.fenceCommit, - gceOperation, - terraformStateSerial: 7, - planObjectGeneration: attempt.planObjectGeneration, - terraformStateObjectGeneration: '222222222', - terraformStateObjectSha256: 'c'.repeat(64), - principalEmail: 'fence-broker@example.gserviceaccount.com' - } - let attested = false - try { - await recoverSupersededCompletedTerraformFence( - config, - { - git: (args) => (args.includes('rev-parse') ? `${config.fenceCommit}\n` : ''), - tmpdir: () => root, - assertCommittedFenceSet: async () => {}, - loadAttempt: async () => attempt, - resolvePlan: async () => ({ generation: null }), - stateObjectBinding: async () => ({ - generation: recovery.terraformStateObjectGeneration, - sha256: recovery.terraformStateObjectSha256, - lineage: stateLineage, - serial: 8 - }), - inspectCompletedProgress: async () => ({ - stateTargetSize: 0, - stateLineage, - stateSerial: 8, - liveTargetSize: 0, - instanceCount: 0, - liveStable: true, - operationStatus: 'DONE', - operationError: false, - gceOperation - }), - markOperation: async () => { - throw new Error('must not rebind') - }, - assertZeroDiff: async () => {}, - postApplyGuard: async () => {}, - attest: async () => { - attested = true - }, - downloadPlan: async () => { - throw new Error('must not download a deleted plan') - }, - deletePlan: async () => { - throw new Error('must not delete an absent plan') - } - }, - recovery - ) - assert.equal(attested, true) - } finally { - rmSync(root, { recursive: true, force: true }) - } -}) diff --git a/cloud/dev/scripts/relay-live-cell-image-overlay.mjs b/cloud/dev/scripts/relay-live-cell-image-overlay.mjs deleted file mode 100644 index b4444e9cfe9..00000000000 --- a/cloud/dev/scripts/relay-live-cell-image-overlay.mjs +++ /dev/null @@ -1,84 +0,0 @@ -import { readFileSync, writeFileSync } from 'node:fs' -import { fileURLToPath } from 'node:url' - -// Committed images lag what same-cap rolls serve, and a URL map target pulls every cell's -// template into the plan, so a non-target cell must be planned at the image it serves. -const IMAGE_PATTERN = /^[a-z0-9.-]+\/[a-z0-9-]+\/[a-z0-9-]+\/relay@sha256:[a-f0-9]{64}$/ - -function liveRelayImage(script, committedImage, cellId) { - const repository = committedImage.split('@')[0] - const pulled = [...script.matchAll(/^docker pull '([^']+)'$/gm)] - .map((match) => match[1]) - .filter((image) => image.split('@')[0] === repository) - const digest = /^\s*printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '(sha256:[a-f0-9]{64})'$/m.exec(script)?.[1] - if (pulled.length !== 1 || !IMAGE_PATTERN.test(pulled[0]) || pulled[0].split('@')[1] !== digest) { - throw new Error(`${cellId} live template has no single pinned Relay image`) - } - return pulled[0] -} - -export function overlayRelayLiveCellImages({ committedCells, liveTemplates, targetCellIds }) { - if (!committedCells || Array.isArray(committedCells) || typeof committedCells !== 'object') { - throw new Error('committed Relay cells must be an object') - } - if (!Array.isArray(liveTemplates)) throw new Error('live templates must be an array') - const targets = new Set(targetCellIds) - for (const cellId of targets) { - if (!committedCells[cellId]) throw new Error(`${cellId} is not a committed Relay cell`) - } - const scripts = new Map() - for (const template of liveTemplates) { - if (typeof template?.index !== 'string' || typeof template.metadata_startup_script !== 'string') { - throw new Error('live template entry is malformed') - } - if (scripts.has(template.index)) throw new Error(`${template.index} has more than one live template`) - scripts.set(template.index, template.metadata_startup_script) - } - const cells = {} - for (const [cellId, cell] of Object.entries(committedCells)) { - if (targets.has(cellId)) { - cells[cellId] = cell - continue - } - const script = scripts.get(cellId) - // A declared cell with no live template would be created here, outside the reviewed wave. - if (script === undefined) throw new Error(`${cellId} is not a target and has no live template`) - cells[cellId] = { ...cell, image: liveRelayImage(script, cell.image, cellId) } - } - return { relay_gce_cells: cells } -} - -function argumentsFrom(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of ['cells-json', 'live-templates-json', 'cell-ids', 'output']) { - if (!values[key]) throw new Error(`missing --${key}`) - } - return values -} - -function readJsonFile(path, label) { - const text = readFileSync(path, 'utf8') - if (!text.trim()) throw new Error(`${label} is empty`) - return JSON.parse(text) -} - -if (process.argv[1] === fileURLToPath(import.meta.url)) { - const values = argumentsFrom(process.argv.slice(2)) - const committedCells = readJsonFile(values['cells-json'], 'committed Relay cells') - const overlay = overlayRelayLiveCellImages({ - committedCells, - liveTemplates: readJsonFile(values['live-templates-json'], 'live Relay templates'), - targetCellIds: values['cell-ids'].split(',').map((value) => value.trim()).filter(Boolean) - }) - writeFileSync(values.output, `${JSON.stringify(overlay)}\n`) - const drifted = Object.keys(committedCells).filter( - (cellId) => overlay.relay_gce_cells[cellId].image !== committedCells[cellId].image - ) - console.log(JSON.stringify({ cells: Object.keys(committedCells).length, drifted })) -} diff --git a/cloud/dev/scripts/relay-live-cell-image-overlay.test.mjs b/cloud/dev/scripts/relay-live-cell-image-overlay.test.mjs deleted file mode 100644 index 267cc974639..00000000000 --- a/cloud/dev/scripts/relay-live-cell-image-overlay.test.mjs +++ /dev/null @@ -1,119 +0,0 @@ -import assert from 'node:assert/strict' -import { spawnSync } from 'node:child_process' -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { test } from 'node:test' -import { fileURLToPath } from 'node:url' -import { overlayRelayLiveCellImages } from './relay-live-cell-image-overlay.mjs' - -const repository = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay' -const committed = `${repository}@sha256:${'a'.repeat(64)}` -const served = `${repository}@sha256:${'b'.repeat(64)}` -const template = readFileSync( - new URL('../../infra/terraform/relay-gce-startup.sh.tftpl', import.meta.url), - 'utf8' -) - -// Renders only the two lines the overlay reads, from the real template, so a template edit fails here. -function startupScript(image) { - return template - .replaceAll('${trimprefix(regex("@sha256:[a-f0-9]{64}$", relay_image), "@")}', image.split('@')[1]) - .replaceAll('${relay_image}', image) - .replaceAll('${cloud_sql_proxy_image}', `gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:${'c'.repeat(64)}`) -} - -const cell = (image) => ({ hostname: 'x', region: 'asia-east2', zone: 'asia-east2-a', image, connection_hard_cap: 3000 }) -const committedCells = { - 'production-gce-c1': cell(committed), - 'production-gce-c27': cell(committed), - 'production-gce-c30': cell(committed) -} -const live = [ - { index: 'production-gce-c1', metadata_startup_script: startupScript(committed) }, - { index: 'production-gce-c27', metadata_startup_script: startupScript(served) } -] - -test('plans every non-target cell at its served image while the target has no template yet', () => { - const overlay = overlayRelayLiveCellImages({ - committedCells, liveTemplates: live, targetCellIds: ['production-gce-c30'] - }) - assert.deepEqual(overlay, { - relay_gce_cells: { - 'production-gce-c1': cell(committed), - 'production-gce-c27': cell(served), - 'production-gce-c30': cell(committed) - } - }) -}) - -test('leaves a target cell at its committed image even once it has a live template', () => { - const overlay = overlayRelayLiveCellImages({ - committedCells, - liveTemplates: [...live, { index: 'production-gce-c30', metadata_startup_script: startupScript(served) }], - targetCellIds: ['production-gce-c30'] - }) - assert.equal(overlay.relay_gce_cells['production-gce-c30'].image, committed) -}) - -test('refuses a non-target cell that has no live template', () => { - assert.throws(() => overlayRelayLiveCellImages({ - committedCells, liveTemplates: live.slice(1), targetCellIds: ['production-gce-c30'] - }), /production-gce-c1 is not a target and has no live template/) -}) - -test('refuses a live template without one pinned Relay image', () => { - const digestMismatch = startupScript(served) - .replace(`%s\\n' 'sha256:${'b'.repeat(64)}'`, `%s\\n' 'sha256:${'d'.repeat(64)}'`) - for (const script of [ - digestMismatch, - startupScript(served).replace(`docker pull '${served}'`, ''), - `${startupScript(served)}\ndocker pull '${committed}'`, - startupScript(`${repository}:latest`) - ]) { - assert.throws(() => overlayRelayLiveCellImages({ - committedCells, - liveTemplates: [live[0], { index: 'production-gce-c27', metadata_startup_script: script }], - targetCellIds: ['production-gce-c30'] - }), /production-gce-c27 live template has no single pinned Relay image/) - } -}) - -test('refuses duplicate live templates and an undeclared target', () => { - assert.throws(() => overlayRelayLiveCellImages({ - committedCells, liveTemplates: [...live, live[1]], targetCellIds: ['production-gce-c30'] - }), /more than one live template/) - assert.throws(() => overlayRelayLiveCellImages({ - committedCells, liveTemplates: live, targetCellIds: ['production-gce-c31'] - }), /production-gce-c31 is not a committed Relay cell/) -}) - -test('builds the overlay from plan variables when the target cell has no template in state yet', () => { - const dir = mkdtempSync(join(tmpdir(), 'relay-live-cell-image-overlay-')) - try { - const cells = join(dir, 'cells.json') - const templates = join(dir, 'templates.json') - const output = join(dir, 'overlay.tfvars.json') - // Plan variables carry the map as written: optional attributes the tfvars omit stay absent. - const { connection_hard_cap: _omitted, ...c30 } = cell(committed) - writeFileSync(cells, JSON.stringify({ ...committedCells, 'production-gce-c30': c30 })) - writeFileSync(templates, JSON.stringify(live)) - assert.ok(!live.some(({ index }) => index === 'production-gce-c30')) - const run = (cellsPath) => spawnSync(process.execPath, [ - fileURLToPath(new URL('./relay-live-cell-image-overlay.mjs', import.meta.url)), - '--cells-json', cellsPath, '--live-templates-json', templates, - '--cell-ids', 'production-gce-c30', '--output', output - ], { encoding: 'utf8' }) - const result = run(cells) - assert.equal(result.status, 0, result.stderr) - assert.deepEqual(JSON.parse(result.stdout), { cells: 3, drifted: ['production-gce-c27'] }) - assert.deepEqual(JSON.parse(readFileSync(output, 'utf8')).relay_gce_cells['production-gce-c30'], c30) - const empty = join(dir, 'empty.json') - writeFileSync(empty, '') - const failed = run(empty) - assert.notEqual(failed.status, 0) - assert.match(failed.stderr, /committed Relay cells is empty/) - } finally { - rmSync(dir, { recursive: true, force: true }) - } -}) diff --git a/cloud/dev/scripts/relay-load-connection-failure.mjs b/cloud/dev/scripts/relay-load-connection-failure.mjs deleted file mode 100644 index a189f62c44a..00000000000 --- a/cloud/dev/scripts/relay-load-connection-failure.mjs +++ /dev/null @@ -1,45 +0,0 @@ -// A refused home cell is not fleet capacity, so it gets its own bucket rather -// than inflating the capacity count a run is read for. -const ASSIGNMENT_REJECTION_BUCKETS = { - relay_capacity_exhausted: 'assignment_capacity_exhausted', - relay_connection_headroom_exhausted: 'assignment_capacity_exhausted', - relay_home_cell_unavailable: 'assignment_home_cell_unavailable' -} - -export function relayLoadFailureReason(error) { - const message = error instanceof Error ? error.message : String(error) - const tokenExchange = /^relay token exchange failed: ([1-5][0-9]{2})$/.exec(message) - if (tokenExchange) return `token_http_${tokenExchange[1]}` - const assignment = - /^relay assignment failed: ([1-5][0-9]{2})(?: (relay_[a-z_]+))?$/.exec(message) - if (assignment?.[1] === '503' && assignment[2]) { - return ASSIGNMENT_REJECTION_BUCKETS[assignment[2]] ?? `assignment_http_${assignment[1]}` - } - if (assignment) return `assignment_http_${assignment[1]}` - const closed = /^control closed: ([0-9]{4})\b/.exec(message) - if (closed) return `control_close_${closed[1]}` - if (message === 'control open timeout') return 'control_open_timeout' - if (message === 'control response timeout') return 'control_response_timeout' - if (message === 'relay token exchange timeout') return 'token_timeout' - if (message === 'relay assignment timeout') return 'assignment_timeout' - if (message === 'WebSocket was closed before the connection was established') { - return 'socket_closed_before_open' - } - if (message === 'relay token exchange omitted token') return 'token_response_invalid' - if (message === 'relay assignment response invalid') return 'assignment_response_invalid' - if (message === 'expected host challenge') return 'host_challenge_invalid' - if (message === 'host proof challenge did not decrypt') return 'host_challenge_decrypt_failed' - if (message === 'expected host hello acknowledgement') return 'host_ack_invalid' - const socketResponse = /^Unexpected server response: ([1-5][0-9]{2})\b/.exec(message) - if (socketResponse) return `socket_http_${socketResponse[1]}` - if (/\b(?:ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ETIMEDOUT)\b/.test(message)) { - return 'socket_transport' - } - return 'unknown' -} - -export function discardFailedLoadSocket(socket) { - if (!socket) return - socket.on('error', () => undefined) - socket.terminate() -} diff --git a/cloud/dev/scripts/relay-load-connection-failure.test.mjs b/cloud/dev/scripts/relay-load-connection-failure.test.mjs deleted file mode 100644 index f583f7b6659..00000000000 --- a/cloud/dev/scripts/relay-load-connection-failure.test.mjs +++ /dev/null @@ -1,57 +0,0 @@ -import assert from 'node:assert/strict' -import { EventEmitter } from 'node:events' -import test from 'node:test' -import { - discardFailedLoadSocket, - relayLoadFailureReason -} from './relay-load-connection-failure.mjs' - -test('classifies only bounded aggregate connection failure reasons', () => { - assert.equal(relayLoadFailureReason(new Error('relay token exchange failed: 503')), 'token_http_503') - assert.equal(relayLoadFailureReason(new Error('relay assignment failed: 503')), 'assignment_http_503') - assert.equal( - relayLoadFailureReason( - new Error('relay assignment failed: 503 relay_connection_headroom_exhausted') - ), - 'assignment_capacity_exhausted' - ) - for (const status of [400, 429, 500]) { - assert.equal( - relayLoadFailureReason( - new Error(`${`relay assignment failed: ${status}`} relay_capacity_exhausted`) - ), - `assignment_http_${status}` - ) - } - assert.equal(relayLoadFailureReason(new Error('control closed: 4404 wrong cell')), 'control_close_4404') - assert.equal(relayLoadFailureReason(new Error('control open timeout')), 'control_open_timeout') - assert.equal(relayLoadFailureReason(new Error('relay token exchange timeout')), 'token_timeout') - assert.equal(relayLoadFailureReason(new Error('relay assignment timeout')), 'assignment_timeout') - assert.equal( - relayLoadFailureReason(new Error('relay token exchange omitted token')), - 'token_response_invalid' - ) - assert.equal( - relayLoadFailureReason(new Error('relay assignment response invalid')), - 'assignment_response_invalid' - ) - assert.equal( - relayLoadFailureReason(new Error('Unexpected server response: 503 Service Unavailable')), - 'socket_http_503' - ) - assert.equal(relayLoadFailureReason(new Error('connect ECONNRESET 127.0.0.1')), 'socket_transport') - assert.equal(relayLoadFailureReason(new Error('expected host challenge')), 'host_challenge_invalid') - assert.equal( - relayLoadFailureReason(new Error('host proof challenge did not decrypt')), - 'host_challenge_decrypt_failed' - ) - assert.equal(relayLoadFailureReason(new Error('expected host hello acknowledgement')), 'host_ack_invalid') - assert.equal(relayLoadFailureReason(new Error('host-sensitive detail')), 'unknown') -}) - -test('absorbs the setup error emitted while discarding a failed socket', () => { - const socket = new EventEmitter() - socket.terminate = () => socket.emit('error', new Error('closed before open')) - - assert.doesNotThrow(() => discardFailedLoadSocket(socket)) -}) diff --git a/cloud/dev/scripts/relay-load-control-peer.mjs b/cloud/dev/scripts/relay-load-control-peer.mjs deleted file mode 100644 index d58fc0c10b6..00000000000 --- a/cloud/dev/scripts/relay-load-control-peer.mjs +++ /dev/null @@ -1,918 +0,0 @@ -import { createHash, createHmac } from 'node:crypto' -import { createRequire } from 'node:module' -import { controlPhase } from './relay-load-model.mjs' -import { discardFailedLoadSocket } from './relay-load-connection-failure.mjs' - -const requireFromRelay = createRequire(new URL('../../apps/relay/package.json', import.meta.url)) -const nacl = requireFromRelay('tweetnacl') -const WebSocket = requireFromRelay('ws') -const { SignJWT } = await import(requireFromRelay.resolve('jose')) -const { buildHostProofMacInput, HOST_CHALLENGE_PLAINTEXT_DOMAIN } = await import( - requireFromRelay.resolve('@orca-cloud/relay-contract') -) - -export const CONTROL_SILENCE_LIMIT_MS = 75_000 -const CONTROL_SILENCE_CHECK_INTERVAL_MS = 15_000 - -const REPORTABLE_ASSIGNMENT_ERRORS = [ - 'relay_capacity_exhausted', - 'relay_connection_headroom_exhausted', - 'relay_home_cell_unavailable' -] - -export function waitForOpen(socket, timeoutMs = 10_000) { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => finish(new Error('control open timeout')), timeoutMs) - const finish = (error) => { - clearTimeout(timer) - socket.off('open', onOpen) - socket.off('close', onClose) - socket.off('error', onError) - if (error) reject(error) - else resolve() - } - const onOpen = () => finish() - const onClose = (code, reason) => finish(new Error(`control closed: ${code} ${reason}`)) - const onError = (error) => finish(error) - socket.once('open', onOpen) - socket.once('close', onClose) - socket.once('error', onError) - }) -} - -export function nextJson(socket, timeoutMs = 10_000) { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => finish(new Error('control response timeout')), timeoutMs) - const finish = (error, value) => { - clearTimeout(timer) - socket.off('message', onMessage) - socket.off('close', onClose) - socket.off('error', onError) - if (error) reject(error) - else resolve(value) - } - const onMessage = (data) => { - try { - finish(undefined, JSON.parse(data.toString())) - } catch (error) { - finish(error) - } - } - const onClose = (code, reason) => finish(new Error(`control closed: ${code} ${reason}`)) - const onError = (error) => finish(error) - socket.once('message', onMessage) - socket.once('close', onClose) - socket.once('error', onError) - }) -} - -export function nextFrame(socket, timeoutMs = 10_000) { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => finish(new Error('relay frame timeout')), timeoutMs) - const finish = (error, value) => { - clearTimeout(timer) - socket.off('message', onMessage) - socket.off('close', onClose) - socket.off('error', onError) - if (error) reject(error) - else resolve(value) - } - const onMessage = (data, binary) => finish(undefined, { bytes: Buffer.from(data), binary }) - const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`)) - const onError = (error) => finish(error) - socket.once('message', onMessage) - socket.once('close', onClose) - socket.once('error', onError) - }) -} - -function receiveBinaryStream(socket, expectedBytes, timeoutMs) { - return new Promise((resolve, reject) => { - let receivedBytes = 0 - const hash = createHash('sha256') - const timer = setTimeout(() => finish(new Error('relay stream timeout')), timeoutMs) - const finish = (error, value) => { - clearTimeout(timer) - socket.off('message', onMessage) - socket.off('close', onClose) - socket.off('error', onError) - if (error) reject(error) - else resolve(value) - } - const onMessage = (data, binary) => { - if (!binary) return finish(new Error('relay changed stream opcode')) - const bytes = Buffer.from(data) - receivedBytes += bytes.byteLength - hash.update(bytes) - if (receivedBytes > expectedBytes) return finish(new Error('relay expanded reader stream')) - if (receivedBytes === expectedBytes) { - finish(undefined, { bytes: receivedBytes, digest: hash.digest('hex') }) - } - } - const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`)) - const onError = (error) => finish(error) - socket.on('message', onMessage) - socket.once('close', onClose) - socket.once('error', onError) - }) -} - -function closeInfo(socket, timeoutMs) { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => finish(new Error('reader close timeout')), timeoutMs) - const finish = (error, value) => { - clearTimeout(timer) - socket.off('close', onClose) - socket.off('error', onError) - if (error) reject(error) - else resolve(value) - } - const onClose = (code, reason) => finish(undefined, { code, reason: reason.toString() }) - const onError = (error) => finish(error) - socket.once('close', onClose) - socket.once('error', onError) - }) -} - -export function relayLoadWedgedCloseAccepted(closeCodes) { - return closeCodes.length === 2 && closeCodes[1] === 4429 && - (closeCodes[0] === 4429 || closeCodes[0] === 1006) -} - -function waitForClose(socket, timeoutMs = 10_000) { - if (!socket || socket.readyState === socket.CLOSED) return Promise.resolve() - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - socket.off('close', onClose) - discardFailedLoadSocket(socket) - reject(new Error('control close timeout')) - }, timeoutMs) - const onClose = () => { - clearTimeout(timer) - resolve() - } - socket.once('close', onClose) - }) -} - -async function cancelResponse(response) { - try { - await response.body?.cancel() - } catch { - // Preserve the bounded failure classification. - } -} - -function proofForChallenge(challenge, hostSecretKey) { - const plaintext = nacl.box.open( - Buffer.from(challenge.ciphertextB64, 'base64'), - Buffer.from(challenge.nonceB64, 'base64'), - Buffer.from(challenge.relayEphemeralPublicKeyB64, 'base64'), - hostSecretKey - ) - if (!plaintext) throw new Error('host proof challenge did not decrypt') - const domain = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`) - const transcriptLength = new DataView( - plaintext.buffer, - plaintext.byteOffset + domain.length, - 4 - ).getUint32(0, false) - const transcriptStart = domain.length + 4 - const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength) - const secret = plaintext.slice(transcriptStart + transcriptLength) - return createHmac('sha256', secret).update(buildHostProofMacInput(transcript)).digest('base64') -} - -export class RelayLoadControlPeer { - constructor(index, options, observe) { - if (options.directorOrigin && options.targetOrigin) { - throw new Error('provide either directorOrigin or targetOrigin, not both') - } - this.index = index - this.options = options - this.observe = observe - // A twin shares a host's keys, so it can seat the same relay host on a second cell. - this.keys = options.keys ?? nacl.box.keyPair() - this.relayHostId = createHash('sha256') - .update(this.keys.publicKey) - .digest('base64url') - .slice(0, 16) - this.phase = controlPhase(index, options.seed) - this.socket = null - this.generation = undefined - this.controlResumeSecret = undefined - this.lastAssignment = undefined - this.refreshTimer = null - this.stopped = false - this.connecting = false - this.inFlight = new Set() - this.shutdownPromise = null - this.abortController = new AbortController() - this.drainExpected = false - this.controlWaiters = new Set() - this.spliceSockets = new Set() - this.spliceSequence = 0 - } - - // `assignment`, when given, is used instead of asking the director. - connect(assignment) { - if ( - this.stopped || - this.connecting || - (this.socket !== null && this.socket.readyState === this.socket.OPEN) - ) { - return Promise.resolve() - } - this.connecting = true - const operation = this.connectOnce(assignment) - this.inFlight.add(operation) - const finish = () => { - this.connecting = false - this.inFlight.delete(operation) - } - operation.then(finish, finish) - return operation - } - - assignedCellUrl() { - return this.lastAssignment?.cellUrl - } - - async connectOnce(givenAssignment) { - let socket = null - try { - const relayToken = await this.relayToken() - if (this.stopped) return - const assignment = givenAssignment ?? (await this.assignment(relayToken)) - if (this.stopped) return - this.lastAssignment = assignment - socket = this.createSocket(assignment, relayToken) - this.socket = socket - this.drainExpected = false - await waitForOpen(socket) - if (this.stopped || this.socket !== socket) return - socket.send( - JSON.stringify({ - type: 'host-hello', - v: 1, - relayHostId: this.relayHostId, - assignmentEpoch: assignment.assignmentEpoch, - hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64'), - appVersion: 'relay-load', - ...(this.generation === undefined ? {} : { previousGeneration: this.generation }), - ...(this.controlResumeSecret === undefined - ? {} - : { controlResumeSecret: this.controlResumeSecret }) - }) - ) - const challenge = await nextJson(socket) - if (this.stopped || this.socket !== socket) return - if (challenge.type !== 'host-challenge') throw new Error('expected host challenge') - socket.send( - JSON.stringify({ - type: 'host-challenge-ack', - challengeId: challenge.challengeId, - proofB64: proofForChallenge(challenge, this.keys.secretKey) - }) - ) - const ack = await nextJson(socket) - if (this.stopped || this.socket !== socket) return - if (ack.type !== 'host-hello-ack') throw new Error('expected host hello acknowledgement') - this.generation = ack.generation - this.controlResumeSecret = ack.controlResumeSecret - socket.on('message', (data) => this.onMessage(socket, data)) - this.watchSilence(socket) - socket.once('close', (code) => this.onClose(socket, code)) - socket.once('error', (error) => this.observe('socketError', { index: this.index, error })) - this.observe('connected', { index: this.index }) - this.scheduleRefresh(this.phase.refreshOffsetMs) - } catch (error) { - discardFailedLoadSocket(socket) - if (this.socket === socket) this.socket = null - if (this.stopped) return - throw error - } - } - - createSocket(assignment, relayToken) { - return new WebSocket(`${assignment.cellUrl.replace(/^http/, 'ws')}/v1/host/control`, { - headers: { authorization: `Bearer ${relayToken}` }, - perMessageDeflate: false - }) - } - - shutdown() { - if (this.shutdownPromise) return this.shutdownPromise - this.stopped = true - this.abortController.abort() - if (this.refreshTimer) clearTimeout(this.refreshTimer) - this.refreshTimer = null - this.shutdownPromise = this.shutdownOnce() - return this.shutdownPromise - } - - async shutdownOnce() { - const socket = this.socket - const closed = waitForClose(socket) - for (const spliceSocket of this.spliceSockets) { - if ( - spliceSocket.readyState !== spliceSocket.CLOSED && - spliceSocket.readyState !== spliceSocket.CLOSING - ) { - spliceSocket.close(1000, 'load complete') - } - } - this.rejectControlWaiters(new Error('control stopped')) - if (socket && socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) { - socket.close(1000, 'load complete') - } - const settled = async () => { - while (this.inFlight.size > 0) { - await Promise.allSettled([...this.inFlight]) - } - } - await Promise.all([closed, settled()]) - this.observe('shutdown', { - index: this.index, - activeControls: this.socket?.readyState === this.socket?.OPEN ? 1 : 0, - activeSpliceSockets: this.spliceSockets.size, - inFlightOperations: this.inFlight.size, - refreshTimerActive: this.refreshTimer !== null - }) - } - - openSplice(options = {}) { - if (this.stopped) return Promise.reject(new Error('control stopped')) - const operation = this.openSpliceOnce(options) - this.inFlight.add(operation) - const finish = () => this.inFlight.delete(operation) - operation.then(finish, finish) - return operation - } - - openInviteOffer() { - if (this.stopped) return Promise.reject(new Error('control stopped')) - const operation = this.openInviteOfferOnce() - this.inFlight.add(operation) - const finish = () => this.inFlight.delete(operation) - operation.then(finish, finish) - return operation - } - - async openInviteOfferOnce() { - if (!this.socket || this.socket.readyState !== this.socket.OPEN) { - throw new Error('active control required for invite offer') - } - const sequence = this.spliceSequence++ - const reqId = `load-offer-${this.index}-${sequence}` - const response = this.waitForControlMessage( - (message) => - message.reqId === reqId && - (message.type === 'invite-created' || message.type === 'control-error') - ) - this.socket.send(JSON.stringify({ - type: 'invite-create', - reqId, - relayDeviceId: `load-offer-device-${this.index}-${sequence}` - })) - const result = await response - if (result.type === 'control-error') throw new Error(`invite offer failed: ${result.code}`) - if ( - typeof result.inviteToken !== 'string' || - !Number.isSafeInteger(result.expiresAt) || - result.expiresAt <= Date.now() - ) throw new Error('relay invite offer response invalid') - } - - async openSpliceOnce({ - payloadBytes = 64, - readerMode = 'normal', - readerHoldMs = 0, - streamBytes = payloadBytes, - frameBytes = payloadBytes, - observeReaderPressure = async () => undefined, - readerDelay = async (ms) => await new Promise((resolve) => setTimeout(resolve, ms)), - slowReaderHoldMs = 0, - holdMs = 0 - } = {}) { - if ( - !this.socket || - this.socket.readyState !== this.socket.OPEN || - this.generation === undefined || - this.lastAssignment === undefined - ) { - throw new Error('active control required for splice') - } - if (!Number.isSafeInteger(payloadBytes) || payloadBytes < 1) { - throw new Error('splice payload bytes must be positive') - } - const sequence = this.spliceSequence++ - const reqId = `load-invite-${this.index}-${sequence}` - const relayDeviceId = `load-device-${this.index}-${sequence}` - let phone - let data - let opened = false - try { - const invitePromise = this.waitForControlMessage( - (message) => message.type === 'invite-created' && message.reqId === reqId - ) - this.socket.send(JSON.stringify({ type: 'invite-create', reqId, relayDeviceId })) - const invite = await invitePromise - if (typeof invite.inviteToken !== 'string') throw new Error('relay invite response invalid') - - phone = this.createClientSocket(this.lastAssignment) - this.trackSpliceSocket(phone) - await waitForOpen(phone) - const connectionPromise = this.waitForControlMessage( - (message) => message.type === 'conn-open' && message.relayDeviceId === relayDeviceId - ) - phone.send( - JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken }) - ) - const connection = await connectionPromise - if (typeof connection.connId !== 'string' || typeof connection.connTicket !== 'string') { - throw new Error('relay connection response invalid') - } - - data = this.createHostDataSocket(this.lastAssignment, connection.connId) - this.trackSpliceSocket(data) - await waitForOpen(data) - const phoneHello = nextJson(phone) - data.send( - JSON.stringify({ - type: 'host-data-auth', - v: 1, - connTicket: connection.connTicket, - generation: this.generation - }) - ) - if ((await phoneHello).ok !== true) throw new Error('relay rejected load splice') - - if (slowReaderHoldMs > 0 && readerMode === 'normal') { - readerMode = 'slow' - readerHoldMs = slowReaderHoldMs - streamBytes = payloadBytes - frameBytes = payloadBytes - } - if (!['normal', 'slow', 'wedged'].includes(readerMode)) { - throw new Error('reader mode is invalid') - } - const pausedSocket = readerMode === 'normal' ? undefined : phone._socket - if (readerMode !== 'normal' && !pausedSocket) throw new Error('reader transport unavailable') - if (readerMode === 'wedged') { - const closes = [closeInfo(phone, readerHoldMs + 10_000), closeInfo(data, readerHoldMs + 10_000)] - pausedSocket.pause() - const readerPausedAt = Date.now() - const readerPressure = observeReaderPressure({ - cellOrigin: this.lastAssignment.cellUrl, - readerMode, - streamBytes - }) - const [sent] = await Promise.all([ - this.sendReaderStream(data, sequence, streamBytes, frameBytes, readerDelay), - readerPressure - ]) - await readerDelay(Math.max(0, readerHoldMs - (Date.now() - readerPausedAt))) - pausedSocket.resume() - const closeEvidence = await Promise.all(closes) - const closeCodes = closeEvidence.map(({ code }) => code) - if (!relayLoadWedgedCloseAccepted(closeCodes)) { - throw new Error(`wedged reader close codes: ${closeCodes.join(',')}`) - } - opened = true - this.observe('spliceOpened', { index: this.index, readerMode }) - this.observe('spliceWedged', { index: this.index, code: 4429, streamBytes: sent.bytes }) - return - } - - const expectedStreamBytes = readerMode === 'slow' ? streamBytes : payloadBytes - const expectedFrameBytes = readerMode === 'slow' ? frameBytes : payloadBytes - const phoneStream = receiveBinaryStream( - phone, - expectedStreamBytes, - readerMode === 'slow' ? readerHoldMs + 10_000 : 10_000 - ) - const readerPausedAt = pausedSocket ? Date.now() : 0 - if (pausedSocket) pausedSocket.pause() - const readerPressure = readerMode === 'slow' - ? observeReaderPressure({ - cellOrigin: this.lastAssignment.cellUrl, - readerMode, - streamBytes: expectedStreamBytes - }) - : Promise.resolve() - const [sent] = await Promise.all([ - this.sendReaderStream( - data, - sequence, - expectedStreamBytes, - expectedFrameBytes, - readerDelay - ), - readerPressure - ]) - if (readerMode === 'slow') { - await readerDelay(Math.max(0, readerHoldMs - (Date.now() - readerPausedAt))) - pausedSocket.resume() - } - const receivedByPhone = await phoneStream - if (receivedByPhone.bytes !== sent.bytes || receivedByPhone.digest !== sent.digest) { - throw new Error('relay changed host-to-client splice payload') - } - - const textPayload = `orca-relay-load:${this.index}:${sequence}:${payloadBytes}` - const dataFrame = nextFrame(data) - phone.send(textPayload) - const receivedByHost = await dataFrame - if (receivedByHost.binary || receivedByHost.bytes.toString() !== textPayload) { - throw new Error('relay changed client-to-host splice payload') - } - opened = true - this.observe('spliceOpened', { index: this.index, readerMode }) - if (holdMs > 0 && !(await this.waitForSpliceHold(holdMs, [phone, data]))) return - this.observe('spliceCompleted', { index: this.index, readerMode }) - } catch (error) { - if (!this.stopped) this.observe('spliceFailed', { index: this.index, error }) - throw error - } finally { - await Promise.all([this.closeSpliceSocket(phone), this.closeSpliceSocket(data)]) - if (opened) this.observe('spliceClosed', { index: this.index }) - } - } - - waitForSpliceHold(holdMs, sockets) { - if (this.stopped) return Promise.resolve(false) - return new Promise((resolve, reject) => { - const finish = (error, completed = false) => { - clearTimeout(timer) - this.abortController.signal.removeEventListener('abort', onAbort) - for (const socket of sockets) { - socket.off('close', onClose) - socket.off('error', onError) - } - if (error) reject(error) - else resolve(completed) - } - const onAbort = () => finish(undefined, false) - const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`)) - const onError = (error) => finish(error) - const timer = setTimeout(() => finish(undefined, true), holdMs) - this.abortController.signal.addEventListener('abort', onAbort, { once: true }) - for (const socket of sockets) { - socket.once('close', onClose) - socket.once('error', onError) - } - }) - } - - createClientSocket(assignment) { - return new WebSocket( - `${assignment.cellUrl.replace(/^http/, 'ws')}/v1/connect/${this.relayHostId}`, - { perMessageDeflate: false } - ) - } - - createHostDataSocket(assignment, connId) { - return new WebSocket( - `${assignment.cellUrl.replace(/^http/, 'ws')}/v1/host/data/${connId}`, - { perMessageDeflate: false } - ) - } - - splicePayload(sequence, payloadBytes) { - const seed = createHash('sha256') - .update(`orca-relay-load:${this.index}:${sequence}`) - .digest() - return Buffer.allocUnsafe(payloadBytes).map((_, index) => seed[index % seed.length]) - } - - async sendReaderStream(socket, sequence, streamBytes, frameBytes, delay) { - const hash = createHash('sha256') - let sentBytes = 0 - let frameIndex = 0 - while (sentBytes < streamBytes) { - const bytes = Math.min(frameBytes, streamBytes - sentBytes) - const payload = this.splicePayload(sequence + frameIndex, bytes) - await this.sendReaderFrame(socket, payload) - hash.update(payload) - sentBytes += bytes - frameIndex++ - while (socket.bufferedAmount > frameBytes) await delay(10) - } - return { bytes: sentBytes, digest: hash.digest('hex') } - } - - sendReaderFrame(socket, payload) { - if (socket.send.length < 2) { - socket.send(payload) - return Promise.resolve() - } - return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error('reader send timeout')), 10_000) - socket.send(payload, (error) => { - clearTimeout(timer) - if (error) reject(error) - else resolve() - }) - }) - } - - trackSpliceSocket(socket) { - this.spliceSockets.add(socket) - socket.once('close', () => this.spliceSockets.delete(socket)) - } - - async closeSpliceSocket(socket) { - if (!socket) return - const closed = waitForClose(socket).catch(() => undefined) - if (socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) { - socket.close(1000, 'splice complete') - } - await closed - this.spliceSockets.delete(socket) - } - - async openRebindProbe() { - if ( - !this.socket || - this.socket.readyState !== this.socket.OPEN || - this.generation === undefined || - this.controlResumeSecret === undefined || - this.lastAssignment === undefined - ) { - throw new Error('active control required for rebind probe') - } - const relayToken = await this.relayToken() - const socket = new WebSocket( - `${this.lastAssignment.cellUrl.replace(/^http/, 'ws')}/v1/host/control`, - { - headers: { authorization: `Bearer ${relayToken}` }, - perMessageDeflate: false - } - ) - try { - await waitForOpen(socket) - socket.send( - JSON.stringify({ - type: 'host-hello', - v: 1, - relayHostId: this.relayHostId, - assignmentEpoch: this.lastAssignment.assignmentEpoch, - hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64'), - appVersion: 'relay-load-rebind-proof', - previousGeneration: this.generation, - controlResumeSecret: this.controlResumeSecret - }) - ) - const challenge = await nextJson(socket) - if (challenge.type !== 'host-challenge') throw new Error('expected host challenge') - socket.on('error', () => undefined) - const closed = new Promise((resolve) => socket.once('close', resolve)) - return { - close: async () => { - const closeCompleted = waitForClose(socket) - if (socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) { - socket.close(1000, 'rebind boundary proved') - } - await closeCompleted - }, - closed, - isOpen: () => socket.readyState === socket.OPEN - } - } catch (error) { - discardFailedLoadSocket(socket) - await waitForClose(socket).catch(() => undefined) - throw error - } - } - - async relayToken() { - const accessToken = this.options.accessTokenProvider - ? await this.options.accessTokenProvider() - : this.options.accessToken - if (accessToken) { - const body = await this.requestJson( - `${this.options.authOrigin}/v1/desktop/auth/relay-token`, - { - method: 'POST', - headers: { - authorization: `Bearer ${accessToken}`, - 'content-type': 'application/json' - }, - body: JSON.stringify({ - relayHostId: this.relayHostId, - hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64') - }) - }, - 'relay token exchange timeout', - (status) => `relay token exchange failed: ${status}` - ) - if (typeof body.relayToken !== 'string') throw new Error('relay token exchange omitted token') - if (!this.stopped) this.observe('token', { index: this.index }) - return body.relayToken - } - const token = await new SignJWT({ - prof: `load-profile-${this.index}`, - org: 'relay-load', - purpose: 'host-control', - relayHostId: this.relayHostId - }) - .setProtectedHeader({ alg: 'ES256', kid: this.options.signingKeyId }) - .setIssuer(this.options.authOrigin) - .setAudience('orca-relay') - .setSubject(`load-user-${this.index}`) - .setIssuedAt() - .setExpirationTime('5m') - .sign(this.options.signingKey) - if (!this.stopped) this.observe('token', { index: this.index }) - return token - } - - async requestAssignment(preferredRegion) { - return await this.assignment(await this.relayToken(), preferredRegion) - } - - async assignment(relayToken, preferredRegion = this.options.preferredRegion) { - if (!this.options.directorOrigin) { - if (!this.options.targetOrigin) throw new Error('relay target origin missing') - return { cellUrl: this.options.targetOrigin, assignmentEpoch: 1 } - } - const body = await this.requestJson( - `${this.options.directorOrigin}/v1/assign`, - { - method: 'POST', - headers: { authorization: `Bearer ${relayToken}`, 'content-type': 'application/json' }, - body: JSON.stringify({ - v: 1, - relayHostId: this.relayHostId, - ...(preferredRegion ? { preferredRegion } : {}), - // Desktops declare a reconnect after an established control; the director's sticky lane needs it. - ...(this.generation === undefined ? {} : { reconnect: true }) - }) - }, - 'relay assignment timeout', - (status, errorCode) => - `relay assignment failed: ${status}${errorCode ? ` ${errorCode}` : ''}`, - REPORTABLE_ASSIGNMENT_ERRORS - ) - if ( - typeof body.cellUrl !== 'string' || - !Number.isSafeInteger(body.assignmentEpoch) || - body.assignmentEpoch < 1 - ) { - throw new Error('relay assignment response invalid') - } - return body - } - - async requestJson(url, init, timeoutMessage, httpErrorMessage, allowedErrorCodes = []) { - const controller = new AbortController() - const onShutdown = () => controller.abort() - if (this.abortController.signal.aborted) controller.abort() - else this.abortController.signal.addEventListener('abort', onShutdown, { once: true }) - let timedOut = false - const timer = setTimeout(() => { - timedOut = true - controller.abort() - }, this.options.requestTimeoutMs ?? 10_000) - try { - const response = await fetch(url, { ...init, signal: controller.signal }) - if (!response.ok) { - let bodyConsumed = false - let errorCode - if (allowedErrorCodes.length > 0) { - try { - const body = await response.json() - bodyConsumed = true - if (allowedErrorCodes.includes(body?.error)) errorCode = body.error - } catch { - // Preserve the bounded status-only classification. - } - } - if (!bodyConsumed) await cancelResponse(response) - throw new Error(httpErrorMessage(response.status, errorCode)) - } - return await response.json() - } catch (error) { - if (timedOut) throw new Error(timeoutMessage, { cause: error }) - throw error - } finally { - clearTimeout(timer) - this.abortController.signal.removeEventListener('abort', onShutdown) - } - } - - onMessage(socket, data) { - let message - try { - message = JSON.parse(data.toString()) - } catch { - this.observe('protocolError', { index: this.index }) - return - } - for (const waiter of this.controlWaiters) { - if (waiter.matches(message)) { - this.controlWaiters.delete(waiter) - clearTimeout(waiter.timer) - waiter.resolve(message) - return - } - } - if (message.type === 'ping') { - socket.send(JSON.stringify({ type: 'pong', t: message.t })) - this.observe('ping', { index: this.index }) - } else if (message.type === 'drain') { - this.drainExpected = true - this.observe('drain', { index: this.index }) - } - } - - // Desktops close a control after 75 s with no inbound frame (the cell pings every 15 s); the - // load balancer keeps a dead backend's client socket open, so without this a host never re-assigns. - watchSilence(socket) { - let lastInboundAt = Date.now() - const noteInbound = () => { - lastInboundAt = Date.now() - } - socket.on('message', noteInbound) - socket.on('ping', noteInbound) - socket.on('pong', noteInbound) - const timer = setInterval(() => { - if (Date.now() - lastInboundAt > CONTROL_SILENCE_LIMIT_MS) socket.terminate() - }, CONTROL_SILENCE_CHECK_INTERVAL_MS) - timer.unref?.() - socket.once('close', () => clearInterval(timer)) - } - - onClose(socket, code) { - if (this.socket !== socket) return - this.socket = null - if (this.refreshTimer) clearTimeout(this.refreshTimer) - this.refreshTimer = null - this.rejectControlWaiters(new Error(`control closed: ${code}`)) - this.observe('closed', { - index: this.index, - code, - stopped: this.stopped, - expectedDrain: this.drainExpected - }) - this.drainExpected = false - } - - waitForControlMessage(matches, timeoutMs = 10_000) { - if (this.stopped) return Promise.reject(new Error('control stopped')) - return new Promise((resolve, reject) => { - const waiter = { - matches, - resolve, - reject, - timer: setTimeout(() => { - this.controlWaiters.delete(waiter) - reject(new Error('control response timeout')) - }, timeoutMs) - } - this.controlWaiters.add(waiter) - }) - } - - rejectControlWaiters(error) { - for (const waiter of this.controlWaiters) { - clearTimeout(waiter.timer) - waiter.reject(error) - } - this.controlWaiters.clear() - } - - scheduleRefresh(delayMs) { - if (this.stopped) return - const socket = this.socket - const reschedule = () => { - if (this.socket === socket) this.scheduleRefresh(this.phase.refreshIntervalMs) - } - this.refreshTimer = setTimeout(() => { - void this.refresh().then(reschedule, reschedule) - }, delayMs) - } - - refresh() { - if (this.stopped) return Promise.resolve() - const operation = this.refreshOnce() - this.inFlight.add(operation) - const finish = () => this.inFlight.delete(operation) - operation.then(finish, finish) - return operation - } - - async refreshOnce() { - const socket = this.socket - if (this.stopped || !socket || socket.readyState !== socket.OPEN) return - try { - const relayJwt = await this.relayToken() - if (this.stopped || this.socket !== socket || socket.readyState !== socket.OPEN) return - socket.send(JSON.stringify({ type: 'auth-refresh', relayJwt })) - this.observe('refresh', { index: this.index }) - } catch (error) { - if (!this.stopped) this.observe('refreshError', { index: this.index, error }) - } - } -} diff --git a/cloud/dev/scripts/relay-load-control-peer.test.mjs b/cloud/dev/scripts/relay-load-control-peer.test.mjs deleted file mode 100644 index 0f7fba19728..00000000000 --- a/cloud/dev/scripts/relay-load-control-peer.test.mjs +++ /dev/null @@ -1,1074 +0,0 @@ -import assert from 'node:assert/strict' -import { generateKeyPairSync } from 'node:crypto' -import { EventEmitter } from 'node:events' -import { createRequire } from 'node:module' -import test from 'node:test' -import { - RelayLoadControlPeer, - relayLoadWedgedCloseAccepted -} from './relay-load-control-peer.mjs' - -const requireFromRelay = createRequire(new URL('../../apps/relay/package.json', import.meta.url)) -const nacl = requireFromRelay('tweetnacl') -const { buildHostChallengePlaintext } = await import( - requireFromRelay.resolve('@orca-cloud/relay-contract') -) - -function deferred() { - let resolve - let reject - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise - reject = rejectPromise - }) - return { promise, resolve, reject } -} - -function peerOptions(overrides = {}) { - const { privateKey } = generateKeyPairSync('ec', { namedCurve: 'P-256' }) - return { - authOrigin: 'https://auth.test', - directorOrigin: 'https://director.test', - reconnectMaxMs: 0, - seed: 1, - signingKey: privateKey, - signingKeyId: 'test-key', - ...overrides - } -} - -function response(body) { - return { ok: true, status: 200, json: async () => body } -} - -function fakeOpenSocket() { - const socket = new EventEmitter() - socket.OPEN = 1 - socket.CLOSING = 2 - socket.CLOSED = 3 - socket.readyState = socket.OPEN - socket.sent = [] - socket.send = (message) => socket.sent.push(message) - socket.close = (code = 1000, reason = '') => { - socket.readyState = socket.CLOSING - queueMicrotask(() => { - socket.readyState = socket.CLOSED - socket.emit('close', code, Buffer.from(reason)) - }) - } - socket.terminate = socket.close - return socket -} - -function fakeHandshakeSocket() { - const socket = fakeOpenSocket() - socket.CONNECTING = 0 - socket.readyState = socket.CONNECTING - socket.open = () => { - socket.readyState = socket.OPEN - socket.emit('open') - } - socket.message = (message) => socket.emit('message', Buffer.from(JSON.stringify(message))) - return socket -} - -function openOnNextTurn(socket) { - queueMicrotask(() => socket.open()) - return socket -} - -function validChallenge(peer) { - const relayKeys = nacl.box.keyPair() - const nonce = nacl.randomBytes(nacl.box.nonceLength) - const plaintext = buildHostChallengePlaintext( - new Uint8Array([1, 2, 3]), - nacl.randomBytes(32) - ) - const ciphertext = nacl.box(plaintext, nonce, peer.keys.publicKey, relayKeys.secretKey) - return { - type: 'host-challenge', - challengeId: 'test-challenge', - ciphertextB64: Buffer.from(ciphertext).toString('base64'), - nonceB64: Buffer.from(nonce).toString('base64'), - relayEphemeralPublicKeyB64: Buffer.from(relayKeys.publicKey).toString('base64') - } -} - -test('shutdown waits for pending assignment and prevents a late connection', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - const assignment = deferred() - const assignmentStarted = deferred() - global.fetch = () => { - assignmentStarted.resolve() - return assignment.promise - } - const observations = [] - const peer = new RelayLoadControlPeer(0, peerOptions(), (type) => observations.push(type)) - - const connecting = peer.connect() - await assignmentStarted.promise - let shutdownFinished = false - const shutdown = peer.shutdown().then(() => { - shutdownFinished = true - }) - await new Promise((resolve) => setImmediate(resolve)) - assert.equal(shutdownFinished, false) - - assignment.resolve(response({ cellUrl: 'https://cell.test', assignmentEpoch: 1 })) - await Promise.all([connecting, shutdown]) - assert.equal(peer.socket, null) - assert.equal(observations.includes('connected'), false) -}) - -test('shutdown after socket open prevents a late handshake', async () => { - const observations = [] - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }), - (type) => observations.push(type) - ) - const socket = fakeHandshakeSocket() - const socketCreated = deferred() - peer.createSocket = () => { - socketCreated.resolve() - return socket - } - - const connecting = peer.connect() - await socketCreated.promise - socket.open() - await Promise.all([connecting, peer.shutdown()]) - - assert.deepEqual(socket.sent, []) - assert.equal(observations.includes('connected'), false) -}) - -test('shutdown after challenge prevents a late acknowledgement', async () => { - const observations = [] - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }), - (type) => observations.push(type) - ) - const socket = fakeHandshakeSocket() - const socketCreated = deferred() - const helloSent = deferred() - socket.send = (message) => { - socket.sent.push(message) - helloSent.resolve() - } - peer.createSocket = () => { - socketCreated.resolve() - return socket - } - - const connecting = peer.connect() - await socketCreated.promise - socket.open() - await helloSent.promise - socket.message({ type: 'host-challenge' }) - await Promise.all([connecting, peer.shutdown()]) - - assert.equal(socket.sent.length, 1) - assert.equal(observations.includes('connected'), false) -}) - -test('shutdown after host acknowledgement prevents a late connected observation', async () => { - const observations = [] - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }), - (type) => observations.push(type) - ) - const socket = fakeHandshakeSocket() - const socketCreated = deferred() - const helloSent = deferred() - const proofSent = deferred() - socket.send = (message) => { - socket.sent.push(message) - if (socket.sent.length === 1) helloSent.resolve() - else proofSent.resolve() - } - peer.createSocket = () => { - socketCreated.resolve() - return socket - } - - const connecting = peer.connect() - await socketCreated.promise - socket.open() - await helloSent.promise - socket.message(validChallenge(peer)) - await proofSent.promise - socket.message({ type: 'host-hello-ack', generation: 1, controlResumeSecret: 'test-secret' }) - await Promise.all([connecting, peer.shutdown()]) - - assert.equal(socket.sent.length, 2) - assert.equal(observations.includes('connected'), false) -}) - -test('shutdown prevents a pending refresh from sending or reporting success', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - const token = deferred() - const tokenStarted = deferred() - global.fetch = () => { - tokenStarted.resolve() - return token.promise - } - const observations = [] - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }), - (type) => observations.push(type) - ) - const socket = fakeOpenSocket() - peer.socket = socket - - const refreshing = peer.refresh() - await tokenStarted.promise - const shutdown = peer.shutdown() - token.resolve(response({ relayToken: 'test-relay-token' })) - await Promise.all([refreshing, shutdown]) - - assert.deepEqual(socket.sent, []) - assert.equal(observations.includes('refresh'), false) - assert.equal(observations.includes('refreshError'), false) -}) - -test('shutdown suppresses a late refresh error', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - const token = deferred() - const tokenStarted = deferred() - global.fetch = () => { - tokenStarted.resolve() - return token.promise - } - const observations = [] - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }), - (type) => observations.push(type) - ) - peer.socket = fakeOpenSocket() - - const refreshing = peer.refresh() - await tokenStarted.promise - const shutdown = peer.shutdown() - token.reject(new Error('late token failure')) - await Promise.all([refreshing, shutdown]) - - assert.equal(observations.includes('refreshError'), false) -}) - -test('shutdown aborts an HTTP request that would otherwise remain pending', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - const requestStarted = deferred() - let aborted = false - global.fetch = (_url, { signal }) => - new Promise((_resolve, reject) => { - requestStarted.resolve() - signal.addEventListener( - 'abort', - () => { - aborted = true - reject(new Error('request aborted')) - }, - { once: true } - ) - }) - const observations = [] - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }), - (type) => observations.push(type) - ) - - const connecting = peer.connect() - await requestStarted.promise - await Promise.all([connecting, peer.shutdown()]) - - assert.equal(aborted, true) - assert.equal(observations.includes('connected'), false) -}) - -test('bounds a pending HTTP request with a classified timeout', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - global.fetch = (_url, { signal }) => - new Promise((_resolve, reject) => { - signal.addEventListener('abort', () => reject(new Error('request aborted')), { once: true }) - }) - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ - accessToken: 'test-access-token', - directorOrigin: undefined, - requestTimeoutMs: 1 - }), - () => undefined - ) - - await assert.rejects(peer.connect(), /relay token exchange timeout/) - await peer.shutdown() -}) - -test('bounds a stalled token response body', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - global.fetch = async (_url, { signal }) => ({ - ok: true, - status: 200, - json: () => - new Promise((_resolve, reject) => { - signal.addEventListener('abort', () => reject(new Error('body aborted')), { once: true }) - }) - }) - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ - accessToken: 'test-access-token', - directorOrigin: undefined, - requestTimeoutMs: 1 - }), - () => undefined - ) - - await assert.rejects(peer.connect(), /relay token exchange timeout/) - await peer.shutdown() -}) - -test('bounds a stalled assignment response body', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - global.fetch = async (_url, { signal }) => ({ - ok: true, - status: 200, - json: () => - new Promise((_resolve, reject) => { - signal.addEventListener('abort', () => reject(new Error('body aborted')), { once: true }) - }) - }) - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ requestTimeoutMs: 1 }), - () => undefined - ) - - await assert.rejects(peer.connect(), /relay assignment timeout/) - await peer.shutdown() -}) - -test('shutdown aborts a stalled successful response body', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - const bodyStarted = deferred() - let bodyAborted = false - global.fetch = async (_url, { signal }) => ({ - ok: true, - status: 200, - json: () => - new Promise((_resolve, reject) => { - bodyStarted.resolve() - signal.addEventListener( - 'abort', - () => { - bodyAborted = true - reject(new Error('body aborted')) - }, - { once: true } - ) - }) - }) - const observations = [] - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }), - (type) => observations.push(type) - ) - - const connecting = peer.connect() - await bodyStarted.promise - await Promise.all([connecting, peer.shutdown()]) - - assert.equal(bodyAborted, true) - assert.equal(observations.includes('connected'), false) -}) - -test('cancels a rejected HTTP response body', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - let canceled = false - global.fetch = async () => ({ - ok: false, - status: 503, - body: { - cancel: async () => { - canceled = true - } - } - }) - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }), - () => undefined - ) - - await assert.rejects(peer.connect(), /relay token exchange failed: 503/) - await peer.shutdown() - assert.equal(canceled, true) -}) - -test('preserves only an exact capacity assignment rejection reason', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - global.fetch = async () => ({ - ok: false, - status: 503, - json: async () => ({ error: 'relay_connection_headroom_exhausted' }) - }) - const peer = new RelayLoadControlPeer(0, peerOptions(), () => undefined) - - await assert.rejects( - peer.connect(), - /relay assignment failed: 503 relay_connection_headroom_exhausted/ - ) - await peer.shutdown() -}) - -test('sends preferred region and preserves the genuine director epoch', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - let assignmentRequest - global.fetch = async (_url, init) => { - assignmentRequest = JSON.parse(init.body) - return response({ cellUrl: 'https://asia-cell.test', assignmentEpoch: 47 }) - } - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ preferredRegion: 'asia-east2' }), - () => undefined - ) - - const assignment = await peer.assignment('relay-token') - - assert.deepEqual(assignmentRequest, { - v: 1, - relayHostId: peer.relayHostId, - preferredRegion: 'asia-east2' - }) - assert.equal(assignment.assignmentEpoch, 47) - await peer.shutdown() -}) - -test('omits preferred region and rejects a fabricated director epoch', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - let assignmentRequest - global.fetch = async (_url, init) => { - assignmentRequest = JSON.parse(init.body) - return response({ cellUrl: 'https://cell.test', assignmentEpoch: 0 }) - } - const peer = new RelayLoadControlPeer(0, peerOptions(), () => undefined) - - await assert.rejects(peer.assignment('relay-token'), /assignment response invalid/) - assert.equal('preferredRegion' in assignmentRequest, false) - await peer.shutdown() -}) - -test('rejects ambiguous direct and director assignment modes', () => { - assert.throws( - () => - new RelayLoadControlPeer( - 0, - peerOptions({ targetOrigin: 'https://cell.test' }), - () => undefined - ), - /either directorOrigin or targetOrigin/ - ) -}) - -test('opens a genuine splice and verifies payloads in both directions', async () => { - const observations = [] - const peer = new RelayLoadControlPeer(3, peerOptions(), (type, detail) => { - observations.push({ type, detail }) - }) - const control = fakeOpenSocket() - const phone = fakeHandshakeSocket() - const data = fakeHandshakeSocket() - let dataPayloadBytes = 0 - let observationStarted = false - let sentBeforeObservation = false - let paused = 0 - let resumed = 0 - phone._socket = { - pause: () => paused++, - resume: () => resumed++ - } - control.send = (raw) => { - const message = JSON.parse(raw) - if (message.type === 'invite-create') { - queueMicrotask(() => - control.emit( - 'message', - Buffer.from( - JSON.stringify({ - type: 'invite-created', - reqId: message.reqId, - inviteToken: 'invite-token' - }) - ) - ) - ) - } - } - phone.send = (raw) => { - if (typeof raw === 'string' && raw.startsWith('{') && JSON.parse(raw).type === 'relay-auth') { - queueMicrotask(() => - control.emit( - 'message', - Buffer.from( - JSON.stringify({ - type: 'conn-open', - relayDeviceId: 'load-device-3-0', - connId: 'connection-1', - connTicket: 'connection-ticket' - }) - ) - ) - ) - return - } - queueMicrotask(() => data.emit('message', Buffer.from(raw), false)) - } - data.send = (raw) => { - if (typeof raw === 'string') { - queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false)) - return - } - const dataPayload = Buffer.from(raw) - if (!observationStarted) sentBeforeObservation = true - dataPayloadBytes += dataPayload.byteLength - queueMicrotask(() => phone.emit('message', dataPayload, true)) - } - peer.socket = control - peer.generation = 9 - peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 47 } - control.on('message', (raw) => peer.onMessage(control, raw)) - peer.createClientSocket = () => openOnNextTurn(phone) - peer.createHostDataSocket = () => openOnNextTurn(data) - - await peer.openSplice({ - readerMode: 'slow', - readerHoldMs: 1, - streamBytes: 300 * 1024, - frameBytes: 64 * 1024, - observeReaderPressure: async () => { observationStarted = true } - }) - - assert.equal(dataPayloadBytes, 300 * 1024) - assert.equal(sentBeforeObservation, false) - assert.equal(paused, 1) - assert.equal(resumed, 1) - assert.equal(observations.filter(({ type }) => type === 'spliceCompleted').length, 1) - assert.equal(observations.some(({ type }) => type === 'spliceFailed'), false) - await peer.shutdown() - assert.deepEqual(observations.at(-1), { - type: 'shutdown', - detail: { - index: 3, - activeControls: 0, - activeSpliceSockets: 0, - inFlightOperations: 0, - refreshTimerActive: false - } - }) -}) - -test('opens invitation leases and preserves exact capacity errors', async () => { - const peer = new RelayLoadControlPeer(4, peerOptions(), () => undefined) - const control = fakeOpenSocket() - peer.socket = control - control.on('message', (raw) => peer.onMessage(control, raw)) - let calls = 0 - control.send = (raw) => { - const request = JSON.parse(raw) - calls++ - queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify( - calls === 1 - ? { - type: 'invite-created', reqId: request.reqId, - inviteToken: 'invite-token', expiresAt: Date.now() + 60_000 - } - : { type: 'control-error', reqId: request.reqId, code: 'relay_capacity_exhausted' } - )))) - } - - await peer.openInviteOffer() - await assert.rejects(peer.openInviteOffer(), /invite offer failed: relay_capacity_exhausted/) - await peer.shutdown() -}) - -test('can request the same assignment with an explicit replacement preference', async (context) => { - const originalFetch = global.fetch - context.after(() => { global.fetch = originalFetch }) - const requests = [] - global.fetch = async (url, init) => { - if (String(url).endsWith('/v1/assign')) { - requests.push(JSON.parse(init.body)) - return response({ cellUrl: 'https://asia-cell.test', assignmentEpoch: 3 }) - } - return response({ relayToken: 'relay-token' }) - } - const peer = new RelayLoadControlPeer( - 5, - peerOptions({ accessToken: 'access-token', preferredRegion: 'asia-east2' }), - () => undefined - ) - - assert.equal((await peer.requestAssignment('us-central1')).cellUrl, 'https://asia-cell.test') - assert.equal(requests[0].preferredRegion, 'us-central1') - await peer.shutdown() -}) - -test('uses a refreshable workflow access-token provider', async (context) => { - const originalFetch = global.fetch - context.after(() => { global.fetch = originalFetch }) - let providerCalls = 0 - let authorization - global.fetch = async (url, init) => { - if (String(url).endsWith('/v1/desktop/auth/relay-token')) { - authorization = init.headers.authorization - return response({ relayToken: 'relay-token' }) - } - return response({ cellUrl: 'https://cell.test', assignmentEpoch: 1 }) - } - const peer = new RelayLoadControlPeer(6, peerOptions({ - accessTokenProvider: async () => { - providerCalls++ - return 'refreshed-access-token' - } - }), () => undefined) - - await peer.requestAssignment('asia-east2') - assert.equal(providerCalls, 1) - assert.equal(authorization, 'Bearer refreshed-access-token') - await peer.shutdown() -}) - -test('accepts a forced close only when the responsive splice leg receives 4429', async () => { - const observations = [] - const peer = new RelayLoadControlPeer(7, peerOptions(), (type, detail) => { - observations.push({ type, detail }) - }) - const control = fakeOpenSocket() - const phone = fakeHandshakeSocket() - const data = fakeHandshakeSocket() - let streamStarted - const streamStartedPromise = new Promise((resolve) => { streamStarted = resolve }) - phone._socket = { pause: () => undefined, resume: () => undefined } - control.send = (raw) => { - const message = JSON.parse(raw) - queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify({ - type: 'invite-created', reqId: message.reqId, inviteToken: 'invite-token' - })))) - } - phone.send = (raw) => { - if (typeof raw === 'string' && raw.startsWith('{')) { - queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify({ - type: 'conn-open', relayDeviceId: 'load-device-7-0', - connId: 'connection-7', connTicket: 'connection-ticket' - })))) - } - } - data.send = (raw) => { - if (typeof raw === 'string') { - queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false)) - } else { - streamStarted() - } - } - peer.socket = control - peer.generation = 11 - peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 9 } - control.on('message', (raw) => peer.onMessage(control, raw)) - peer.createClientSocket = () => openOnNextTurn(phone) - peer.createHostDataSocket = () => openOnNextTurn(data) - - await peer.openSplice({ - readerMode: 'wedged', - readerHoldMs: 10_001, - streamBytes: 300 * 1024, - frameBytes: 64 * 1024, - observeReaderPressure: async () => { - await streamStartedPromise - phone.close(1006) - data.close(4429, 'wedged relay link') - }, - readerDelay: async () => undefined - }) - - assert.equal(observations.filter(({ type }) => type === 'spliceWedged').length, 1) - assert.equal(observations.find(({ type }) => type === 'spliceWedged').detail.code, 4429) - await peer.shutdown() -}) - -test('requires one 4429 and rejects unrelated wedged close codes', () => { - assert.equal(relayLoadWedgedCloseAccepted([4429, 4429]), true) - assert.equal(relayLoadWedgedCloseAccepted([1006, 4429]), true) - assert.equal(relayLoadWedgedCloseAccepted([4429, 1006]), false) - assert.equal(relayLoadWedgedCloseAccepted([1006, 1006]), false) - assert.equal(relayLoadWedgedCloseAccepted([1000, 4429]), false) - assert.equal(relayLoadWedgedCloseAccepted([4429]), false) - assert.equal(relayLoadWedgedCloseAccepted([1006, 4429, 4429]), false) -}) - -test('streams reader frames with bounded send-side backpressure', async () => { - const peer = new RelayLoadControlPeer(9, peerOptions(), () => undefined) - let inFlight = 0 - let peakInFlight = 0 - let sentBytes = 0 - const socket = { - bufferedAmount: 0, - send(payload, callback) { - inFlight++ - peakInFlight = Math.max(peakInFlight, inFlight) - sentBytes += payload.byteLength - this.bufferedAmount = payload.byteLength - queueMicrotask(() => { - this.bufferedAmount = 0 - inFlight-- - callback() - }) - } - } - - const result = await peer.sendReaderStream( - socket, - 0, - 1024 * 1024, - 64 * 1024, - async () => undefined - ) - - assert.equal(result.bytes, 1024 * 1024) - assert.equal(sentBytes, 1024 * 1024) - assert.equal(peakInFlight, 1) - await peer.shutdown() -}) - -test('reports a bidirectional splice payload mismatch', async () => { - const observations = [] - const peer = new RelayLoadControlPeer(2, peerOptions(), (type) => observations.push(type)) - const control = fakeOpenSocket() - const phone = fakeHandshakeSocket() - const data = fakeHandshakeSocket() - control.send = (raw) => { - const message = JSON.parse(raw) - queueMicrotask(() => - control.emit( - 'message', - Buffer.from( - JSON.stringify({ type: 'invite-created', reqId: message.reqId, inviteToken: 'token' }) - ) - ) - ) - } - phone.send = (raw) => { - if (typeof raw === 'string' && raw.startsWith('{') && JSON.parse(raw).type === 'relay-auth') { - queueMicrotask(() => - control.emit( - 'message', - Buffer.from( - JSON.stringify({ - type: 'conn-open', - relayDeviceId: 'load-device-2-0', - connId: 'connection-2', - connTicket: 'ticket' - }) - ) - ) - ) - } - } - data.send = (raw) => { - if (typeof raw === 'string') { - queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false)) - } else { - queueMicrotask(() => phone.emit('message', Buffer.alloc(Buffer.from(raw).byteLength), true)) - } - } - peer.socket = control - peer.generation = 4 - peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 2 } - control.on('message', (raw) => peer.onMessage(control, raw)) - peer.createClientSocket = () => openOnNextTurn(phone) - peer.createHostDataSocket = () => openOnNextTurn(data) - - await assert.rejects(peer.openSplice(), /changed host-to-client splice payload/) - assert.equal(observations.includes('spliceFailed'), true) - await peer.shutdown() -}) - -test('shutdown closes both splice legs and waits for the in-flight splice', async () => { - const spliceOpened = deferred() - const observations = [] - const peer = new RelayLoadControlPeer(5, peerOptions(), (type, detail) => { - observations.push({ type, detail }) - if (type === 'spliceOpened') spliceOpened.resolve() - }) - const control = fakeOpenSocket() - const phone = fakeHandshakeSocket() - const data = fakeHandshakeSocket() - control.send = (raw) => { - const message = JSON.parse(raw) - queueMicrotask(() => - control.emit( - 'message', - Buffer.from( - JSON.stringify({ type: 'invite-created', reqId: message.reqId, inviteToken: 'token' }) - ) - ) - ) - } - phone.send = (raw) => { - if (typeof raw === 'string' && raw.startsWith('{')) { - queueMicrotask(() => - control.emit( - 'message', - Buffer.from( - JSON.stringify({ - type: 'conn-open', - relayDeviceId: 'load-device-5-0', - connId: 'connection-5', - connTicket: 'ticket' - }) - ) - ) - ) - } else { - queueMicrotask(() => data.emit('message', Buffer.from(raw), false)) - } - } - data.send = (raw) => { - if (typeof raw === 'string') { - queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false)) - } else { - queueMicrotask(() => phone.emit('message', Buffer.from(raw), true)) - } - } - peer.socket = control - peer.generation = 8 - peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 3 } - control.on('message', (raw) => peer.onMessage(control, raw)) - peer.createClientSocket = () => openOnNextTurn(phone) - peer.createHostDataSocket = () => openOnNextTurn(data) - - const splice = peer.openSplice({ holdMs: 60_000 }) - await spliceOpened.promise - await Promise.all([splice, peer.shutdown()]) - - assert.equal(phone.readyState, phone.CLOSED) - assert.equal(data.readyState, data.CLOSED) - assert.equal(peer.inFlight.size, 0) - assert.equal(observations.at(-1).type, 'shutdown') - assert.equal(observations.at(-1).detail.activeSpliceSockets, 0) -}) - -function trackRefreshTimers(context) { - const schedule = global.setTimeout - const cancel = global.clearTimeout - const timers = new Map() - context.mock.method(global, 'setTimeout', (callback, milliseconds, ...args) => { - const timer = schedule(() => { - timers.delete(timer) - callback(...args) - }, milliseconds) - timers.set(timer, milliseconds) - return timer - }) - context.mock.method(global, 'clearTimeout', (timer) => { - timers.delete(timer) - cancel(timer) - }) - context.after(() => { - for (const timer of timers.keys()) cancel(timer) - }) - return () => [...timers.values()].filter((milliseconds) => milliseconds >= 180_000).length -} - -function reconnectingPeer(context, observations) { - const peer = new RelayLoadControlPeer( - 0, - peerOptions({ - directorOrigin: undefined, - targetOrigin: 'https://cell.test', - accessToken: 'test-access-token' - }), - (type) => observations.push(type) - ) - peer.phase.refreshOffsetMs = 180_000 - peer.phase.refreshIntervalMs = 200_000 - peer.createSocket = () => { - const socket = fakeHandshakeSocket() - socket.send = (raw) => { - const message = JSON.parse(raw) - if (message.type === 'host-hello') { - queueMicrotask(() => socket.message(validChallenge(peer))) - } else if (message.type === 'host-challenge-ack') { - queueMicrotask(() => - socket.message({ - type: 'host-hello-ack', - generation: 1, - controlResumeSecret: 'test-secret' - }) - ) - } - } - return openOnNextTurn(socket) - } - context.after(() => peer.shutdown()) - return peer -} - -for (const outcome of ['success', 'failure']) { - for (const owner of ['current connection', 'replacement connection', 'shutdown']) { - test(`refresh ${outcome} respects its ${owner} ownership`, async (context) => { - const refreshTimers = trackRefreshTimers(context) - const observations = [] - const peer = reconnectingPeer(context, observations) - const pendingResponse = deferred() - const requestStarted = deferred() - let delayResponse = false - context.mock.method(global, 'fetch', async () => { - if (delayResponse) { - requestStarted.resolve() - return await pendingResponse.promise - } - return response({ relayToken: 'test-relay-token' }) - }) - await peer.connect() - assert.equal(refreshTimers(), 1) - clearTimeout(peer.refreshTimer) - delayResponse = true - peer.scheduleRefresh(0) - await requestStarted.promise - delayResponse = false - if (owner !== 'current connection') { - peer.socket.close(1006) - await new Promise((resolve) => setImmediate(resolve)) - await peer.connect() - assert.equal(refreshTimers(), 1) - } - const shutdown = owner === 'shutdown' ? peer.shutdown() : undefined - if (outcome === 'success') pendingResponse.resolve(response({ relayToken: 'fresh-token' })) - else pendingResponse.reject(new Error('token request failed')) - await new Promise((resolve) => setImmediate(resolve)) - const timersAfterCompletion = refreshTimers() - await (shutdown ?? peer.shutdown()) - - assert.equal(timersAfterCompletion, owner === 'shutdown' ? 0 : 1) - assert.equal(refreshTimers(), 0) - assert.equal( - observations.filter((type) => type === 'refresh').length, - owner === 'current connection' && outcome === 'success' ? 1 : 0 - ) - }) - } -} - -test('a twin shares its host keys and connects with a given assignment, not the director', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - const fetched = [] - global.fetch = async (url) => { - fetched.push(String(url)) - throw new Error('the director must not be asked') - } - const primary = new RelayLoadControlPeer(0, peerOptions(), () => undefined) - const twin = new RelayLoadControlPeer(0, peerOptions({ keys: primary.keys }), () => undefined) - assert.equal(twin.relayHostId, primary.relayHostId) - const socket = fakeHandshakeSocket() - let socketCell - twin.createSocket = (assignment) => { - socketCell = assignment.cellUrl - openOnNextTurn(socket) - return socket - } - - const connecting = twin.connect({ cellUrl: 'https://c3.relay.test', assignmentEpoch: 7 }) - await new Promise((resolve) => setTimeout(resolve, 10)) - await twin.shutdown() - await connecting.catch(() => undefined) - - assert.equal(socketCell, 'https://c3.relay.test') - assert.equal(JSON.parse(socket.sent[0]).assignmentEpoch, 7) - assert.equal(fetched.some((url) => url.endsWith('/v1/assign')), false) -}) - -test('declares a reconnect only after an established control', async (context) => { - const originalFetch = global.fetch - context.after(() => { - global.fetch = originalFetch - }) - const bodies = [] - global.fetch = async (_url, { body }) => { - bodies.push(JSON.parse(body)) - return response({ cellUrl: 'https://cell.test', assignmentEpoch: 1 }) - } - const peer = new RelayLoadControlPeer(0, peerOptions(), () => undefined) - const token = await peer.relayToken() - await peer.assignment(token) - peer.generation = 1 - await peer.assignment(token) - assert.equal(bodies[0].reconnect, undefined) - assert.equal(bodies[1].reconnect, true) -}) - -test('a silent control is terminated after the desktop silence limit', (context) => { - context.mock.timers.enable({ apis: ['setInterval', 'Date'] }) - const peer = new RelayLoadControlPeer(0, peerOptions(), () => undefined) - const socket = fakeOpenSocket() - let terminated = 0 - socket.terminate = () => { - terminated++ - socket.emit('close', 1006, Buffer.from('')) - } - peer.watchSilence(socket) - context.mock.timers.tick(60_000) - socket.emit('ping') - context.mock.timers.tick(60_000) - assert.equal(terminated, 0) - context.mock.timers.tick(30_000) - context.mock.timers.tick(60_000) - assert.equal(terminated, 1) -}) diff --git a/cloud/dev/scripts/relay-load-director-capacity-gate.mjs b/cloud/dev/scripts/relay-load-director-capacity-gate.mjs deleted file mode 100644 index a1701d1a17e..00000000000 --- a/cloud/dev/scripts/relay-load-director-capacity-gate.mjs +++ /dev/null @@ -1,147 +0,0 @@ -import { setTimeout as delayDefault } from 'node:timers/promises' - -function integer(value) { - return Number.isSafeInteger(value) && value >= 0 ? value : undefined -} - -export function assertRelayLoadDirectorCapacityToken(config, now = Date.now, timeoutMs = 0) { - if (!config.adminToken || config.adminToken.length > 8_192) { - throw new Error('director capacity identity token is unavailable') - } - const origin = new URL(config.directorOrigin) - if (origin.protocol !== 'https:' || origin.origin !== config.directorOrigin) { - throw new Error('director capacity origin must be canonical HTTPS') - } - let claims - try { - const parts = config.adminToken.split('.') - if (parts.length !== 3) throw new Error('invalid token shape') - claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')) - } catch { - throw new Error('director capacity identity token is invalid') - } - const expectedAudience = new URL('/v1/admin/drain', origin).toString() - const audiences = Array.isArray(claims.aud) ? claims.aud : [claims.aud] - const expiresAt = integer(claims.exp) - if ( - !audiences.includes(expectedAudience) || - typeof claims.email !== 'string' || - claims.email.length === 0 || - claims.email_verified !== true || - expiresAt === undefined || - expiresAt * 1_000 <= now() + timeoutMs - ) { - throw new Error('director capacity identity token is not bound to this proof') - } -} - -function matchingHeartbeat(status, config) { - const capacity = status?.connectionCapacity - const runtime = status?.runtime - const heartbeatAt = integer(runtime?.lastHeartbeatAt) - const matches = - status?.cellId === config.cellId && - status?.admissionState === 'general' && - runtime?.ready === true && - runtime?.heartbeatFresh === true && - capacity?.heartbeatFresh === true && - integer(capacity?.hardCap) === config.hardCap && - integer(capacity?.unobservedBound) === config.unobservedBound && - integer(capacity?.normalAdmissionPause) === config.requiredConnections && - integer(capacity?.observedConnections) === config.requiredConnections && - integer(capacity?.enforcedConnectionUnits) === config.requiredConnections && - integer(capacity?.inFlightConnections) === 0 && - integer(capacity?.reservedConnectionUnits) === 0 && - integer(capacity?.pendingControlReservations) === 0 && - heartbeatAt !== undefined - return matches ? heartbeatAt : undefined -} - -async function cellStatus(fetchImpl, config) { - const response = await fetchImpl(`${config.directorOrigin}/v1/admin/cell-status`, { - method: 'POST', - headers: { - authorization: `Bearer ${config.adminToken}`, - 'content-type': 'application/json' - }, - body: JSON.stringify({ v: 1, cellId: config.cellId }), - signal: AbortSignal.timeout(30_000) - }) - if (response.status === 401 || response.status === 403) { - throw new Error('director capacity identity was rejected') - } - if (!response.ok) { - await response.arrayBuffer().catch(() => undefined) - return undefined - } - const result = await response.json().catch(() => undefined) - if (!result?.status) throw new Error('director capacity status is invalid') - return result.status -} - -export async function waitForRelayLoadDirectorCapacity(config, overrides = {}) { - const fetchImpl = overrides.fetch ?? fetch - const delay = overrides.delay ?? delayDefault - const now = overrides.now ?? Date.now - const timeoutMs = overrides.timeoutMs ?? 120_000 - const pollMs = overrides.pollMs ?? 1_000 - assertRelayLoadDirectorCapacityToken(config, now, timeoutMs) - const deadline = now() + timeoutMs - let baselineHeartbeatAt = config.baselineHeartbeatAt - let previousHeartbeatAt - let matchingSamples = 0 - const requiredSamples = config.requiredSamples ?? 2 - for (;;) { - const status = await cellStatus(fetchImpl, config) - const currentHeartbeatAt = integer(status?.runtime?.lastHeartbeatAt) - if (baselineHeartbeatAt === undefined && currentHeartbeatAt !== undefined) { - baselineHeartbeatAt = currentHeartbeatAt - } - const heartbeatAt = status ? matchingHeartbeat(status, config) : undefined - if (heartbeatAt !== undefined && heartbeatAt > baselineHeartbeatAt) { - if (previousHeartbeatAt === undefined || heartbeatAt > previousHeartbeatAt) { - previousHeartbeatAt = heartbeatAt - matchingSamples++ - if (matchingSamples === requiredSamples) return { heartbeatAt } - } - } else { - previousHeartbeatAt = undefined - matchingSamples = 0 - } - if (now() >= deadline) throw new Error('director capacity did not converge after recovery') - await delay(pollMs) - } -} - -function matchingRequestUnits(status, config) { - return ( - status?.cellId === config.cellId && - status?.admissionState === 'general' && - status?.capacityRequests === config.capacityRequests && - status?.reservedRequests === config.expectedRequestUnits && - status?.activityRequestUnits === config.expectedRequestUnits && - status?.activityLeases === config.expectedActivityLeases && - status?.runtime?.observedRequests === config.expectedRequestUnits && - status?.runtime?.ready === true && - status?.runtime?.heartbeatFresh === true - ) -} - -export async function waitForRelayLoadRequestUnits(config, overrides = {}) { - const fetchImpl = overrides.fetch ?? fetch - const delay = overrides.delay ?? delayDefault - const now = overrides.now ?? Date.now - const timeoutMs = overrides.timeoutMs ?? config.timeoutMs ?? 120_000 - const pollMs = overrides.pollMs ?? 1_000 - const requiredSamples = overrides.requiredSamples ?? 2 - assertRelayLoadDirectorCapacityToken(config, now, timeoutMs) - const deadline = now() + timeoutMs - let matches = 0 - for (;;) { - const status = await cellStatus(fetchImpl, config) - matches = matchingRequestUnits(status, config) ? matches + 1 : 0 - if (matches === requiredSamples) return - if (now() >= deadline) throw new Error('Relay request-unit accounting did not converge') - await delay(pollMs) - } -} diff --git a/cloud/dev/scripts/relay-load-director-capacity-gate.test.mjs b/cloud/dev/scripts/relay-load-director-capacity-gate.test.mjs deleted file mode 100644 index fe0a2813b80..00000000000 --- a/cloud/dev/scripts/relay-load-director-capacity-gate.test.mjs +++ /dev/null @@ -1,260 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { - assertRelayLoadDirectorCapacityToken, - waitForRelayLoadDirectorCapacity, - waitForRelayLoadRequestUnits -} from './relay-load-director-capacity-gate.mjs' - -function token(claims) { - const encode = (value) => Buffer.from(JSON.stringify(value)).toString('base64url') - return `${encode({ alg: 'none' })}.${encode(claims)}.signature` -} - -const config = { - directorOrigin: 'https://relay-staging.example.com', - adminToken: token({ - aud: 'https://relay-staging.example.com/v1/admin/drain', - email: 'capacity@example.com', - email_verified: true, - exp: 4_000_000_000 - }), - cellId: 'staging-gce-c3', - hardCap: 1_000, - unobservedBound: 60, - requiredConnections: 840 -} - -function status(lastHeartbeatAt, overrides = {}) { - return { - cellId: config.cellId, - admissionState: 'general', - runtime: { ready: true, heartbeatFresh: true, lastHeartbeatAt, observedRequests: 0 }, - connectionCapacity: { - hardCap: 1_000, - unobservedBound: 60, - normalAdmissionPause: 840, - observedConnections: 840, - enforcedConnectionUnits: 840, - inFlightConnections: 0, - reservedConnectionUnits: 0, - pendingControlReservations: 0, - heartbeatFresh: true, - ...overrides - } - } -} - -function response(value, responseStatus = 200) { - return { - ok: responseStatus >= 200 && responseStatus < 300, - status: responseStatus, - json: async () => value, - arrayBuffer: async () => new ArrayBuffer(0) - } -} - -test('requires two advancing exact director capacity heartbeats', async () => { - const heartbeats = [101, 116, 131] - let calls = 0 - const result = await waitForRelayLoadDirectorCapacity(config, { - fetch: async () => response({ status: status(heartbeats[calls++]) }), - delay: async () => undefined - }) - assert.equal(calls, 3) - assert.deepEqual(result, { heartbeatAt: 131 }) -}) - -test('resets after a newer heartbeat undercounts recovered controls', async () => { - const samples = [ - status(101), - status(116), - status(131, { observedConnections: 839 }), - status(146), - status(161) - ] - let calls = 0 - await waitForRelayLoadDirectorCapacity(config, { - fetch: async () => response({ status: samples[calls++] }), - delay: async () => undefined - }) - assert.equal(calls, 5) -}) - -test('fails closed when exact advancing telemetry never converges', async () => { - let elapsed = 0 - await assert.rejects( - waitForRelayLoadDirectorCapacity(config, { - fetch: async () => response({ status: status(101) }), - delay: async (milliseconds) => { - elapsed += milliseconds - }, - now: () => elapsed, - timeoutMs: 2_000, - pollMs: 1_000 - }), - /did not converge/ - ) -}) - -test('rejects an unauthorized capacity identity without retrying', async () => { - let calls = 0 - await assert.rejects( - waitForRelayLoadDirectorCapacity(config, { - fetch: async () => { - calls++ - return response({}, 401) - }, - delay: async () => undefined - }), - /identity was rejected/ - ) - assert.equal(calls, 1) -}) - -test('binds the admin token to the canonical director audience', async () => { - await assert.rejects( - waitForRelayLoadDirectorCapacity( - { - ...config, - directorOrigin: 'https://relay-staging.example.com/path' - }, - { fetch: async () => response({ status: status(101) }), now: () => 0 } - ), - /canonical HTTPS/ - ) - await assert.rejects( - waitForRelayLoadDirectorCapacity( - { - ...config, - adminToken: token({ - aud: 'https://other.example.com/v1/admin/drain', - email: 'capacity@example.com', - email_verified: true, - exp: 4_000_000_000 - }) - }, - { fetch: async () => response({ status: status(101) }), now: () => 0 } - ), - /not bound/ - ) -}) - -test('rejects a missing or malformed admin token during startup preflight', () => { - assert.throws( - () => assertRelayLoadDirectorCapacityToken({ ...config, adminToken: undefined }, () => 0), - /unavailable/ - ) - assert.throws( - () => assertRelayLoadDirectorCapacityToken({ ...config, adminToken: 'not-a-jwt' }, () => 0), - /invalid/ - ) - assert.throws( - () => - assertRelayLoadDirectorCapacityToken( - { - ...config, - adminToken: token({ - aud: 'https://relay-staging.example.com/v1/admin/drain', - exp: 4_000_000_000 - }) - }, - () => 0 - ), - /not bound/ - ) -}) - -test('supports one newer exact post-probe heartbeat', async () => { - const heartbeats = [146, 161] - let calls = 0 - const result = await waitForRelayLoadDirectorCapacity( - { ...config, requiredSamples: 1 }, - { - fetch: async () => response({ status: status(heartbeats[calls++]) }), - delay: async () => undefined, - now: () => 0 - } - ) - assert.equal(calls, 2) - assert.deepEqual(result, { heartbeatAt: 161 }) -}) - -test('requires consecutive exact request-unit accounting samples', async () => { - const requestConfig = { - ...config, - capacityRequests: 6_000, - expectedRequestUnits: 6_000, - expectedActivityLeases: 6_000 - } - const samples = [5_999, 6_000, 6_000] - let calls = 0 - await waitForRelayLoadRequestUnits(requestConfig, { - fetch: async () => response({ - status: { - ...status(100), - runtime: { - ...status(100).runtime, - observedRequests: samples[calls] - }, - capacityRequests: 6_000, - reservedRequests: samples[calls], - activityRequestUnits: samples[calls], - activityLeases: samples[calls++] - } - }), - delay: async () => undefined, - now: () => 0 - }) - assert.equal(calls, 3) -}) - -test('requires the cell runtime to observe every request unit', async () => { - let elapsed = 0 - await assert.rejects(waitForRelayLoadRequestUnits({ - ...config, - capacityRequests: 6_000, - expectedRequestUnits: 6_000, - expectedActivityLeases: 6_000, - timeoutMs: 1_000 - }, { - fetch: async () => response({ - status: { - ...status(100), - runtime: { ...status(100).runtime, observedRequests: 5_999 }, - capacityRequests: 6_000, - reservedRequests: 6_000, - activityRequestUnits: 6_000, - activityLeases: 6_000 - } - }), - delay: async (milliseconds) => { elapsed += milliseconds }, - now: () => elapsed, - pollMs: 1_000 - }), /did not converge/) -}) - -test('fails closed when request-unit accounting does not clean up', async () => { - let elapsed = 0 - await assert.rejects(waitForRelayLoadRequestUnits({ - ...config, - capacityRequests: 6_000, - expectedRequestUnits: 0, - expectedActivityLeases: 0, - timeoutMs: 2_000 - }, { - fetch: async () => response({ - status: { - ...status(100), - runtime: { ...status(100).runtime, observedRequests: 1 }, - capacityRequests: 6_000, - reservedRequests: 1, - activityRequestUnits: 1, - activityLeases: 1 - } - }), - delay: async (milliseconds) => { elapsed += milliseconds }, - now: () => elapsed, - pollMs: 1_000 - }), /did not converge/) -}) diff --git a/cloud/dev/scripts/relay-load-duplicate-assign.mjs b/cloud/dev/scripts/relay-load-duplicate-assign.mjs deleted file mode 100644 index 4aceec865bf..00000000000 --- a/cloud/dev/scripts/relay-load-duplicate-assign.mjs +++ /dev/null @@ -1,66 +0,0 @@ -// Step 5's equal-epoch case: two fresh assigns for one host that reach two director instances -// can each book a cell at the same epoch. The host connects with the first answer, and a twin -// with the same keys holds the other seat half-open, as a desktop that joined both would. -export async function proveRelayLoadDuplicateAssign({ pairs, holdMs, delay, failureReason }) { - const result = { - probes: pairs.length, - bothAnswered: 0, - splitCells: 0, - sameEpochSplits: 0, - joinedBoth: 0, - refusalsByReason: {}, - connectFailuresByReason: {}, - // Not secrets: synthetic load hosts, listed so the ledger rows can be checked afterwards. - splits: [] - } - const count = (bucket, error) => { - const reason = failureReason(error) - bucket[reason] = (bucket[reason] ?? 0) + 1 - } - const twins = [] - await Promise.all( - pairs.map(async ({ primary, twin }) => { - const answers = await Promise.allSettled([ - primary.requestAssignment(), - primary.requestAssignment() - ]) - for (const answer of answers) { - if (answer.status === 'rejected') count(result.refusalsByReason, answer.reason) - } - const [first, second] = answers - .filter((answer) => answer.status === 'fulfilled') - .map((answer) => answer.value) - if (!first) return - if (second) result.bothAnswered++ - const split = second !== undefined && second.cellUrl !== first.cellUrl - if (split) { - result.splitCells++ - if (second.assignmentEpoch === first.assignmentEpoch) result.sameEpochSplits++ - result.splits.push({ - relayHostId: primary.relayHostId, - first: { cellUrl: first.cellUrl, assignmentEpoch: first.assignmentEpoch }, - second: { cellUrl: second.cellUrl, assignmentEpoch: second.assignmentEpoch } - }) - } - try { - await primary.connect(first) - } catch (error) { - count(result.connectFailuresByReason, error) - return - } - if (!split) return - try { - await twin.connect(second) - result.joinedBoth++ - twins.push(twin) - } catch (error) { - count(result.connectFailuresByReason, error) - } - }) - ) - if (twins.length > 0) { - await delay(holdMs) - await Promise.all(twins.map((twin) => twin.shutdown())) - } - return result -} diff --git a/cloud/dev/scripts/relay-load-duplicate-assign.test.mjs b/cloud/dev/scripts/relay-load-duplicate-assign.test.mjs deleted file mode 100644 index aaf15d5d4aa..00000000000 --- a/cloud/dev/scripts/relay-load-duplicate-assign.test.mjs +++ /dev/null @@ -1,127 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { proveRelayLoadDuplicateAssign } from './relay-load-duplicate-assign.mjs' - -const c2 = 'https://c2.relay-staging.onorca.dev' -const c3 = 'https://c3.relay-staging.onorca.dev' - -function pair(relayHostId, answers, { twinFails = false } = {}) { - const queue = [...answers] - const log = { primary: [], twin: [], twinShutdowns: 0 } - const primary = { - relayHostId, - requestAssignment: async () => { - const answer = queue.shift() - if (answer instanceof Error) throw answer - return answer - }, - connect: async (assignment) => { - log.primary.push(assignment) - } - } - const twin = { - connect: async (assignment) => { - if (twinFails) throw new Error('control closed: 4409 wrong cell') - log.twin.push(assignment) - }, - shutdown: async () => { - log.twinShutdowns++ - } - } - return { pair: { primary, twin }, log } -} - -const failureReason = (error) => (error.message.startsWith('control closed: 4409') ? 'control_close_4409' : 'other') - -test('counts same-epoch splits, joins the second cell with a twin, then lets it go', async () => { - const split = pair('aaaaaaaaaaaaaaaa', [ - { cellUrl: c2, assignmentEpoch: 5 }, - { cellUrl: c3, assignmentEpoch: 5 } - ]) - const same = pair('bbbbbbbbbbbbbbbb', [ - { cellUrl: c2, assignmentEpoch: 3 }, - { cellUrl: c2, assignmentEpoch: 3 } - ]) - const delays = [] - const result = await proveRelayLoadDuplicateAssign({ - pairs: [split.pair, same.pair], - holdMs: 20_000, - delay: async (ms) => { - delays.push(ms) - }, - failureReason - }) - - assert.equal(result.probes, 2) - assert.equal(result.bothAnswered, 2) - assert.equal(result.splitCells, 1) - assert.equal(result.sameEpochSplits, 1) - assert.equal(result.joinedBoth, 1) - assert.deepEqual(result.splits, [ - { - relayHostId: 'aaaaaaaaaaaaaaaa', - first: { cellUrl: c2, assignmentEpoch: 5 }, - second: { cellUrl: c3, assignmentEpoch: 5 } - } - ]) - assert.deepEqual(split.log.primary, [{ cellUrl: c2, assignmentEpoch: 5 }]) - assert.deepEqual(split.log.twin, [{ cellUrl: c3, assignmentEpoch: 5 }]) - assert.equal(split.log.twinShutdowns, 1) - assert.deepEqual(same.log.twin, []) - assert.deepEqual(delays, [20_000]) -}) - -test('a split at different epochs is not a same-epoch split', async () => { - const split = pair('cccccccccccccccc', [ - { cellUrl: c2, assignmentEpoch: 5 }, - { cellUrl: c3, assignmentEpoch: 6 } - ]) - const result = await proveRelayLoadDuplicateAssign({ - pairs: [split.pair], - holdMs: 1_000, - delay: async () => undefined, - failureReason - }) - assert.equal(result.splitCells, 1) - assert.equal(result.sameEpochSplits, 0) -}) - -test('a refused second assign still connects the host once and holds nothing', async () => { - const refused = pair('dddddddddddddddd', [ - { cellUrl: c2, assignmentEpoch: 2 }, - new Error('relay assignment failed: 503') - ]) - const delays = [] - const result = await proveRelayLoadDuplicateAssign({ - pairs: [refused.pair], - holdMs: 20_000, - delay: async (ms) => { - delays.push(ms) - }, - failureReason - }) - assert.equal(result.bothAnswered, 0) - assert.deepEqual(result.refusalsByReason, { other: 1 }) - assert.deepEqual(refused.log.primary, [{ cellUrl: c2, assignmentEpoch: 2 }]) - assert.deepEqual(delays, []) -}) - -test('a twin the second cell refuses is counted, not joined', async () => { - const split = pair( - 'eeeeeeeeeeeeeeee', - [ - { cellUrl: c2, assignmentEpoch: 5 }, - { cellUrl: c3, assignmentEpoch: 5 } - ], - { twinFails: true } - ) - const result = await proveRelayLoadDuplicateAssign({ - pairs: [split.pair], - holdMs: 20_000, - delay: async () => undefined, - failureReason - }) - assert.equal(result.joinedBoth, 0) - assert.deepEqual(result.connectFailuresByReason, { control_close_4409: 1 }) - assert.equal(split.log.twinShutdowns, 0) -}) diff --git a/cloud/dev/scripts/relay-load-model.mjs b/cloud/dev/scripts/relay-load-model.mjs deleted file mode 100644 index 77de00422c6..00000000000 --- a/cloud/dev/scripts/relay-load-model.mjs +++ /dev/null @@ -1,76 +0,0 @@ -const HEARTBEAT_INTERVAL_MS = 15_000 -const REFRESH_MIN_MS = 180_000 -const REFRESH_MAX_MS = 240_000 - -function mix32(value) { - let mixed = value >>> 0 - mixed = Math.imul(mixed ^ (mixed >>> 16), 0x21f0aaad) - mixed = Math.imul(mixed ^ (mixed >>> 15), 0x735a2d97) - return (mixed ^ (mixed >>> 15)) >>> 0 -} - -function fraction(seed, index, stream) { - return mix32(seed ^ Math.imul(index + 1, 0x9e3779b1) ^ stream) / 0x1_0000_0000 -} - -export function controlPhase(controlIndex, seed = 0x4f524341) { - const refreshIntervalMs = Math.round( - REFRESH_MIN_MS + fraction(seed, controlIndex, 2) * (REFRESH_MAX_MS - REFRESH_MIN_MS) - ) - return { - heartbeatOffsetMs: Math.floor(fraction(seed, controlIndex, 1) * HEARTBEAT_INTERVAL_MS), - refreshIntervalMs, - refreshOffsetMs: Math.floor(fraction(seed, controlIndex, 3) * refreshIntervalMs), - reconnectJitterMs: Math.floor(fraction(seed, controlIndex, 4) * 30_000) - } -} - -export function modeledRelayLoad(controlCount, durationMs = 15 * 60_000, seed) { - if (!Number.isInteger(controlCount) || controlCount < 1) throw new Error('controlCount must be positive') - const bins = Array.from({ length: Math.ceil(durationMs / 1000) }, () => ({ pings: 0, refreshes: 0 })) - for (let controlIndex = 0; controlIndex < controlCount; controlIndex++) { - const phase = controlPhase(controlIndex, seed) - for (let at = phase.heartbeatOffsetMs; at < durationMs; at += HEARTBEAT_INTERVAL_MS) { - bins[Math.floor(at / 1000)].pings++ - } - for (let at = phase.refreshOffsetMs; at < durationMs; at += phase.refreshIntervalMs) { - bins[Math.floor(at / 1000)].refreshes++ - } - } - const totals = bins.reduce( - (result, bin) => ({ - pings: result.pings + bin.pings, - refreshes: result.refreshes + bin.refreshes - }), - { pings: 0, refreshes: 0 } - ) - const durationSeconds = durationMs / 1000 - return { - controlCount, - durationMs, - expectedPingRate: controlCount / (HEARTBEAT_INTERVAL_MS / 1000), - expectedRefreshRate: controlCount / ((REFRESH_MIN_MS + REFRESH_MAX_MS) / 2 / 1000), - observedPingRate: totals.pings / durationSeconds, - observedRefreshRate: totals.refreshes / durationSeconds, - maxPingBurst: Math.max(...bins.map(({ pings }) => pings)), - maxRefreshBurst: Math.max(...bins.map(({ refreshes }) => refreshes)) - } -} - -export function assertSpreadModel(model) { - const pingTolerance = model.expectedPingRate * 0.03 + 1 - const refreshTolerance = model.expectedRefreshRate * 0.08 + 1 - if (Math.abs(model.observedPingRate - model.expectedPingRate) > pingTolerance) { - throw new Error('modeled heartbeat rate diverged from the 15-second contract') - } - if (Math.abs(model.observedRefreshRate - model.expectedRefreshRate) > refreshTolerance) { - throw new Error('modeled token refresh rate diverged from the 180-240 second contract') - } - if (model.maxPingBurst > model.expectedPingRate * 1.3 + 5) { - throw new Error('heartbeat phase spreading produced a reconnect cliff') - } - // One-second bins have Poisson-sized tails even with uniform phase spreading. - if (model.maxRefreshBurst > model.expectedRefreshRate * 1.75 + 5) { - throw new Error('refresh phase spreading produced an auth herd') - } -} diff --git a/cloud/dev/scripts/relay-load-model.test.mjs b/cloud/dev/scripts/relay-load-model.test.mjs deleted file mode 100644 index e38548efe3b..00000000000 --- a/cloud/dev/scripts/relay-load-model.test.mjs +++ /dev/null @@ -1,25 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { assertSpreadModel, controlPhase, modeledRelayLoad } from './relay-load-model.mjs' - -test('phases are deterministic, bounded, and separated by stream', () => { - assert.deepEqual(controlPhase(42), controlPhase(42)) - assert.notDeepEqual(controlPhase(42), controlPhase(43)) - const phase = controlPhase(42) - assert.ok(phase.heartbeatOffsetMs >= 0 && phase.heartbeatOffsetMs < 15_000) - assert.ok(phase.refreshIntervalMs >= 180_000 && phase.refreshIntervalMs <= 240_000) - assert.ok(phase.refreshOffsetMs >= 0 && phase.refreshOffsetMs < phase.refreshIntervalMs) - assert.ok(phase.reconnectJitterMs >= 0 && phase.reconnectJitterMs < 30_000) -}) - -for (const [controls, expectedPings, expectedRefreshes] of [ - [4_000, 267, 19], - [10_000, 667, 48] -]) { - test(`${controls} modeled controls spread heartbeat and token refresh load`, () => { - const model = modeledRelayLoad(controls) - assert.equal(Math.round(model.expectedPingRate), expectedPings) - assert.equal(Math.round(model.expectedRefreshRate), expectedRefreshes) - assert.doesNotThrow(() => assertSpreadModel(model)) - }) -} diff --git a/cloud/dev/scripts/relay-load-phase-barrier.mjs b/cloud/dev/scripts/relay-load-phase-barrier.mjs deleted file mode 100644 index 03e0e800074..00000000000 --- a/cloud/dev/scripts/relay-load-phase-barrier.mjs +++ /dev/null @@ -1,37 +0,0 @@ -import { access, mkdir, open } from 'node:fs/promises' -import { join } from 'node:path' -import { setTimeout as delayDefault } from 'node:timers/promises' - -export async function waitForRelayLoadPhaseBarrier(config, overrides = {}) { - const delay = overrides.delay ?? delayDefault - const now = overrides.now ?? Date.now - const timeoutMs = overrides.timeoutMs ?? config.timeoutMs - if ( - typeof config.directory !== 'string' || config.directory.length === 0 || - !Number.isSafeInteger(config.shardCount) || config.shardCount < 2 || - !Number.isSafeInteger(config.shardIndex) || config.shardIndex < 0 || - config.shardIndex >= config.shardCount || - !Number.isSafeInteger(timeoutMs) || timeoutMs < 1 - ) throw new Error('invalid Relay load phase barrier') - - await mkdir(config.directory, { recursive: true }) - const marker = join(config.directory, `${config.shardIndex}.ready`) - const handle = await open(marker, 'wx') - await handle.close() - const deadline = now() + timeoutMs - for (;;) { - const ready = await Promise.all( - Array.from({ length: config.shardCount }, async (_, index) => { - try { - await access(join(config.directory, `${index}.ready`)) - return true - } catch { - return false - } - }) - ) - if (ready.every(Boolean)) return - if (now() >= deadline) throw new Error('Relay load phase barrier timed out') - await delay(100) - } -} diff --git a/cloud/dev/scripts/relay-load-phase-barrier.test.mjs b/cloud/dev/scripts/relay-load-phase-barrier.test.mjs deleted file mode 100644 index 2fc16d3650a..00000000000 --- a/cloud/dev/scripts/relay-load-phase-barrier.test.mjs +++ /dev/null @@ -1,65 +0,0 @@ -import assert from 'node:assert/strict' -import { mkdtemp, readFile, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import test from 'node:test' -import { waitForRelayLoadPhaseBarrier } from './relay-load-phase-barrier.mjs' - -const loadHarness = await readFile(new URL('./load-relay-controls.mjs', import.meta.url), 'utf8') - -test('releases every shard only after all readiness markers exist', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-load-barrier-')) - try { - let firstResolved = false - const first = waitForRelayLoadPhaseBarrier({ - directory, shardCount: 2, shardIndex: 0, timeoutMs: 1_000 - }).then(() => { firstResolved = true }) - await new Promise((resolve) => setTimeout(resolve, 20)) - assert.equal(firstResolved, false) - await Promise.all([ - first, - waitForRelayLoadPhaseBarrier({ - directory, shardCount: 2, shardIndex: 1, timeoutMs: 1_000 - }) - ]) - assert.equal(firstResolved, true) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -test('fails closed on a duplicate shard or incomplete barrier', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-load-barrier-')) - try { - const nowValues = [0, 2] - await assert.rejects( - waitForRelayLoadPhaseBarrier( - { directory, shardCount: 2, shardIndex: 0, timeoutMs: 1 }, - { now: () => nowValues.shift() ?? 2, delay: async () => undefined } - ), - /timed out/ - ) - await assert.rejects( - waitForRelayLoadPhaseBarrier({ - directory, shardCount: 2, shardIndex: 0, timeoutMs: 1 - }), - /EEXIST/ - ) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -test('synchronizes splice ramps after every shard finishes reader baselines', () => { - assert.match( - loadHarness, - /createRelayLoadReaderEvidence[\s\S]*?phaseBarrierDir\}-splices[\s\S]*?splicePromises/ - ) -}) - -test('budgets both shard barriers and the splice ramp in token lifetime', () => { - assert.match( - loadHarness, - /phaseBarrierDir \? 2 \* config\.phaseBarrierTimeoutMs : 0[\s\S]*?config\.spliceRampMs/ - ) -}) diff --git a/cloud/dev/scripts/relay-load-placement-boundary.mjs b/cloud/dev/scripts/relay-load-placement-boundary.mjs deleted file mode 100644 index 84f19b1dc80..00000000000 --- a/cloud/dev/scripts/relay-load-placement-boundary.mjs +++ /dev/null @@ -1,29 +0,0 @@ -export async function proveRelayLoadPlacementBoundary({ peer, failureReason }) { - let connected = false - try { - await peer.connect() - connected = true - } catch (error) { - const reason = failureReason(error) - if (reason !== 'assignment_capacity_exhausted') { - throw new Error(`placement overflow was not rejected: ${reason}`) - } - return reason - } finally { - await peer.shutdown() - } - if (connected) throw new Error('placement overflow unexpectedly connected') -} - -export async function proveRelayLoadRegionalFallback({ peer, blockedOrigin }) { - try { - await peer.connect() - const assignedOrigin = peer.assignedCellUrl() - if (!assignedOrigin || assignedOrigin === blockedOrigin) { - throw new Error('regional fallback did not leave the full preferred cell') - } - return true - } finally { - await peer.shutdown() - } -} diff --git a/cloud/dev/scripts/relay-load-placement-boundary.test.mjs b/cloud/dev/scripts/relay-load-placement-boundary.test.mjs deleted file mode 100644 index 89dd338816f..00000000000 --- a/cloud/dev/scripts/relay-load-placement-boundary.test.mjs +++ /dev/null @@ -1,53 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { - proveRelayLoadPlacementBoundary, - proveRelayLoadRegionalFallback -} from './relay-load-placement-boundary.mjs' - -function peer(connect) { - return { connect, shutdown: async () => undefined } -} - -test('requires the next fresh placement to receive HTTP 503', async () => { - assert.equal( - await proveRelayLoadPlacementBoundary({ - peer: peer(async () => { - throw new Error('relay assignment failed: 503 relay_connection_headroom_exhausted') - }), - failureReason: (error) => - error.message === 'relay assignment failed: 503 relay_connection_headroom_exhausted' - ? 'assignment_capacity_exhausted' - : 'unknown' - }), - 'assignment_capacity_exhausted' - ) - await assert.rejects( - proveRelayLoadPlacementBoundary({ - peer: peer(async () => undefined), - failureReason: () => 'unknown' - }), - /placement overflow unexpectedly connected/ - ) -}) - -test('requires a preferred-region fallback to leave the full cell', async () => { - let shutdowns = 0 - assert.equal(await proveRelayLoadRegionalFallback({ - peer: { - connect: async () => undefined, - assignedCellUrl: () => 'https://c3.relay-staging.onorca.dev', - shutdown: async () => { shutdowns++ } - }, - blockedOrigin: 'https://c4.relay-staging.onorca.dev' - }), true) - assert.equal(shutdowns, 1) - await assert.rejects(proveRelayLoadRegionalFallback({ - peer: { - connect: async () => undefined, - assignedCellUrl: () => 'https://c4.relay-staging.onorca.dev', - shutdown: async () => undefined - }, - blockedOrigin: 'https://c4.relay-staging.onorca.dev' - }), /did not leave/) -}) diff --git a/cloud/dev/scripts/relay-load-profile.mjs b/cloud/dev/scripts/relay-load-profile.mjs deleted file mode 100644 index 7d01a0501d9..00000000000 --- a/cloud/dev/scripts/relay-load-profile.mjs +++ /dev/null @@ -1,440 +0,0 @@ -export const RELAY_CONTROL_LEASE_HORIZON_SECONDS = 105 -export const RELAY_LOAD_SPLICE_HIGH_WATER_BYTES = 256 * 1024 -export const RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS = 10_000 -export const RELAY_LOAD_MAX_AGGREGATE_READER_SPLICES = 16 -export const RELAY_LOAD_MAX_AGGREGATE_READER_BYTES = 64 * 1024 * 1024 - -const DEFAULT_SLOW_READER_STREAM_BYTES = 1024 * 1024 -const DEFAULT_WEDGED_READER_STREAM_BYTES = 8 * 1024 * 1024 -const DEFAULT_READER_FRAME_BYTES = 64 * 1024 - -export function parseRelayLoadArguments(argv) { - const values = new Map() - const flags = new Set() - for (let index = 0; index < argv.length; index++) { - const argument = argv[index] - if (argument === '--') continue - if ( - [ - '--allow-partial', - '--allow-planned-transition-retries', - '--skip-rebind-overflow-check' - ].includes(argument) - ) { - flags.add(argument) - continue - } - if (!argument.startsWith('--') || index + 1 >= argv.length) { - throw new Error(`invalid argument: ${argument}`) - } - values.set(argument, argv[++index]) - } - const integer = (name, fallback) => { - const parsed = Number(values.get(name) ?? fallback) - if (!Number.isSafeInteger(parsed) || parsed < 0) { - throw new Error(`${name} must be a nonnegative integer`) - } - return parsed - } - const targetOrigin = values.get('--target-origin')?.replace(/\/$/, '') - const directorOrigin = values.get('--director-origin')?.replace(/\/$/, '') - const authOrigin = values.get('--auth-origin')?.replace(/\/$/, '') - if ((!targetOrigin && !directorOrigin) || (targetOrigin && directorOrigin) || !authOrigin) { - throw new Error('provide --auth-origin and exactly one target or director origin') - } - const controls = integer('--controls', 100) - const maxRampConnectionFailures = integer('--max-ramp-connection-failures', 0) - const maxUnexpectedCloses = integer('--max-unexpected-closes', 0) - const rebindProbes = integer('--rebind-probes', 0) - const placementOverflowProbes = integer('--placement-overflow-probes', 0) - const regionalFallbackProbes = integer('--regional-fallback-probes', 0) - const regionBehaviorProbes = integer('--region-behavior-probes', 0) - const duplicateAssignProbes = integer('--duplicate-assign-probes', 0) - const duplicateAssignHoldMs = integer('--duplicate-assign-hold-ms', 20_000) - const requestUnitInvites = integer('--request-unit-invites', 0) - const requestUnitInvitesPerSecond = integer('--request-unit-invites-per-second', 0) - const requestUnitPrincipalCount = integer('--request-unit-principals', 0) - const relayAsiaLoadPrincipalCount = integer('--relay-asia-load-principals', 0) - const requestUnitOverflowProbes = integer('--request-unit-overflow-probes', 0) - const requestUnitCapacity = values.has('--request-unit-capacity') - ? integer('--request-unit-capacity', 0) - : undefined - const requestUnitCleanupTimeoutMs = integer( - '--request-unit-cleanup-timeout-seconds', - 0 - ) * 1000 - const rebindHoldMs = integer('--rebind-hold-ms', 4_000) - const rebindDelayMs = integer('--rebind-delay-seconds', 0) * 1000 - const capacityCellId = values.get('--capacity-cell-id') - const capacityCellOrigin = values.get('--capacity-cell-origin')?.replace(/\/$/, '') - const capacityHardCap = values.has('--capacity-hard-cap') - ? integer('--capacity-hard-cap', 0) - : undefined - const capacityUnobservedBound = values.has('--capacity-unobserved-bound') - ? integer('--capacity-unobserved-bound', 0) - : undefined - const shardCount = integer('--shard-count', 1) - const shardIndex = integer('--shard-index', 0) - const durationMs = integer('--duration-seconds', 900) * 1000 - const spliceHoldMs = integer( - '--splice-hold-seconds', - values.get('--duration-seconds') ?? 900 - ) * 1000 - const splices = integer('--splices', 0) - const resumePhones = integer('--resume-phones', 0) - const resumeIntervalMs = integer('--resume-interval-seconds', 20) * 1000 - const resumeHoldMs = integer('--resume-hold-seconds', 10) * 1000 - const spliceRampMs = integer('--splice-ramp-seconds', 0) * 1000 - const slowReaderSplices = integer('--slow-reader-splices', 0) - const wedgedReaderSplices = integer('--wedged-reader-splices', 0) - const splicePayloadBytes = integer('--splice-payload-bytes', 1_024) - const slowReaderStreamBytes = integer( - '--slow-reader-stream-bytes', - DEFAULT_SLOW_READER_STREAM_BYTES - ) - const wedgedReaderStreamBytes = integer( - '--wedged-reader-stream-bytes', - DEFAULT_WEDGED_READER_STREAM_BYTES - ) - const readerFrameBytes = integer('--reader-frame-bytes', DEFAULT_READER_FRAME_BYTES) - const slowReaderHoldMs = integer('--slow-reader-hold-ms', 2_000) - const wedgedReaderHoldMs = integer('--wedged-reader-hold-ms', 12_000) - const maxGeneratorRssGrowthMiB = integer('--max-generator-rss-growth-mib', 512) - const requiredLeaseHorizons = integer('--required-lease-horizons', 0) - const aggregateControls = values.has('--aggregate-controls') - ? integer('--aggregate-controls', 0) - : controls - const aggregateSplices = values.has('--aggregate-splices') - ? integer('--aggregate-splices', 0) - : splices - const aggregateRequestUnitInvites = values.has('--aggregate-request-unit-invites') - ? integer('--aggregate-request-unit-invites', 0) - : requestUnitInvites - const phaseBarrierDir = values.get('--phase-barrier-dir') - const phaseBarrierTimeoutMs = integer('--phase-barrier-timeout-seconds', 180) * 1000 - if (controls < 1 || controls > 10_000) throw new Error('--controls must be between 1 and 10000') - if (rebindProbes > controls) throw new Error('--rebind-probes cannot exceed --controls') - if (splices > controls) throw new Error('--splices cannot exceed --controls') - if (splices + resumePhones > controls) { - throw new Error('--splices plus --resume-phones cannot exceed --controls') - } - if (resumePhones > 0 && resumeIntervalMs < 1_000) { - throw new Error('--resume-interval-seconds must be at least 1') - } - if (shardCount > 1 && capacityHardCap !== undefined) { - if (!values.has('--aggregate-controls') || !values.has('--aggregate-splices')) { - throw new Error('capacity-bound sharding requires explicit aggregate controls and splices') - } - if (aggregateControls !== controls * shardCount || aggregateSplices !== splices * shardCount) { - throw new Error('aggregate controls and splices must match every equal-sized shard') - } - } else if (aggregateControls !== controls || aggregateSplices !== splices) { - throw new Error('aggregate controls and splices require matching sharded local counts') - } - if ( - capacityHardCap !== undefined && - aggregateControls + 2 * aggregateSplices > capacityHardCap - 100 - ) { - throw new Error('controls plus splice connection units exceed ordinary cell admission') - } - if (slowReaderSplices + wedgedReaderSplices > splices) { - throw new Error('reader splice counts cannot exceed --splices') - } - if (splices > 0 && (spliceHoldMs < 1_000 || spliceHoldMs > durationMs)) { - throw new Error('--splice-hold-seconds must be between 1 and the steady duration') - } - if (slowReaderSplices + wedgedReaderSplices > 0 && !directorOrigin) { - throw new Error('reader evidence requires --director-origin') - } - if (splicePayloadBytes < 1 || splicePayloadBytes > 1_048_576) { - throw new Error('--splice-payload-bytes must be between 1 and 1048576') - } - if ( - slowReaderSplices > 0 && - slowReaderStreamBytes <= RELAY_LOAD_SPLICE_HIGH_WATER_BYTES - ) { - throw new Error('--slow-reader-stream-bytes must exceed the 256 KiB splice high-water mark') - } - if ( - wedgedReaderSplices > 0 && - wedgedReaderStreamBytes <= RELAY_LOAD_SPLICE_HIGH_WATER_BYTES - ) { - throw new Error('--wedged-reader-stream-bytes must exceed the 256 KiB splice high-water mark') - } - const localReaderSplices = slowReaderSplices + wedgedReaderSplices - const localReaderBytes = slowReaderSplices * slowReaderStreamBytes + - wedgedReaderSplices * wedgedReaderStreamBytes - const aggregateReaderSplices = values.has('--aggregate-reader-splices') - ? integer('--aggregate-reader-splices', 0) - : localReaderSplices * shardCount - const aggregateReaderBytes = values.has('--aggregate-reader-bytes') - ? integer('--aggregate-reader-bytes', 0) - : localReaderBytes * shardCount - if ( - aggregateReaderSplices < localReaderSplices || - aggregateReaderBytes < localReaderBytes || - (shardCount === 1 && - (aggregateReaderSplices !== localReaderSplices || aggregateReaderBytes !== localReaderBytes)) - ) { - throw new Error('aggregate reader bounds do not cover the local shard') - } - if (aggregateReaderSplices > RELAY_LOAD_MAX_AGGREGATE_READER_SPLICES) { - throw new Error('aggregate reader splice count exceeds the reviewed bound') - } - if (aggregateReaderBytes > RELAY_LOAD_MAX_AGGREGATE_READER_BYTES) { - throw new Error('aggregate reader stream bytes exceed the reviewed bound') - } - if (readerFrameBytes < 1 || readerFrameBytes > 1_048_576) { - throw new Error('--reader-frame-bytes must be between 1 and 1048576') - } - if (slowReaderSplices > 0 && slowReaderHoldMs >= RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS) { - throw new Error('--slow-reader-hold-ms must stay below the wedged timeout') - } - if (wedgedReaderSplices > 0 && wedgedReaderHoldMs <= RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS) { - throw new Error('--wedged-reader-hold-ms must exceed the wedged timeout') - } - if (wedgedReaderHoldMs > 30_000) { - throw new Error('--wedged-reader-hold-ms cannot exceed 30000') - } - if (maxGeneratorRssGrowthMiB < 1) { - throw new Error('--max-generator-rss-growth-mib must be positive') - } - if (placementOverflowProbes > 1) { - throw new Error('--placement-overflow-probes must be zero or one') - } - if (regionalFallbackProbes > 1) { - throw new Error('--regional-fallback-probes must be zero or one') - } - if (regionBehaviorProbes > 1 || requestUnitOverflowProbes > 1) { - throw new Error('regional behavior and request-unit overflow probes must be zero or one') - } - if (placementOverflowProbes > 0 && shardCount > 1) { - throw new Error('placement overflow proof requires one coordinated generator') - } - if ( - placementOverflowProbes > 0 && - (!directorOrigin || - !capacityCellId || - capacityHardCap === undefined || - capacityUnobservedBound === undefined) - ) { - throw new Error('placement overflow requires exact capacity cell, hard cap, and bound') - } - if (regionalFallbackProbes > 0 && ( - !directorOrigin || !capacityCellId || !capacityCellOrigin || - capacityHardCap === undefined || capacityUnobservedBound === undefined || - (shardCount > 1 && shardIndex !== 0) - )) throw new Error('regional fallback requires the coordinating capacity shard') - if (regionBehaviorProbes > 0 && ( - !directorOrigin || !capacityCellOrigin || preferredRegionValue(values) !== 'asia-east2' || - (shardCount > 1 && shardIndex !== 0) - )) throw new Error('regional behavior proof requires the coordinating Asia shard') - if (duplicateAssignProbes > 1_000) { - throw new Error('--duplicate-assign-probes cannot exceed 1000') - } - if (duplicateAssignProbes > 0 && !directorOrigin) { - throw new Error('duplicate assign probes require --director-origin') - } - if (duplicateAssignHoldMs < 1_000 || duplicateAssignHoldMs > 120_000) { - throw new Error('--duplicate-assign-hold-ms must be between 1000 and 120000') - } - if (phaseBarrierDir && shardCount < 2) { - throw new Error('phase barrier requires multiple shards') - } - if (phaseBarrierTimeoutMs < 1_000) { - throw new Error('phase barrier timeout must be at least one second') - } - if (requestUnitInvites > 0) { - if ( - !directorOrigin || !capacityCellId || requestUnitCapacity === undefined || - requestUnitCapacity < 1 || requestUnitInvitesPerSecond < 1 || - requestUnitInvitesPerSecond > 20 || requestUnitPrincipalCount < 1 || - requestUnitPrincipalCount > 32 || requestUnitInvites > requestUnitPrincipalCount * 30 || - aggregateSplices !== 0 || !phaseBarrierDir || - aggregateRequestUnitInvites !== requestUnitInvites * shardCount || - aggregateControls + aggregateRequestUnitInvites !== requestUnitCapacity - ) throw new Error('request-unit proof does not reach the exact reviewed capacity') - } else if ( - requestUnitCapacity !== undefined || requestUnitInvitesPerSecond !== 0 || - requestUnitPrincipalCount !== 0 || - requestUnitOverflowProbes > 0 || - requestUnitCleanupTimeoutMs > 0 || aggregateRequestUnitInvites !== 0 - ) throw new Error('request-unit proof options require invite offers') - if ( - requestUnitOverflowProbes > 0 && - (shardIndex !== 0 || requestUnitCleanupTimeoutMs < 600_000) - ) throw new Error('request-unit overflow requires the cleanup-owning coordinator') - if (requestUnitCleanupTimeoutMs > 0 && requestUnitOverflowProbes !== 1) { - throw new Error('request-unit cleanup requires the overflow proof') - } - // A US cell's canary uses the same synthetic principals, aimed at the root region instead. - if ( - relayAsiaLoadPrincipalCount > 32 || - (relayAsiaLoadPrincipalCount > 0 && - (!directorOrigin || !['asia-east2', 'us-central1'].includes(preferredRegionValue(values)))) - ) throw new Error('Relay Asia load principals require a regional director proof') - if (capacityCellOrigin) { - const origin = new URL(capacityCellOrigin) - if (origin.protocol !== 'https:' || origin.origin !== capacityCellOrigin) { - throw new Error('--capacity-cell-origin must be canonical HTTPS') - } - } - if (shardCount < 1 || shardIndex >= shardCount) throw new Error('invalid shard index/count') - const minimumDurationMs = requiredLeaseHorizons * RELAY_CONTROL_LEASE_HORIZON_SECONDS * 1000 - if (durationMs < minimumDurationMs) { - throw new Error(`--duration-seconds must cover ${requiredLeaseHorizons} lease horizons`) - } - return { - targetOrigin, - directorOrigin, - authOrigin, - preferredRegion: preferredRegionValue(values), - controls, - maxRampConnectionFailures, - maxUnexpectedCloses, - rebindProbes, - placementOverflowProbes, - regionalFallbackProbes, - regionBehaviorProbes, - duplicateAssignProbes, - duplicateAssignHoldMs, - requestUnitInvites, - requestUnitInvitesPerSecond, - requestUnitPrincipalCount, - relayAsiaLoadPrincipalCount, - requestUnitOverflowProbes, - requestUnitCapacity, - requestUnitCleanupTimeoutMs, - rebindHoldMs, - rebindDelayMs, - capacityCellId, - capacityCellOrigin, - capacityHardCap, - capacityUnobservedBound, - durationMs, - spliceHoldMs, - rampMs: integer('--ramp-seconds', 60) * 1000, - rampStartDelayMs: integer('--ramp-start-delay-ms', 0), - reconnectMaxMs: integer('--reconnect-max-seconds', 30) * 1000, - // 0 keeps the old behaviour: the first splice failure fails the run. - spliceRetryMaxMs: integer('--splice-retry-max-seconds', 0) * 1000, - shardCount, - shardIndex, - aggregateControls, - aggregateSplices, - aggregateRequestUnitInvites, - phaseBarrierDir, - phaseBarrierTimeoutMs, - aggregateReaderSplices, - aggregateReaderBytes, - signingKeyFile: values.get('--signing-key-file'), - splices, - resumePhones, - resumeIntervalMs, - resumeHoldMs, - spliceRampMs, - splicePayloadBytes, - slowReaderSplices, - wedgedReaderSplices, - slowReaderStreamBytes, - wedgedReaderStreamBytes, - readerFrameBytes, - slowReaderHoldMs, - wedgedReaderHoldMs, - maxGeneratorRssGrowthMiB, - requiredLeaseHorizons, - allowPartial: flags.has('--allow-partial'), - allowPlannedTransitionRetries: flags.has('--allow-planned-transition-retries'), - requireRebindOverflow: !flags.has('--skip-rebind-overflow-check') - } -} - -function preferredRegionValue(values) { - return values.get('--preferred-region') -} - -export function relayLoadSpliceIndexes(config) { - return Array.from( - { length: config.splices }, - (_, localIndex) => localIndex * config.shardCount + config.shardIndex - ) -} - -// Resume phones ride the controls after the splice controls, so the two never share a peer. -export function relayLoadResumePhoneIndexes(config) { - return Array.from( - { length: config.resumePhones }, - (_, localIndex) => (config.splices + localIndex) * config.shardCount + config.shardIndex - ) -} - -export function relayLoadSpliceStartDelayMs(config, localIndex) { - if (!Number.isSafeInteger(localIndex) || localIndex < 0 || localIndex >= config.splices) { - throw new Error('invalid local splice index') - } - const totalSplices = config.splices * config.shardCount - if (totalSplices <= 1) return 0 - const globalOrdinal = localIndex * config.shardCount + config.shardIndex - return Math.floor(globalOrdinal * config.spliceRampMs / (totalSplices - 1)) -} - -export function relayLoadPrincipalIndex(peerIndex, shardCount, principalCount) { - if ( - !Number.isSafeInteger(peerIndex) || peerIndex < 0 || - !Number.isSafeInteger(shardCount) || shardCount < 1 || - !Number.isSafeInteger(principalCount) || principalCount < 1 - ) throw new Error('invalid Relay load principal mapping') - return Math.floor(peerIndex / shardCount) % principalCount -} - -export function relayLoadSpliceProfile(config, spliceIndex) { - if (spliceIndex < config.wedgedReaderSplices) { - return { - readerMode: 'wedged', - readerHoldMs: config.wedgedReaderHoldMs, - streamBytes: config.wedgedReaderStreamBytes, - frameBytes: config.readerFrameBytes - } - } - if (spliceIndex < config.wedgedReaderSplices + config.slowReaderSplices) { - return { - readerMode: 'slow', - readerHoldMs: config.slowReaderHoldMs, - streamBytes: config.slowReaderStreamBytes, - frameBytes: config.readerFrameBytes - } - } - return { - readerMode: 'normal', - readerHoldMs: 0, - streamBytes: config.splicePayloadBytes, - frameBytes: config.splicePayloadBytes - } -} - -export function relayLoadReaderEvidenceError(result, config) { - const readerSplices = config.slowReaderSplices + config.wedgedReaderSplices - if (readerSplices > 0 && result.generatorRssGrowthMiB > config.maxGeneratorRssGrowthMiB) { - return 'load generator exceeded its RSS growth budget' - } - if (result.slowReaderSplicesCompleted !== config.slowReaderSplices) { - return 'slow-reader streams did not all complete' - } - if (result.wedgedReaderSplicesClosed !== config.wedgedReaderSplices) { - return 'wedged-reader streams did not all close at the relay limit' - } - if ( - readerSplices > 0 && - (result.readerQueueEvidence.length === 0 || - result.readerQueueEvidence.some(({ increaseBytes }) => increaseBytes < 1)) - ) { - return 'reader streams produced no causal Relay queued-byte evidence' - } - if ( - config.wedgedReaderSplices > 0 && - result.readerClosesByCode['4429'] !== config.wedgedReaderSplices - ) { - return 'wedged-reader streams did not close with 4429' - } - return undefined -} diff --git a/cloud/dev/scripts/relay-load-profile.test.mjs b/cloud/dev/scripts/relay-load-profile.test.mjs deleted file mode 100644 index c5508453e76..00000000000 --- a/cloud/dev/scripts/relay-load-profile.test.mjs +++ /dev/null @@ -1,427 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { - parseRelayLoadArguments, - relayLoadPrincipalIndex, - relayLoadReaderEvidenceError, - relayLoadResumePhoneIndexes, - relayLoadSpliceIndexes, - relayLoadSpliceProfile, - relayLoadSpliceStartDelayMs -} from './relay-load-profile.mjs' - -const required = ['--auth-origin', 'https://auth.test', '--director-origin', 'https://relay.test'] - -test('accepts the 2840-control two-lease-horizon Asia proof profile', () => { - const config = parseRelayLoadArguments([ - ...required, - '--controls', - '2840', - '--duration-seconds', - '210', - '--required-lease-horizons', - '2', - '--preferred-region', - 'asia-east2', - '--relay-asia-load-principals', - '32', - '--capacity-hard-cap', - '3000' - ]) - - assert.equal(config.controls, 2840) - assert.equal(config.durationMs, 210_000) - assert.equal(config.requiredLeaseHorizons, 2) - assert.equal(config.preferredRegion, 'asia-east2') - assert.equal(config.relayAsiaLoadPrincipalCount, 32) - assert.equal(config.splices, 0) -}) - -test('binds synthetic Relay principals to a bounded Asia director proof', () => { - assert.throws(() => parseRelayLoadArguments([ - ...required, '--relay-asia-load-principals', '1' - ]), /require a regional director proof/) - assert.throws(() => parseRelayLoadArguments([ - ...required, '--preferred-region', 'asia-east2', - '--relay-asia-load-principals', '33' - ]), /require a regional director proof/) - assert.throws(() => parseRelayLoadArguments([ - ...required, '--preferred-region', 'europe-west1', '--relay-asia-load-principals', '1' - ]), /require a regional director proof/) - // A US cell's canary aims the one synthetic principal at us-central1. - assert.equal(parseRelayLoadArguments([ - ...required, '--preferred-region', 'us-central1', '--relay-asia-load-principals', '1' - ]).preferredRegion, 'us-central1') -}) - -test('rejects a mixed profile beyond the ordinary 2900-unit boundary', () => { - assert.throws( - () => parseRelayLoadArguments([ - ...required, - '--controls', '2840', - '--splices', '31', - '--capacity-hard-cap', '3000' - ]), - /exceed ordinary cell admission/ - ) - expectMixedProfile(parseRelayLoadArguments([ - ...required, - '--controls', '2600', - '--splices', '120', - '--capacity-hard-cap', '3000' - ])) -}) - -test('requires reviewed aggregate totals for capacity-bound shards', () => { - assert.throws( - () => parseRelayLoadArguments([ - ...required, '--controls', '2840', '--capacity-hard-cap', '3000', - '--shard-count', '4', '--shard-index', '0' - ]), - /requires explicit aggregate controls and splices/ - ) - assert.throws( - () => parseRelayLoadArguments([ - ...required, '--controls', '2840', '--capacity-hard-cap', '3000', - '--shard-count', '4', '--shard-index', '0', - '--aggregate-controls', '2840', '--aggregate-splices', '0' - ]), - /must match every equal-sized shard/ - ) - const config = parseRelayLoadArguments([ - ...required, '--controls', '710', '--capacity-hard-cap', '3000', - '--shard-count', '4', '--shard-index', '0', - '--aggregate-controls', '2840', '--aggregate-splices', '0' - ]) - assert.equal(config.aggregateControls, 2840) - assert.equal(config.aggregateSplices, 0) -}) - -test('allows one coordinating shard to prove regional fallback', () => { - const config = parseRelayLoadArguments([ - ...required, '--controls', '710', '--capacity-hard-cap', '3000', - '--shard-count', '4', '--shard-index', '0', - '--aggregate-controls', '2840', '--aggregate-splices', '0', - '--regional-fallback-probes', '1', '--capacity-cell-id', 'staging-gce-c4', - '--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev', - '--capacity-unobserved-bound', '60', '--rebind-probes', '160' - ]) - assert.equal(config.regionalFallbackProbes, 1) - assert.equal(config.capacityCellOrigin, 'https://c4.relay-staging.onorca.dev') - assert.throws(() => parseRelayLoadArguments([ - ...required, '--controls', '710', '--capacity-hard-cap', '3000', - '--shard-count', '4', '--shard-index', '1', - '--aggregate-controls', '2840', '--aggregate-splices', '0', - '--regional-fallback-probes', '1', '--capacity-cell-id', 'staging-gce-c4', - '--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev', - '--capacity-unobserved-bound', '60' - ]), /coordinating capacity shard/) -}) - -test('accepts the exact sharded request-unit and region behavior proof', () => { - const config = parseRelayLoadArguments([ - ...required, '--preferred-region', 'asia-east2', - '--controls', '710', '--capacity-hard-cap', '3000', - '--shard-count', '4', '--shard-index', '0', - '--aggregate-controls', '2840', '--aggregate-splices', '0', - '--capacity-cell-id', 'staging-gce-c4', - '--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev', - '--request-unit-invites', '790', '--request-unit-invites-per-second', '2', - '--request-unit-principals', '32', - '--aggregate-request-unit-invites', '3160', - '--request-unit-capacity', '6000', '--request-unit-overflow-probes', '1', - '--request-unit-cleanup-timeout-seconds', '630', - '--region-behavior-probes', '1', '--phase-barrier-dir', '/tmp/load-barrier' - ]) - assert.equal(config.aggregateRequestUnitInvites, 3_160) - assert.equal(config.requestUnitCapacity, 6_000) - assert.equal(config.requestUnitPrincipalCount, 32) - assert.equal(config.requestUnitCleanupTimeoutMs, 630_000) - assert.equal(config.regionBehaviorProbes, 1) - assert.equal(config.phaseBarrierDir, '/tmp/load-barrier') - - assert.throws(() => parseRelayLoadArguments([ - ...required, '--controls', '710', '--capacity-hard-cap', '3000', - '--shard-count', '4', '--shard-index', '0', - '--aggregate-controls', '2840', '--aggregate-splices', '0', - '--capacity-cell-id', 'staging-gce-c4', '--request-unit-invites', '789', - '--request-unit-invites-per-second', '2', - '--request-unit-principals', '32', - '--aggregate-request-unit-invites', '3156', '--request-unit-capacity', '6000', - '--phase-barrier-dir', '/tmp/load-barrier' - ]), /does not reach the exact reviewed capacity/) - - assert.throws(() => parseRelayLoadArguments([ - ...required, '--controls', '710', '--capacity-hard-cap', '3000', - '--shard-count', '4', '--shard-index', '0', - '--aggregate-controls', '2840', '--aggregate-splices', '0', - '--capacity-cell-id', 'staging-gce-c4', '--request-unit-invites', '790', - '--request-unit-invites-per-second', '2', '--request-unit-principals', '26', - '--aggregate-request-unit-invites', '3160', '--request-unit-capacity', '6000', - '--phase-barrier-dir', '/tmp/load-barrier' - ]), /does not reach the exact reviewed capacity/) -}) - -function expectMixedProfile(config) { - assert.equal(config.controls + 2 * config.splices, 2840) -} - -test('rejects a run shorter than its required lease horizons', () => { - assert.throws( - () => - parseRelayLoadArguments([ - ...required, - '--duration-seconds', - '209', - '--required-lease-horizons', - '2' - ]), - /must cover 2 lease horizons/ - ) -}) - -test('bounds an explicit splice hold within the steady window', () => { - const config = parseRelayLoadArguments([ - ...required, - '--controls', '1', - '--splices', '1', - '--duration-seconds', '300', - '--splice-hold-seconds', '60' - ]) - assert.equal(config.durationMs, 300_000) - assert.equal(config.spliceHoldMs, 60_000) - assert.throws(() => parseRelayLoadArguments([ - ...required, - '--controls', '1', - '--splices', '1', - '--duration-seconds', '300', - '--splice-hold-seconds', '301' - ]), /between 1 and the steady duration/) -}) - -test('maps splice ownership deterministically within a shard', () => { - const config = parseRelayLoadArguments([ - ...required, - '--controls', - '4', - '--splices', - '3', - '--shard-count', - '4', - '--shard-index', - '2' - ]) - - assert.deepEqual(relayLoadSpliceIndexes(config), [2, 6, 10]) -}) - -test('puts resume phones on the controls after the splice controls', () => { - const config = parseRelayLoadArguments([ - ...required, '--controls', '4', '--splices', '1', '--resume-phones', '2', - '--shard-count', '4', '--shard-index', '2' - ]) - - assert.deepEqual(relayLoadSpliceIndexes(config), [2]) - assert.deepEqual(relayLoadResumePhoneIndexes(config), [6, 10]) - assert.throws( - () => parseRelayLoadArguments([...required, '--controls', '2', '--splices', '1', '--resume-phones', '2']), - /--splices plus --resume-phones cannot exceed --controls/ - ) -}) - -test('staggered shards form one deterministic splice ramp', () => { - const config = parseRelayLoadArguments([ - ...required, - '--controls', '4', - '--splices', '3', - '--splice-ramp-seconds', '11', - '--shard-count', '4', - '--shard-index', '2' - ]) - - assert.equal(config.spliceRampMs, 11_000) - assert.deepEqual( - [0, 1, 2].map((index) => relayLoadSpliceStartDelayMs(config, index)), - [2_000, 6_000, 10_000] - ) - assert.throws(() => relayLoadSpliceStartDelayMs(config, 3), /invalid local splice index/) -}) - -test('distributes each shard invite wave below the account rate limit', () => { - const identities = Array.from( - { length: 710 }, - (_, localIndex) => relayLoadPrincipalIndex(localIndex * 4 + 2, 4, 32) - ) - const offers = Array.from({ length: 790 }, (_, index) => identities[index % identities.length]) - const counts = new Map() - for (const principal of offers) counts.set(principal, (counts.get(principal) ?? 0) + 1) - assert.equal(counts.size, 32) - assert.equal(Math.max(...counts.values()), 26) -}) - -test('requires exactly one assignment mode', () => { - assert.throws( - () => - parseRelayLoadArguments([ - ...required, - '--target-origin', - 'https://cell.test' - ]), - /exactly one target or director origin/ - ) -}) - -test('requires separate recoverable and wedged reader profiles', () => { - const config = parseRelayLoadArguments([ - ...required, - '--controls', '4', - '--splices', '3', - '--slow-reader-splices', '1', - '--wedged-reader-splices', '1', - '--slow-reader-stream-bytes', '524288', - '--wedged-reader-stream-bytes', '1048576', - '--slow-reader-hold-ms', '9000', - '--wedged-reader-hold-ms', '11000' - ]) - - assert.deepEqual(relayLoadSpliceProfile(config, 0), { - readerMode: 'wedged', readerHoldMs: 11_000, streamBytes: 1_048_576, frameBytes: 65_536 - }) - assert.deepEqual(relayLoadSpliceProfile(config, 1), { - readerMode: 'slow', readerHoldMs: 9_000, streamBytes: 524_288, frameBytes: 65_536 - }) - assert.equal(relayLoadSpliceProfile(config, 2).readerMode, 'normal') -}) - -test('bounds reader stream, timeout, and splice load per shard', () => { - assert.throws( - () => parseRelayLoadArguments([...required, '--controls', '2', '--splices', '3']), - /splices cannot exceed/ - ) - assert.throws( - () => - parseRelayLoadArguments([ - ...required, - '--splices', - '1', - '--slow-reader-splices', - '2' - ]), - /reader splice counts cannot exceed/ - ) - assert.throws( - () => parseRelayLoadArguments([ - ...required, '--splices', '1', '--slow-reader-splices', '1', - '--slow-reader-stream-bytes', '262144' - ]), - /must exceed the 256 KiB/ - ) - assert.throws( - () => parseRelayLoadArguments([ - ...required, '--splices', '1', '--wedged-reader-splices', '1', - '--wedged-reader-stream-bytes', '262144' - ]), - /must exceed the 256 KiB/ - ) - assert.throws( - () => parseRelayLoadArguments([ - ...required, '--splices', '1', '--slow-reader-splices', '1', - '--slow-reader-hold-ms', '10000' - ]), - /must stay below the wedged timeout/ - ) - assert.throws( - () => parseRelayLoadArguments([ - ...required, '--splices', '1', '--wedged-reader-splices', '1', - '--wedged-reader-hold-ms', '10000' - ]), - /must exceed the wedged timeout/ - ) - assert.throws( - () => parseRelayLoadArguments([ - ...required, '--controls', '17', '--splices', '17', '--slow-reader-splices', '17' - ]), - /reader splice count exceeds/ - ) - assert.throws( - () => parseRelayLoadArguments([ - ...required, '--controls', '9', '--splices', '9', '--wedged-reader-splices', '9' - ]), - /reader stream bytes exceed/ - ) -}) - -test('supports one bounded reader-owning shard without multiplying its pressure', () => { - const owner = parseRelayLoadArguments([ - ...required, - '--controls', '650', '--splices', '30', '--capacity-hard-cap', '3000', - '--shard-count', '4', '--shard-index', '0', - '--aggregate-controls', '2600', '--aggregate-splices', '120', - '--slow-reader-splices', '10', '--wedged-reader-splices', '1', - '--aggregate-reader-splices', '11', '--aggregate-reader-bytes', '18874368' - ]) - const peer = parseRelayLoadArguments([ - ...required, - '--controls', '650', '--splices', '30', '--capacity-hard-cap', '3000', - '--shard-count', '4', '--shard-index', '1', - '--aggregate-controls', '2600', '--aggregate-splices', '120', - '--aggregate-reader-splices', '11', '--aggregate-reader-bytes', '18874368' - ]) - - assert.equal(owner.aggregateReaderSplices, 11) - assert.equal(owner.aggregateReaderBytes, 18 * 1024 * 1024) - assert.equal(peer.slowReaderSplices + peer.wedgedReaderSplices, 0) - assert.equal(peer.aggregateReaderSplices, 11) -}) - -test('requires queue, memory, and expected close evidence', () => { - const config = parseRelayLoadArguments([ - ...required, '--splices', '2', '--slow-reader-splices', '1', - '--wedged-reader-splices', '1', '--max-generator-rss-growth-mib', '100' - ]) - const passing = { - generatorRssGrowthMiB: 50, - slowReaderSplicesCompleted: 1, - wedgedReaderSplicesClosed: 1, - readerQueueEvidence: [ - { origin: 'https://cell.test', baselineBytes: 8, peakBytes: 65_544, increaseBytes: 65_536 } - ], - readerClosesByCode: { '4429': 1 } - } - - assert.equal(relayLoadReaderEvidenceError(passing, config), undefined) - assert.match( - relayLoadReaderEvidenceError({ - ...passing, - readerQueueEvidence: [ - { origin: 'https://cell.test', baselineBytes: 8, peakBytes: 8, increaseBytes: 0 } - ] - }, config), - /no causal Relay queued-byte evidence/ - ) - assert.match( - relayLoadReaderEvidenceError({ ...passing, generatorRssGrowthMiB: 101 }, config), - /RSS growth budget/ - ) - assert.match( - relayLoadReaderEvidenceError({ ...passing, readerClosesByCode: { '4429': 0 } }, config), - /did not close with 4429/ - ) -}) - -test('bounds duplicate-assign probes to a director run', () => { - const config = parseRelayLoadArguments([...required, '--duplicate-assign-probes', '100']) - assert.equal(config.duplicateAssignProbes, 100) - assert.equal(config.duplicateAssignHoldMs, 20_000) - assert.equal(parseRelayLoadArguments(required).duplicateAssignProbes, 0) - assert.throws(() => parseRelayLoadArguments([ - '--auth-origin', 'https://auth.test', '--target-origin', 'https://cell.test', - '--duplicate-assign-probes', '1' - ]), /require --director-origin/) - assert.throws(() => parseRelayLoadArguments([ - ...required, '--duplicate-assign-probes', '1001' - ]), /cannot exceed 1000/) - assert.throws(() => parseRelayLoadArguments([ - ...required, '--duplicate-assign-probes', '1', '--duplicate-assign-hold-ms', '500' - ]), /between 1000 and 120000/) -}) diff --git a/cloud/dev/scripts/relay-load-reader-evidence.mjs b/cloud/dev/scripts/relay-load-reader-evidence.mjs deleted file mode 100644 index f20b5029587..00000000000 --- a/cloud/dev/scripts/relay-load-reader-evidence.mjs +++ /dev/null @@ -1,51 +0,0 @@ -const DEFAULT_TIMEOUT_MS = 8_000 -const DEFAULT_POLL_MS = 100 - -export async function createRelayLoadReaderEvidence(origins, dependencies) { - const distinctOrigins = [...new Set(origins)].sort() - const baselines = new Map(await Promise.all(distinctOrigins.map(async (origin) => [ - origin, - await dependencies.readQueuedBytes(origin) - ]))) - const peaks = new Map(baselines) - const pending = new Map() - const now = dependencies.now ?? Date.now - const delay = dependencies.delay - const timeoutMs = dependencies.timeoutMs ?? DEFAULT_TIMEOUT_MS - const pollMs = dependencies.pollMs ?? DEFAULT_POLL_MS - - const observe = async ({ cellOrigin }) => { - if (!baselines.has(cellOrigin)) throw new Error('reader origin lacks a run baseline') - if (peaks.get(cellOrigin) > baselines.get(cellOrigin)) return - const current = pending.get(cellOrigin) - if (current) return await current - const proof = (async () => { - const baseline = baselines.get(cellOrigin) - const deadline = now() + timeoutMs - for (;;) { - const queuedBytes = await dependencies.readQueuedBytes(cellOrigin) - peaks.set(cellOrigin, Math.max(peaks.get(cellOrigin), queuedBytes)) - if (queuedBytes > baseline) return - if (now() >= deadline) { - throw new Error('reader stream produced no causal Relay queued-byte increase') - } - await delay(pollMs) - } - })() - pending.set(cellOrigin, proof) - try { - await proof - } finally { - pending.delete(cellOrigin) - } - } - - const snapshot = () => distinctOrigins.map((origin) => ({ - origin, - baselineBytes: baselines.get(origin), - peakBytes: peaks.get(origin), - increaseBytes: peaks.get(origin) - baselines.get(origin) - })) - - return { observe, snapshot } -} diff --git a/cloud/dev/scripts/relay-load-reader-evidence.test.mjs b/cloud/dev/scripts/relay-load-reader-evidence.test.mjs deleted file mode 100644 index ce417f5bc42..00000000000 --- a/cloud/dev/scripts/relay-load-reader-evidence.test.mjs +++ /dev/null @@ -1,76 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs' - -test('requires a queue increase above the pre-injection baseline for every origin', async () => { - const samples = new Map([ - ['https://a.test', [7, 7, 11]], - ['https://b.test', [0, 3]] - ]) - let now = 0 - const evidence = await createRelayLoadReaderEvidence([...samples.keys()], { - readQueuedBytes: async (origin) => samples.get(origin).shift(), - delay: async (ms) => { now += ms }, - now: () => now - }) - - await Promise.all([ - evidence.observe({ cellOrigin: 'https://a.test' }), - evidence.observe({ cellOrigin: 'https://b.test' }) - ]) - - assert.deepEqual(evidence.snapshot(), [ - { origin: 'https://a.test', baselineBytes: 7, peakBytes: 11, increaseBytes: 4 }, - { origin: 'https://b.test', baselineBytes: 0, peakBytes: 3, increaseBytes: 3 } - ]) -}) - -test('shares one causal proof across concurrent readers on the same cell', async () => { - const samples = [4, 4, 9] - let reads = 0 - let now = 0 - const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], { - readQueuedBytes: async () => { reads++; return samples.shift() }, - delay: async (ms) => { now += ms }, - now: () => now - }) - - await Promise.all([ - evidence.observe({ cellOrigin: 'https://cell.test' }), - evidence.observe({ cellOrigin: 'https://cell.test' }) - ]) - - assert.equal(reads, 3) - assert.equal(evidence.snapshot()[0].increaseBytes, 5) -}) - -test('reuses a completed causal proof for later readers on the same cell', async () => { - const samples = [4, 9] - let reads = 0 - const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], { - readQueuedBytes: async () => { reads++; return samples.shift() }, - delay: async () => undefined - }) - - await evidence.observe({ cellOrigin: 'https://cell.test' }) - await evidence.observe({ cellOrigin: 'https://cell.test' }) - - assert.equal(reads, 2) - assert.equal(evidence.snapshot()[0].increaseBytes, 5) -}) - -test('rejects a pre-existing nonzero queue that never increases', async () => { - let now = 0 - const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], { - readQueuedBytes: async () => 9, - delay: async (ms) => { now += ms }, - now: () => now, - timeoutMs: 200, - pollMs: 100 - }) - - await assert.rejects( - evidence.observe({ cellOrigin: 'https://cell.test' }), - /no causal Relay queued-byte increase/ - ) -}) diff --git a/cloud/dev/scripts/relay-load-rebind-boundary.mjs b/cloud/dev/scripts/relay-load-rebind-boundary.mjs deleted file mode 100644 index 2eb72c7d6ba..00000000000 --- a/cloud/dev/scripts/relay-load-rebind-boundary.mjs +++ /dev/null @@ -1,59 +0,0 @@ -export async function waitForRelayLoadRebindGate({ - delay, - delayMs, - activeCount, - requiredCount -}) { - await delay(delayMs) - const active = activeCount() - if (active !== requiredCount) { - throw new Error(`rebind boundary requires ${requiredCount} active controls, found ${active}`) - } -} - -export async function proveRelayLoadRebindBoundary({ - peers, - probeCount, - holdMs, - delay, - failureReason, - requireOverflow = true -}) { - if (probeCount === 0) return { opened: 0, overflowReason: null } - if (peers.length < probeCount) throw new Error('insufficient active controls for rebind proof') - - const probes = [] - try { - const opened = await Promise.allSettled( - peers.slice(0, probeCount).map((peer) => peer.openRebindProbe()) - ) - for (const result of opened) { - if (result.status === 'fulfilled') probes.push(result.value) - } - const rejected = opened.find((result) => result.status === 'rejected') - if (rejected) throw rejected.reason - - let overflowReason = null - if (requireOverflow) { - try { - const overflow = await peers[0].openRebindProbe() - await overflow.close() - } catch (error) { - overflowReason = failureReason(error) - } - if (overflowReason !== 'socket_http_503') { - throw new Error(`rebind overflow was not rejected at the hard cap: ${overflowReason}`) - } - } - const closedIndex = await Promise.race([ - delay(holdMs).then(() => -1), - ...probes.map((probe, index) => probe.closed.then(() => index)) - ]) - if (closedIndex >= 0 || probes.some((probe) => !probe.isOpen())) { - throw new Error('rebind probe closed before the hold completed') - } - return { opened: probes.length, overflowReason } - } finally { - await Promise.all(probes.map((probe) => probe.close())) - } -} diff --git a/cloud/dev/scripts/relay-load-rebind-boundary.test.mjs b/cloud/dev/scripts/relay-load-rebind-boundary.test.mjs deleted file mode 100644 index 50ad2a48383..00000000000 --- a/cloud/dev/scripts/relay-load-rebind-boundary.test.mjs +++ /dev/null @@ -1,164 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { - proveRelayLoadRebindBoundary, - waitForRelayLoadRebindGate -} from './relay-load-rebind-boundary.mjs' - -function peer(open) { - return { openRebindProbe: open } -} - -function probe(onClose = () => undefined) { - let open = true - let resolveClosed - const closed = new Promise((resolve) => { - resolveClosed = resolve - }) - return { - close: () => { - if (!open) return - open = false - onClose() - resolveClosed() - }, - closed, - isOpen: () => open - } -} - -test('holds the requested rebind overlap and requires a hard-cap rejection', async () => { - let openCalls = 0 - let closes = 0 - let heldFor = null - const peers = [ - peer(async () => { - openCalls++ - if (openCalls === 3) throw new Error('Unexpected server response: 503') - return probe(() => closes++) - }), - peer(async () => { - openCalls++ - return probe(() => closes++) - }) - ] - const result = await proveRelayLoadRebindBoundary({ - peers, - probeCount: 2, - holdMs: 4_000, - delay: async (milliseconds) => { - heldFor = milliseconds - }, - failureReason: (error) => - error.message.includes('503') ? 'socket_http_503' : 'unknown' - }) - - assert.deepEqual(result, { opened: 2, overflowReason: 'socket_http_503' }) - assert.equal(heldFor, 4_000) - assert.equal(closes, 2) -}) - -test('closes successful probes when a boundary probe fails', async () => { - let closes = 0 - const peers = [ - peer(async () => probe(() => closes++)), - peer(async () => { - throw new Error('probe failed') - }) - ] - - await assert.rejects( - proveRelayLoadRebindBoundary({ - peers, - probeCount: 2, - holdMs: 0, - delay: async () => undefined, - failureReason: () => 'unknown' - }), - /probe failed/ - ) - assert.equal(closes, 1) -}) - -test('waits for every replacement socket to finish closing', async () => { - let finishClose - const closeFinished = new Promise((resolve) => { - finishClose = resolve - }) - const closingProbe = probe() - closingProbe.close = () => closeFinished - let completed = false - const boundary = proveRelayLoadRebindBoundary({ - peers: [peer(async () => closingProbe)], - probeCount: 1, - holdMs: 0, - delay: async () => undefined, - failureReason: () => 'unknown', - requireOverflow: false - }).then(() => { - completed = true - }) - - await new Promise((resolve) => setImmediate(resolve)) - assert.equal(completed, false) - finishClose() - await boundary - assert.equal(completed, true) -}) - -test('can prove reserved replacement headroom below the physical cap', async () => { - let closes = 0 - const result = await proveRelayLoadRebindBoundary({ - peers: [peer(async () => probe(() => closes++))], - probeCount: 1, - holdMs: 0, - delay: async () => undefined, - failureReason: () => 'unknown', - requireOverflow: false - }) - assert.deepEqual(result, { opened: 1, overflowReason: null }) - assert.equal(closes, 1) -}) - -test('fails when a replacement closes before the hold completes', async () => { - let heldProbe - await assert.rejects( - proveRelayLoadRebindBoundary({ - peers: [ - peer(async () => { - heldProbe = probe() - return heldProbe - }) - ], - probeCount: 1, - holdMs: 4_000, - delay: async () => { - heldProbe.close() - }, - failureReason: () => 'unknown', - requireOverflow: false - }), - /closed before the hold completed/ - ) -}) - -test('delays the boundary until every ordinary control has recovered', async () => { - let active = 899 - await assert.rejects( - waitForRelayLoadRebindGate({ - delay: async () => undefined, - delayMs: 0, - activeCount: () => active, - requiredCount: 900 - }), - /requires 900 active controls/ - ) - await waitForRelayLoadRebindGate({ - delay: async () => { - active = 900 - }, - delayMs: 1, - activeCount: () => active, - requiredCount: 900 - }) -}) diff --git a/cloud/dev/scripts/relay-load-region-behavior.mjs b/cloud/dev/scripts/relay-load-region-behavior.mjs deleted file mode 100644 index b0333b32abe..00000000000 --- a/cloud/dev/scripts/relay-load-region-behavior.mjs +++ /dev/null @@ -1,35 +0,0 @@ -const ASSIGNMENT_RETRY_DELAY_MS = 5_100 - -const waitPastAssignmentRateLimit = (schedule) => - new Promise((resolve) => schedule(resolve, ASSIGNMENT_RETRY_DELAY_MS)) - -export async function proveRelayLoadRegionBehavior({ - oldClientPeer, - stickyPeer, - asiaOrigin, - scheduleAssignmentRetry = setTimeout -}) { - try { - await oldClientPeer.connect() - if (!oldClientPeer.assignedCellUrl() || oldClientPeer.assignedCellUrl() === asiaOrigin) { - throw new Error('unhinted client did not use the US-first path') - } - } finally { - await oldClientPeer.shutdown() - } - - try { - await stickyPeer.connect() - if (stickyPeer.assignedCellUrl() !== asiaOrigin) { - throw new Error('preferred Asia client did not reach the Asia cell') - } - await waitPastAssignmentRateLimit(scheduleAssignmentRetry) - const reassigned = await stickyPeer.requestAssignment('us-central1') - if (reassigned.cellUrl !== asiaOrigin) { - throw new Error('valid sticky assignment moved after preference changed') - } - } finally { - await stickyPeer.shutdown() - } - return { oldClientUsFirst: true, stickyAssignmentPreserved: true } -} diff --git a/cloud/dev/scripts/relay-load-region-behavior.test.mjs b/cloud/dev/scripts/relay-load-region-behavior.test.mjs deleted file mode 100644 index 5882e3021a1..00000000000 --- a/cloud/dev/scripts/relay-load-region-behavior.test.mjs +++ /dev/null @@ -1,47 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { proveRelayLoadRegionBehavior } from './relay-load-region-behavior.mjs' - -function peer(origin, reassigned = origin) { - let shutdowns = 0 - return { - connect: async () => undefined, - assignedCellUrl: () => origin, - requestAssignment: async () => ({ cellUrl: reassigned }), - shutdown: async () => { shutdowns++ }, - shutdowns: () => shutdowns - } -} - -test('proves unhinted US-first placement and sticky Asia preservation', async () => { - const oldClientPeer = peer('https://c3.relay-staging.onorca.dev') - const stickyPeer = peer('https://c4.relay-staging.onorca.dev') - let retryDelayMs = 0 - assert.deepEqual(await proveRelayLoadRegionBehavior({ - oldClientPeer, - stickyPeer, - asiaOrigin: 'https://c4.relay-staging.onorca.dev', - scheduleAssignmentRetry: (resolve, delayMs) => { - retryDelayMs = delayMs - resolve() - } - }), { oldClientUsFirst: true, stickyAssignmentPreserved: true }) - assert.equal(retryDelayMs, 5_100) - assert.equal(oldClientPeer.shutdowns(), 1) - assert.equal(stickyPeer.shutdowns(), 1) -}) - -test('rejects Asia placement for an unhinted client or a moved sticky assignment', async () => { - await assert.rejects(proveRelayLoadRegionBehavior({ - oldClientPeer: peer('https://c4.relay-staging.onorca.dev'), - stickyPeer: peer('https://c4.relay-staging.onorca.dev'), - asiaOrigin: 'https://c4.relay-staging.onorca.dev', - scheduleAssignmentRetry: (resolve) => resolve() - }), /US-first/) - await assert.rejects(proveRelayLoadRegionBehavior({ - oldClientPeer: peer('https://c3.relay-staging.onorca.dev'), - stickyPeer: peer('https://c4.relay-staging.onorca.dev', 'https://c3.relay-staging.onorca.dev'), - asiaOrigin: 'https://c4.relay-staging.onorca.dev', - scheduleAssignmentRetry: (resolve) => resolve() - }), /sticky assignment moved/) -}) diff --git a/cloud/dev/scripts/relay-load-request-unit-boundary.mjs b/cloud/dev/scripts/relay-load-request-unit-boundary.mjs deleted file mode 100644 index a5dbb59d96b..00000000000 --- a/cloud/dev/scripts/relay-load-request-unit-boundary.mjs +++ /dev/null @@ -1,43 +0,0 @@ -import { setTimeout as delayDefault } from 'node:timers/promises' - -export async function openRelayLoadInviteOffers({ - peers, - count, - ratePerSecond, - concurrency = 8, - delay = delayDefault, - now = Date.now -}) { - if ( - !Array.isArray(peers) || peers.length === 0 || - !Number.isSafeInteger(count) || count < 0 || - !Number.isSafeInteger(ratePerSecond) || ratePerSecond < 1 || ratePerSecond > 20 || - !Number.isSafeInteger(concurrency) || concurrency < 1 - ) throw new Error('invalid Relay invite-offer load') - let next = 0 - let nextStartAt = now() - const workers = Array.from({ length: Math.min(concurrency, count) }, async () => { - for (;;) { - const index = next++ - if (index >= count) return - const scheduledAt = Math.max(nextStartAt, now()) - nextStartAt = scheduledAt + 1_000 / ratePerSecond - await delay(Math.max(0, scheduledAt - now())) - await peers[index % peers.length].openInviteOffer() - } - }) - await Promise.all(workers) - return count -} - -export async function proveRelayLoadRequestUnitBoundary(peer) { - try { - await peer.openInviteOffer() - } catch (error) { - if (error instanceof Error && error.message === 'invite offer failed: relay_capacity_exhausted') { - return 'relay_capacity_exhausted' - } - throw new Error('request-unit overflow was not rejected safely', { cause: error }) - } - throw new Error('request-unit overflow unexpectedly succeeded') -} diff --git a/cloud/dev/scripts/relay-load-request-unit-boundary.test.mjs b/cloud/dev/scripts/relay-load-request-unit-boundary.test.mjs deleted file mode 100644 index 739e055c93a..00000000000 --- a/cloud/dev/scripts/relay-load-request-unit-boundary.test.mjs +++ /dev/null @@ -1,52 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { - openRelayLoadInviteOffers, - proveRelayLoadRequestUnitBoundary -} from './relay-load-request-unit-boundary.mjs' - -test('distributes the exact invite count with bounded concurrency', async () => { - let active = 0 - let peak = 0 - const delays = [] - const calls = [0, 0, 0] - const peers = calls.map((_, index) => ({ - async openInviteOffer() { - calls[index]++ - active++ - peak = Math.max(peak, active) - await Promise.resolve() - active-- - } - })) - assert.equal(await openRelayLoadInviteOffers({ - peers, - count: 8, - ratePerSecond: 2, - concurrency: 2, - delay: async (milliseconds) => { delays.push(milliseconds) }, - now: () => 0 - }), 8) - assert.deepEqual(calls, [3, 3, 2]) - assert.ok(peak <= 2) - assert.equal(delays.length, 8) - assert.ok(Math.max(...delays) >= 3_500) -}) - -test('accepts only the exact request-unit exhaustion error', async () => { - assert.equal(await proveRelayLoadRequestUnitBoundary({ - openInviteOffer: async () => { - throw new Error('invite offer failed: relay_capacity_exhausted') - } - }), 'relay_capacity_exhausted') - await assert.rejects( - proveRelayLoadRequestUnitBoundary({ - openInviteOffer: async () => { throw new Error('control response timeout') } - }), - /not rejected safely/ - ) - await assert.rejects( - proveRelayLoadRequestUnitBoundary({ openInviteOffer: async () => undefined }), - /unexpectedly succeeded/ - ) -}) diff --git a/cloud/dev/scripts/relay-load-resume-phone.mjs b/cloud/dev/scripts/relay-load-resume-phone.mjs deleted file mode 100644 index c010f27b9e3..00000000000 --- a/cloud/dev/scripts/relay-load-resume-phone.mjs +++ /dev/null @@ -1,220 +0,0 @@ -import { createHash, randomBytes } from 'node:crypto' -import { nextFrame, nextJson, waitForOpen } from './relay-load-control-peer.mjs' - -// A paired phone's reconnects: one device install over an invite splice, then real device -// resumes (accept, then the desktop's device-resume-confirm), timed from the phone's side. - -const ECHO_INTERVAL_MS = 5_000 - -const resumeTokenHash = (token) => createHash('sha256').update(token).digest('base64url') - -function controlAnswer(peer, reqId, type) { - return peer.waitForControlMessage( - (message) => - message.reqId === reqId && (message.type === type || message.type === 'control-error') - ) -} - -function closeCode(socket, timeoutMs = 2_000) { - if (socket.readyState === socket.CLOSED) return Promise.resolve(undefined) - return new Promise((resolve) => { - const timer = setTimeout(() => resolve(undefined), timeoutMs) - socket.once('close', (code) => { - clearTimeout(timer) - resolve(code) - }) - }) -} - -// The phone dials, the desktop's control answers conn-open, and the desktop attaches its data leg. -async function dialPhone(peer, credential, relayDeviceId) { - if (!peer.socket || peer.socket.readyState !== peer.socket.OPEN || peer.generation === undefined) { - throw Object.assign(new Error('active control required for a phone'), { reason: 'no-control' }) - } - const assignment = peer.lastAssignment - const phone = peer.createClientSocket(assignment) - peer.trackSpliceSocket(phone) - let data - try { - const setData = (socket) => { - data = socket - } - return await attachPhone(peer, { assignment, phone, credential, relayDeviceId, setData }) - } catch (error) { - // A close before any hello names its code, e.g. 4503 while the database is down. - const closed = /closed: (\d+)/.exec(String(error.message)) - throw Object.assign(error, { - reason: error.reason ?? (closed ? `hello-${closed[1]}` : undefined), - phone, - data - }) - } -} - -async function attachPhone(peer, { assignment, phone, credential, relayDeviceId, setData }) { - await waitForOpen(phone) - const connectionPromise = peer.waitForControlMessage( - (message) => message.type === 'conn-open' && message.relayDeviceId === relayDeviceId - ) - connectionPromise.catch(() => undefined) - const hello = nextJson(phone) - const startedAt = performance.now() - phone.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential })) - // A refused phone gets a failed hello and no conn-open. - const first = await Promise.race([ - connectionPromise.then((connection) => ({ connection })), - hello.then((answer) => ({ answer })) - ]) - if (first.answer) { - const code = first.answer.code ?? (await closeCode(phone)) - throw Object.assign(new Error(`phone refused: ${code}`), { reason: `hello-${code}` }) - } - const { connection } = first - const data = peer.createHostDataSocket(assignment, connection.connId) - setData(data) - peer.trackSpliceSocket(data) - await waitForOpen(data) - data.send( - JSON.stringify({ - type: 'host-data-auth', - v: 1, - connTicket: connection.connTicket, - generation: peer.generation - }) - ) - const answer = await hello - const acceptMs = performance.now() - startedAt - if (answer.ok !== true) { - const code = answer.code ?? (await closeCode(phone)) - throw Object.assign(new Error(`phone refused: ${code}`), { reason: `hello-${code}` }) - } - return { phone, data, connId: connection.connId, acceptMs } -} - -async function echo(phone, data, payload) { - const toHost = nextFrame(data) - phone.send(payload) - if ((await toHost).bytes.toString() !== payload) throw new Error('relay changed phone payload') - const toPhone = nextFrame(phone) - data.send(Buffer.from(payload)) - if ((await toPhone).bytes.toString() !== payload) throw new Error('relay changed host payload') -} - -function rttMs(socket, timeoutMs = 5_000) { - return new Promise((resolve) => { - const startedAt = performance.now() - const timer = setTimeout(() => finish(undefined), timeoutMs) - const finish = (value) => { - clearTimeout(timer) - socket.off('pong', onPong) - resolve(value) - } - const onPong = () => finish(performance.now() - startedAt) - socket.on('pong', onPong) - socket.ping() - }) -} - -export async function installRelayLoadResumeDevice(peer, relayDeviceId) { - const inviteReqId = `load-resume-invite-${peer.index}-${peer.spliceSequence++}` - const invite = controlAnswer(peer, inviteReqId, 'invite-created') - peer.socket.send(JSON.stringify({ type: 'invite-create', reqId: inviteReqId, relayDeviceId })) - const created = await invite - if (created.type !== 'invite-created') throw new Error(`invite failed: ${created.code}`) - const splice = await dialPhone(peer, created.inviteToken, relayDeviceId) - try { - const token = randomBytes(32).toString('base64url') - const reqId = `load-resume-install-${peer.index}-${peer.spliceSequence++}` - const installed = controlAnswer(peer, reqId, 'device-credential-installed') - peer.socket.send( - JSON.stringify({ - type: 'device-credential-install', - v: 1, - reqId, - relayDeviceId, - newResumeTokenHash: resumeTokenHash(token), - authorization: { mode: 'relay-basis', basisConnId: splice.connId } - }) - ) - const result = await installed - if (result.type !== 'device-credential-installed') { - throw new Error(`device install failed: ${result.code}`) - } - return token - } finally { - await Promise.all([peer.closeSpliceSocket(splice.phone), peer.closeSpliceSocket(splice.data)]) - } -} - -// One resume: accept, confirm, then hold the splice with periodic echoes. Never throws. -export async function resumeRelayLoadPhoneOnce(peer, { token, relayDeviceId }, { holdMs, stopping, delay }) { - const attempt = { at: Date.now(), cellUrl: peer.lastAssignment?.cellUrl } - let splice - try { - splice = await dialPhone(peer, token, relayDeviceId) - attempt.acceptMs = Math.round(splice.acceptMs) - const reqId = `load-resume-confirm-${peer.index}-${peer.spliceSequence++}` - const confirmed = controlAnswer(peer, reqId, 'device-resume-confirmed') - const confirmStartedAt = performance.now() - peer.socket.send( - JSON.stringify({ type: 'device-resume-confirm', v: 1, reqId, basisConnId: splice.connId }) - ) - const result = await confirmed - attempt.confirmMs = Math.round(performance.now() - confirmStartedAt) - if (result.type !== 'device-resume-confirmed') { - attempt.confirmFailure = String(result.code) - } else { - attempt.renewed = result.renewed - } - const rtt = await rttMs(splice.phone) - if (rtt !== undefined) attempt.rttMs = Math.round(rtt) - const holdUntil = Date.now() + holdMs - attempt.echoes = 0 - while (!stopping() && Date.now() < holdUntil) { - await echo(splice.phone, splice.data, `orca-relay-resume:${peer.index}:${attempt.echoes}`) - attempt.echoes++ - await delay(Math.min(ECHO_INTERVAL_MS, Math.max(0, holdUntil - Date.now()))) - } - attempt.ok = attempt.confirmFailure === undefined - } catch (error) { - attempt.ok = false - attempt.failure = error.reason ?? (splice ? 'held-splice' : 'accept') - attempt.error = String(error.message).slice(0, 160) - if (splice) attempt.heldCloseCode = await closeCode(splice.phone, 100) - splice ??= { phone: error.phone, data: error.data } - } finally { - await Promise.all([peer.closeSpliceSocket(splice?.phone), peer.closeSpliceSocket(splice?.data)]) - } - return attempt -} - -const percentile = (values, fraction) => { - if (values.length === 0) return undefined - const sorted = [...values].sort((left, right) => left - right) - return sorted[Math.min(sorted.length - 1, Math.floor(fraction * sorted.length))] -} - -export function relayLoadResumeSummary(attempts) { - const accepts = attempts.flatMap((attempt) => - attempt.acceptMs === undefined ? [] : [attempt.acceptMs] - ) - const confirms = attempts.flatMap((attempt) => - attempt.confirmMs === undefined || attempt.confirmFailure ? [] : [attempt.confirmMs] - ) - const rtts = attempts.flatMap((attempt) => (attempt.rttMs === undefined ? [] : [attempt.rttMs])) - const failures = {} - for (const attempt of attempts) { - const reason = attempt.confirmFailure ? `confirm-${attempt.confirmFailure}` : attempt.failure - if (reason) failures[reason] = (failures[reason] ?? 0) + 1 - } - return { - attempts: attempts.length, - ok: attempts.filter((attempt) => attempt.ok).length, - acceptMsP50: percentile(accepts, 0.5), - acceptMsP95: percentile(accepts, 0.95), - confirmMsP50: percentile(confirms, 0.5), - confirmMsP95: percentile(confirms, 0.95), - rttMsP50: percentile(rtts, 0.5), - failures - } -} diff --git a/cloud/dev/scripts/relay-load-resume-phone.test.mjs b/cloud/dev/scripts/relay-load-resume-phone.test.mjs deleted file mode 100644 index 26fd3a1938b..00000000000 --- a/cloud/dev/scripts/relay-load-resume-phone.test.mjs +++ /dev/null @@ -1,24 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { relayLoadResumeSummary } from './relay-load-resume-phone.mjs' - -test('summarizes resume latency from completed steps and failures by reason', () => { - const summary = relayLoadResumeSummary([ - { ok: true, acceptMs: 100, confirmMs: 20, rttMs: 10 }, - { ok: true, acceptMs: 300, confirmMs: 40, rttMs: 10 }, - { ok: false, acceptMs: 200, confirmMs: 900, confirmFailure: 'database_unavailable' }, - { ok: false, failure: 'hello-4503' }, - { ok: false, failure: 'hello-4503' } - ]) - - assert.deepEqual(summary, { - attempts: 5, - ok: 2, - acceptMsP50: 200, - acceptMsP95: 300, - confirmMsP50: 40, - confirmMsP95: 40, - rttMsP50: 10, - failures: { 'confirm-database_unavailable': 1, 'hello-4503': 2 } - }) -}) diff --git a/cloud/dev/scripts/relay-load-run-lifecycle.mjs b/cloud/dev/scripts/relay-load-run-lifecycle.mjs deleted file mode 100644 index dffd76a813b..00000000000 --- a/cloud/dev/scripts/relay-load-run-lifecycle.mjs +++ /dev/null @@ -1,25 +0,0 @@ -export function assertRelayLoadRampAccepted(rampConnectionFailures, maximum) { - if (rampConnectionFailures > maximum) { - throw new Error('relay load ramp exceeded the allowed connection failures') - } -} - -export async function runRelayLoadWithShutdown(operation, shutdown) { - try { - return await operation() - } finally { - await shutdown() - } -} - -export function relayLoadRunHasDisallowedFailures(result, config) { - return ( - result.rampConnectionFailures > config.maxRampConnectionFailures || - (!config.allowPlannedTransitionRetries && result.transitionConnectionFailures > 0) || - result.steadyConnectionFailures > 0 || - result.unexpectedCloses > config.maxUnexpectedCloses || - result.protocolErrors > 0 || - result.refreshErrors > 0 || - result.socketErrors > 0 - ) -} diff --git a/cloud/dev/scripts/relay-load-run-lifecycle.test.mjs b/cloud/dev/scripts/relay-load-run-lifecycle.test.mjs deleted file mode 100644 index 5aec8e08ab9..00000000000 --- a/cloud/dev/scripts/relay-load-run-lifecycle.test.mjs +++ /dev/null @@ -1,68 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { - assertRelayLoadRampAccepted, - relayLoadRunHasDisallowedFailures, - runRelayLoadWithShutdown -} from './relay-load-run-lifecycle.mjs' - -test('always shuts down peers when a load phase fails', async () => { - const events = [] - await assert.rejects( - runRelayLoadWithShutdown( - async () => { - events.push('run') - throw new Error('boundary failed') - }, - async () => events.push('shutdown') - ), - /boundary failed/ - ) - assert.deepEqual(events, ['run', 'shutdown']) -}) - -test('fails immediately when the strict ramp budget is exceeded', () => { - assert.doesNotThrow(() => assertRelayLoadRampAccepted(0, 0)) - assert.throws(() => assertRelayLoadRampAccepted(1, 0), /ramp exceeded/) -}) - -test('rejects connection failures during the transition window', () => { - const result = { - rampConnectionFailures: 0, - transitionConnectionFailures: 1, - steadyConnectionFailures: 0, - unexpectedCloses: 0, - protocolErrors: 0, - refreshErrors: 0, - socketErrors: 0 - } - assert.equal( - relayLoadRunHasDisallowedFailures(result, { - maxRampConnectionFailures: 0, - maxUnexpectedCloses: 0 - }), - true - ) -}) - -test('allows only explicitly planned transition retries', () => { - const result = { - rampConnectionFailures: 0, - transitionConnectionFailures: 1, - steadyConnectionFailures: 0, - unexpectedCloses: 0, - protocolErrors: 0, - refreshErrors: 0, - socketErrors: 0 - } - const config = { - allowPlannedTransitionRetries: true, - maxRampConnectionFailures: 0, - maxUnexpectedCloses: 0 - } - assert.equal(relayLoadRunHasDisallowedFailures(result, config), false) - assert.equal( - relayLoadRunHasDisallowedFailures({ ...result, steadyConnectionFailures: 1 }, config), - true - ) -}) diff --git a/cloud/dev/scripts/relay-load-splice-retry.mjs b/cloud/dev/scripts/relay-load-splice-retry.mjs deleted file mode 100644 index 4c4f441d145..00000000000 --- a/cloud/dev/scripts/relay-load-splice-retry.mjs +++ /dev/null @@ -1,40 +0,0 @@ -// A phone reopens its splice after a close, the way the app does, instead of failing the run. -// Without retries (retryMaxMs 0) the first failure still rejects, as before. -export async function holdRelayLoadSplice({ - open, - retryMaxMs, - stopping, - onFailure = () => undefined, - delay, - random = Math.random -}) { - if (retryMaxMs <= 0) return await open() - while (!stopping()) { - try { - return await open() - } catch (error) { - if (stopping()) return undefined - onFailure(relayLoadSpliceFailureReason(error)) - await delay(Math.ceil(random() * retryMaxMs)) - } - } - return undefined -} - -// Stable buckets for the report: close codes and HTTP statuses, never ids. -export function relayLoadSpliceFailureReason(error) { - const message = error instanceof Error ? error.message : String(error) - const closed = /^splice closed: (\d+)/.exec(message) - if (closed) return `closed_${closed[1]}` - const response = /^Unexpected server response: (\d{3})/.exec(message) - if (response) return `http_${response[1]}` - if (message.startsWith('active control required')) return 'no_active_control' - return 'other' -} - -export function relayLoadSpliceReopenSummary(gapsMs, awaiting) { - const sorted = [...gapsMs].sort((left, right) => left - right) - const at = (quantile) => - sorted.length === 0 ? null : sorted[Math.min(sorted.length - 1, Math.floor(quantile * sorted.length))] - return { reopened: sorted.length, awaiting, p50Ms: at(0.5), p95Ms: at(0.95), maxMs: sorted.at(-1) ?? null } -} diff --git a/cloud/dev/scripts/relay-load-splice-retry.test.mjs b/cloud/dev/scripts/relay-load-splice-retry.test.mjs deleted file mode 100644 index 9da7f963cf2..00000000000 --- a/cloud/dev/scripts/relay-load-splice-retry.test.mjs +++ /dev/null @@ -1,81 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { - holdRelayLoadSplice, - relayLoadSpliceFailureReason, - relayLoadSpliceReopenSummary -} from './relay-load-splice-retry.mjs' - -const noDelay = async () => undefined - -test('without retries the first splice failure still fails the run', async () => { - await assert.rejects( - holdRelayLoadSplice({ - open: async () => { throw new Error('splice closed: 1006 ') }, - retryMaxMs: 0, - stopping: () => false, - delay: noDelay - }), - /splice closed: 1006/ - ) -}) - -test('with retries a closed splice reopens and its reasons are counted', async () => { - const failures = [] - const waits = [] - let attempts = 0 - const result = await holdRelayLoadSplice({ - open: async () => { - attempts++ - if (attempts === 1) throw new Error('splice closed: 1006 ') - if (attempts === 2) throw new Error('Unexpected server response: 429') - return 'held' - }, - retryMaxMs: 2_000, - stopping: () => false, - onFailure: (reason) => failures.push(reason), - delay: async (ms) => { waits.push(ms) }, - random: () => 0.5 - }) - assert.equal(result, 'held') - assert.deepEqual(failures, ['closed_1006', 'http_429']) - assert.deepEqual(waits, [1_000, 1_000]) -}) - -test('a failure during shutdown is not counted and does not retry', async () => { - let stopping = false - const failures = [] - const result = await holdRelayLoadSplice({ - open: async () => { stopping = true; throw new Error('splice closed: 1000 ') }, - retryMaxMs: 1_000, - stopping: () => stopping, - onFailure: (reason) => failures.push(reason), - delay: noDelay - }) - assert.equal(result, undefined) - assert.deepEqual(failures, []) -}) - -test('failure reasons bucket close codes, HTTP statuses and a missing control', () => { - assert.equal(relayLoadSpliceFailureReason(new Error('splice closed: 4409 demoted')), 'closed_4409') - assert.equal(relayLoadSpliceFailureReason(new Error('Unexpected server response: 503')), 'http_503') - assert.equal(relayLoadSpliceFailureReason(new Error('active control required for splice')), 'no_active_control') - assert.equal(relayLoadSpliceFailureReason(new Error('relay frame timeout')), 'other') -}) - -test('the reopen summary reports quantiles and splices still waiting', () => { - assert.deepEqual(relayLoadSpliceReopenSummary([300, 100, 200], 1), { - reopened: 3, - awaiting: 1, - p50Ms: 200, - p95Ms: 300, - maxMs: 300 - }) - assert.deepEqual(relayLoadSpliceReopenSummary([], 0), { - reopened: 0, - awaiting: 0, - p50Ms: null, - p95Ms: null, - maxMs: null - }) -}) diff --git a/cloud/dev/scripts/relay-lock-contention-alerts.test.mjs b/cloud/dev/scripts/relay-lock-contention-alerts.test.mjs deleted file mode 100644 index a922921b277..00000000000 --- a/cloud/dev/scripts/relay-lock-contention-alerts.test.mjs +++ /dev/null @@ -1,83 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { fileURLToPath } from 'node:url' - -// Why: both lock alerts key on text another component writes (a relay runtime field and a -// Postgres error message). A rename on either side silences the alert without failing anything. - -const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8') -const terraform = read('../../infra/terraform/relay-observability.tf') - -const block = (kind, name) => { - const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1] - assert.ok(body, `${kind}.${name} not found in relay-observability.tf`) - return body -} - -const incidentMetric = (key) => { - const body = new RegExp(`\\n ${key} = \\{([\\s\\S]*?)\\n \\}`).exec(terraform)?.[1] - assert.ok(body, `relay_incident_metrics.${key} not found`) - return body -} - -test('the long-hold metric thresholds the field the relay emits', () => { - const emitter = read('../../apps/relay/src/cell-inventory-hold-samples.ts') - assert.match(emitter, /\bcellInventoryHoldMsMax: number\b/) - const metric = block('google_logging_metric', 'relay_long_cell_inventory_hold') - assert.match(metric, /filter\s*=\s*"\$\{local\.relay_runtime_log_filter\} AND jsonPayload\.cellInventoryHoldMsMax>=1000"/) - assert.match(metric, /value_extractor\s*=\s*"EXTRACT\(jsonPayload\.cellInventoryHoldMsMax\)"/) -}) - -test('the long-hold buckets start at the filter bar so any sample reads above zero', () => { - const metric = block('google_logging_metric', 'relay_long_cell_inventory_hold') - const bounds = /bounds\s*=\s*\[([^\]]*)\]/.exec(metric)?.[1].split(',').map(Number) - assert.ok(bounds && bounds.length > 0) - assert.equal(bounds[0], 1000) - for (const name of ['relay_long_cell_inventory_hold', 'relay_director_cell_inventory_hold']) { - const policy = block('google_monitoring_alert_policy', name) - const conditions = [...policy.matchAll(/threshold_value\s*=\s*(\d+)\n/g)] - assert.ok(conditions.length > 0) - for (const match of conditions) assert.equal(match[1], '0') - } -}) - -test('only cell holds page; director holds stay visible without paging', () => { - // Director holds recur with rehoming paused, and the paging runbook's first step is to pause it. - const paging = block('google_monitoring_alert_policy', 'relay_long_cell_inventory_hold') - const pagingRoles = [...paging.matchAll(/metric\.label\.\\"role\\"=\\"([a-z]+)\\"/g)].map((m) => m[1]) - assert.ok(pagingRoles.length > 0) - assert.deepEqual([...new Set(pagingRoles)], ['cell']) - assert.equal([...paging.matchAll(/condition_threshold \{/g)].length, pagingRoles.length) - - const director = block('google_monitoring_alert_policy', 'relay_director_cell_inventory_hold') - const directorRoles = [...director.matchAll(/metric\.label\.\\"role\\"=\\"([a-z]+)\\"/g)].map((m) => m[1]) - assert.deepEqual(directorRoles, ['director']) - assert.match(director, /notification_channels\s*=\s*\[\]/) -}) - -test('the lock-timeout metric counts relay cancels only, never NOWAIT refusals or auth', () => { - const metric = incidentMetric('cloud_sql_lock_timeouts') - const filter = /filter\s*=\s*"(.*)"$/m.exec(metric)?.[1] - assert.ok(filter) - assert.match(filter, /resource\.type=\\"cloudsql_database\\"/) - assert.match(filter, /textPayload:\\"db=orca_relay,\\"/) - assert.match(filter, /textPayload:\\"canceling statement due to lock timeout\\"/) - // Would match the `SET LOCAL lock_timeout` STATEMENT line Postgres logs after every cancel. - assert.doesNotMatch(filter, /lock_timeout/) - // Background sweeps refuse at ~160/min with rehome paused; including them pins the alert on. - assert.doesNotMatch(filter, /could not obtain lock/) -}) - -test('the burst policy fires at 20 relay cancels in a minute and pages the relay channel', () => { - const policy = block('google_monitoring_alert_policy', 'relay_lock_timeout_burst') - assert.match(policy, /logging\.googleapis\.com\/user\/orca_relay_cloud_sql_lock_timeouts/) - assert.match(policy, /comparison\s*=\s*"COMPARISON_GT"\s*\n\s*threshold_value\s*=\s*19\n/) - assert.match(policy, /alignment_period\s*=\s*"60s"\s*\n\s*per_series_aligner\s*=\s*"ALIGN_SUM"/) - for (const name of ['relay_lock_timeout_burst', 'relay_long_cell_inventory_hold']) { - assert.match( - block('google_monitoring_alert_policy', name), - /notification_channels\s*=\s*var\.relay_alert_notification_channels/ - ) - } -}) diff --git a/cloud/dev/scripts/relay-monitor-evidence.mjs b/cloud/dev/scripts/relay-monitor-evidence.mjs deleted file mode 100644 index 425771cc030..00000000000 --- a/cloud/dev/scripts/relay-monitor-evidence.mjs +++ /dev/null @@ -1,359 +0,0 @@ -import { createHash } from 'node:crypto' -import { chmod, readFile, readdir, stat, writeFile } from 'node:fs/promises' -import { basename, join, resolve } from 'node:path' -import { pathToFileURL } from 'node:url' -import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs' - -const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{1,127}$/ -const SHA = /^[a-f0-9]{40}$/ -const JWT = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/ -const EVIDENCE_MAX_AGE_MS = 5 * 60_000 -const EVIDENCE_SAMPLE_INTERVAL_MS = 60_000 -const EVIDENCE_MAX_LINEAGE_MS = 25 * 60_000 -const MIGRATION_POLICIES = new Set([ - 'strict', - 'recover-forward', - 'capacity-transition' -]) -const MUTATION_MODES = new Set([ - 'capacity-transition', - 'continue-evacuation', - 'disable-cell', - 'enable-empty-cell', - 'execute', - 'fence-source', - 'recover-forward', - 'reset-empty-candidate' -]) - -function argumentsByName(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const name = argv[index] - const value = argv[index + 1] - if (!name?.startsWith('--') || !value || value.startsWith('--')) { - throw new Error('relay monitor evidence arguments are invalid') - } - values[name.slice(2)] = value - } - return values -} - -async function sha256(path) { - return createHash('sha256').update(await readFile(path)).digest('hex') -} - -async function regularFile(path) { - try { - return (await stat(path)).isFile() - } catch { - return false - } -} - -function provenance(values) { - const runAttempt = Number(values['run-attempt']) - if ( - !SAFE_ID.test(values['incident-id'] ?? '') || - !SAFE_ID.test(values['run-id'] ?? '') || - !Number.isSafeInteger(runAttempt) || - runAttempt < 1 || - !SHA.test(values['commit-sha'] ?? '') || - !['dry-run', 'monitor'].includes(values.mode) - ) { - throw new Error('relay monitor evidence provenance is invalid') - } - return { - incidentId: values['incident-id'], - runId: values['run-id'], - runAttempt, - commitSha: values['commit-sha'], - mode: values.mode - } -} - -export async function createEvidenceManifest(argv) { - const values = argumentsByName(argv) - const directory = resolve(values.directory ?? '') - const expected = provenance(values) - const candidates = [ - `${expected.incidentId}.state.json`, - `${expected.incidentId}.summaries.jsonl`, - `${expected.incidentId}.summary.md` - ] - const files = {} - for (const name of candidates) { - const path = join(directory, name) - if (await regularFile(path)) files[name] = await sha256(path) - } - if (!files[`${expected.incidentId}.state.json`]) { - throw new Error('relay monitor durable state is missing') - } - const manifest = { - schemaVersion: 1, - ...expected, - files - } - const path = join(directory, 'evidence-manifest.json') - await writeFile(path, `${JSON.stringify(manifest)}\n`, { mode: 0o600 }) - await chmod(path, 0o600) - return manifest -} - -async function readAndVerifyManifest(directory, expected, sameCodeCommit) { - const manifest = JSON.parse( - await readFile(join(directory, 'evidence-manifest.json'), 'utf8') - ) - if ( - manifest.schemaVersion !== 1 || - manifest.incidentId !== expected.incidentId || - manifest.runId !== expected.runId || - manifest.runAttempt !== expected.runAttempt || - !SHA.test(manifest.commitSha ?? '') || - manifest.mode !== expected.mode || - (!sameCodeCommit && manifest.commitSha !== expected.commitSha) - ) { - throw new Error('relay monitor evidence provenance does not match') - } - // Unrelated merges land on main every few minutes, so the deployer resolves a newer commit than - // the monitor it must trust; identical monitor and mutation code is the property the SHA stood in - // for. Restore and mutation keep the exact-SHA bind: both run at the commit that sealed them. - if (sameCodeCommit) { - requireSameEvidenceCode({ - sealedSha: manifest.commitSha, - currentSha: expected.commitSha, - label: 'relay monitor evidence', - ...sameCodeCommit - }) - } - const names = Object.keys(manifest.files ?? {}) - if (!names.includes(`${expected.incidentId}.state.json`)) { - throw new Error('relay monitor evidence has no durable state') - } - for (const name of names) { - if (basename(name) !== name || !/^[A-Za-z0-9._-]+$/.test(name)) { - throw new Error('relay monitor evidence file name is invalid') - } - if (await sha256(join(directory, name)) !== manifest.files[name]) { - throw new Error('relay monitor evidence hash does not match') - } - } - const allowed = new Set([...names, 'evidence-manifest.json']) - const unexpected = (await readdir(directory)).filter((name) => !allowed.has(name)) - if (unexpected.length > 0) throw new Error('relay monitor evidence has unexpected files') - return manifest -} - -function validMigrationPolicyState(state) { - return ( - ( - state.migrationPolicy === 'strict' && - state.recoverySourceCellId === null && - state.capacityCellId === null - ) || - ( - state.migrationPolicy === 'recover-forward' && - state.capacityCellId === null && - typeof state.recoverySourceCellId === 'string' && - state.expectedSelector?.membership?.existingOnly?.includes( - state.recoverySourceCellId - ) - ) || - ( - state.migrationPolicy === 'capacity-transition' && - state.recoverySourceCellId === null && - typeof state.capacityCellId === 'string' && - state.expectedSelector?.membership?.general?.includes(state.capacityCellId) - ) - ) -} - -export async function verifyRestoredEvidence(argv) { - const values = argumentsByName(argv) - const directory = resolve(values.directory ?? '') - const expected = provenance(values) - await readAndVerifyManifest(directory, expected) - const state = JSON.parse( - await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8') - ) - if ( - state.schemaVersion !== 4 || - state.incidentId !== expected.incidentId || - state.environment !== 'production' || - state.preDrainDryRun !== (expected.mode === 'dry-run') || - !MIGRATION_POLICIES.has(state.migrationPolicy) || - !validMigrationPolicyState(state) - ) { - throw new Error('relay monitor restored state does not match provenance') - } - return state -} - -function validCompletedDryRunState(state, expected, nowMs, maxAgeMs) { - const completedAt = Date.parse(state.completedAt) - const startedAt = Date.parse(state.startedAt) - const windowStartedAt = Date.parse(state.windowStartedAt) - const lastSampleAt = Date.parse(state.lastSampleAt) - const age = nowMs - completedAt - return ( - state.schemaVersion === 4 && - state.incidentId === expected.incidentId && - state.environment === 'production' && - state.preDrainDryRun === true && - validMigrationPolicyState(state) && - state.durationMinutes === 15 && - state.intervalMs === EVIDENCE_SAMPLE_INTERVAL_MS && - state.sampleCount >= 16 && - state.frozenAt === null && - Number.isFinite(startedAt) && - completedAt - startedAt >= 0 && - completedAt - startedAt <= EVIDENCE_MAX_LINEAGE_MS && - Number.isFinite(windowStartedAt) && - completedAt - windowStartedAt >= 15 * 60_000 && - Number.isFinite(lastSampleAt) && - lastSampleAt <= completedAt && - completedAt - lastSampleAt <= state.intervalMs && - Number.isFinite(completedAt) && - age >= 0 && - age <= maxAgeMs - ) -} - -/** - * Holds the evidence to 5 minutes on this process's own clock. The single-job rehome enable is - * the only caller; it verifies, consumes, and mutates in one job. - */ -export async function verifyDryRunAuthority(argv, now = Date.now, repositoryRoot) { - const values = argumentsByName(argv) - const directory = resolve(values.directory ?? '') - const expected = provenance(values) - if (expected.mode !== 'dry-run') throw new Error('relay mutation requires dry-run evidence') - const manifest = await readAndVerifyManifest(directory, expected, { repositoryRoot }) - const state = JSON.parse( - await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8') - ) - const requiredMigrationPolicy = values['required-migration-policy'] - if ( - !MIGRATION_POLICIES.has(requiredMigrationPolicy) || - state.migrationPolicy !== requiredMigrationPolicy || - !validCompletedDryRunState(state, expected, now(), EVIDENCE_MAX_AGE_MS) - ) { - throw new Error('relay monitor dry-run authority is incomplete or stale') - } - return { manifest, state } -} - -function exactSelector(actual, expected) { - const membership = (selector) => { - if ( - !selector?.membership || - !['existingOnly', 'migrationOnly', 'general'].every((key) => - Array.isArray(selector.membership[key]) - ) - ) return null - const normalized = Object.fromEntries( - ['existingOnly', 'migrationOnly', 'general'].map((key) => [ - key, - [...selector.membership[key]].sort() - ]) - ) - const all = Object.values(normalized).flat() - return new Set(all).size === all.length ? normalized : null - } - const actualMembership = membership(actual) - const expectedMembership = membership(expected) - return Boolean( - actualMembership && - expectedMembership && - actual?.generation === expected?.generation && - JSON.stringify(actualMembership) === JSON.stringify(expectedMembership) - ) -} - -export async function verifyMutationEvidence( - argv, - environment = process.env, - fetchImpl = fetch, - now = Date.now -) { - const values = argumentsByName(argv) - const directory = resolve(values.directory ?? '') - const expected = provenance(values) - if (expected.mode !== 'dry-run') throw new Error('mutation requires dry-run evidence') - const manifest = await readAndVerifyManifest(directory, expected) - const state = JSON.parse( - await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8') - ) - const mutationMode = values['mutation-mode'] - if (!MUTATION_MODES.has(mutationMode)) { - throw new Error('relay monitor mutation mode is invalid') - } - const scopedRecoverySourceCellId = - values['scoped-recovery-source-cell-id'] - const recoveryMutation = ['fence-source', 'recover-forward'].includes(mutationMode) - const scopedRecoveryMutation = - ['execute', 'recover-forward'].includes(mutationMode) && - Boolean(scopedRecoverySourceCellId) - if (scopedRecoverySourceCellId && !scopedRecoveryMutation) { - throw new Error('relay monitor scoped recovery evidence is invalid') - } - const requiredMigrationPolicy = mutationMode === 'capacity-transition' - ? 'capacity-transition' - : recoveryMutation || scopedRecoveryMutation ? 'recover-forward' : 'strict' - if (state.migrationPolicy !== requiredMigrationPolicy) { - throw new Error('relay monitor migration policy does not match mutation') - } - const expectedRecoverySourceCellId = scopedRecoveryMutation - ? scopedRecoverySourceCellId - : values['source-cell-id'] - if ( - (recoveryMutation || scopedRecoveryMutation) && - state.recoverySourceCellId !== expectedRecoverySourceCellId - ) { - throw new Error('relay monitor recovery source does not match mutation') - } - if ( - mutationMode === 'capacity-transition' && - state.capacityCellId !== values['source-cell-id'] - ) { - throw new Error('relay monitor capacity cell does not match mutation') - } - if ( - !validCompletedDryRunState(state, expected, now(), EVIDENCE_MAX_AGE_MS) - ) { - throw new Error('relay monitor dry-run evidence is incomplete or stale') - } - const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN - const origin = values['director-origin'] - if (!token || !JWT.test(token) || !origin?.startsWith('https://')) { - throw new Error('relay monitor live selector verification is unavailable') - } - const response = await fetchImpl(`${origin}/v1/admin/admission-selector/status`, { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify({ v: 1 }), - signal: AbortSignal.timeout(30_000) - }) - if (!response.ok) throw new Error('relay monitor live selector verification failed') - const current = (await response.json()).selector - if (!exactSelector(current, state.expectedSelector)) { - throw new Error('relay admission selector changed after the dry run') - } - return { manifest, state } -} - -async function main() { - const [command, ...argv] = process.argv.slice(2) - if (command === 'create') await createEvidenceManifest(argv) - else if (command === 'verify-restore') await verifyRestoredEvidence(argv) - else if (command === 'verify-authority') await verifyDryRunAuthority(argv) - else if (command === 'verify-mutation') await verifyMutationEvidence(argv) - else throw new Error('relay monitor evidence command is invalid') -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - console.error(error instanceof Error ? error.message : 'relay monitor evidence failed') - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/relay-monitor-evidence.test.mjs b/cloud/dev/scripts/relay-monitor-evidence.test.mjs deleted file mode 100644 index 1a27f7ae48d..00000000000 --- a/cloud/dev/scripts/relay-monitor-evidence.test.mjs +++ /dev/null @@ -1,640 +0,0 @@ -import assert from 'node:assert/strict' -import { execFileSync } from 'node:child_process' -import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { dirname, join } from 'node:path' -import test from 'node:test' -import { TRUSTED_EVIDENCE_CODE_PATHS } from './relay-evidence-code-provenance.mjs' -import { - RELAY_REPOSITORY_ROOT, - relayWorkflowPath, - relayWorkflowUrl -} from './relay-repository.mjs' -import { - createEvidenceManifest, - verifyDryRunAuthority, - verifyMutationEvidence, - verifyRestoredEvidence -} from './relay-monitor-evidence.mjs' - -const now = Date.parse('2026-07-28T12:00:00.000Z') -const provenanceFor = (commitSha) => [ - '--incident-id', - 'relay-123', - '--run-id', - '123', - '--run-attempt', - '1', - '--commit-sha', - commitSha, - '--mode', - 'dry-run' -] -const provenance = provenanceFor('a'.repeat(40)) -const selector = { - generation: 2, - membership: { - existingOnly: ['c1'], - migrationOnly: ['c2'], - general: ['c3'] - } -} - -async function evidenceDirectory(migrationPolicy = 'strict') { - const directory = await mkdtemp(join(tmpdir(), 'relay-monitor-evidence-')) - const state = { - schemaVersion: 4, - incidentId: 'relay-123', - environment: 'production', - preDrainDryRun: true, - migrationPolicy, - recoverySourceCellId: migrationPolicy === 'recover-forward' ? 'c1' : null, - capacityCellId: migrationPolicy === 'capacity-transition' ? 'c3' : null, - startedAt: new Date(now - 17 * 60_000).toISOString(), - durationMinutes: 15, - intervalMs: 60_000, - sampleCount: 16, - windowStartedAt: new Date(now - 16 * 60_000).toISOString(), - lastSampleAt: new Date(now - 60_007).toISOString(), - completedAt: new Date(now - 60_000).toISOString(), - frozenAt: null, - expectedSelector: selector - } - await writeFile( - join(directory, 'relay-123.state.json'), - `${JSON.stringify(state)}\n` - ) - return directory -} - -test('creates and verifies exact restart provenance and hashes', async () => { - const directory = await evidenceDirectory() - try { - await createEvidenceManifest(['--directory', directory, ...provenance]) - await assert.doesNotReject( - verifyRestoredEvidence(['--directory', directory, ...provenance]) - ) - await writeFile(join(directory, 'relay-123.state.json'), '{}\n') - await assert.rejects( - verifyRestoredEvidence(['--directory', directory, ...provenance]), - /hash does not match/ - ) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -test('requires fresh green evidence and rechecks the live selector', async () => { - const directory = await evidenceDirectory() - try { - await createEvidenceManifest(['--directory', directory, ...provenance]) - const verifyAuthority = () => verifyDryRunAuthority( - [ - '--directory', - directory, - ...provenance, - '--required-migration-policy', - 'strict' - ], - () => now - ) - await assert.doesNotReject(verifyAuthority()) - const fetchImpl = async (_input, init) => { - assert.equal( - new Headers(init.headers).get('authorization'), - 'Bearer aaa.bbb.ccc' - ) - return Response.json({ selector }) - } - await assert.doesNotReject( - verifyMutationEvidence( - [ - '--directory', - directory, - ...provenance, - '--mutation-mode', - 'execute', - '--source-cell-id', - 'c1', - '--director-origin', - 'https://relay.example' - ], - { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, - fetchImpl, - () => now - ) - ) - const statePath = join(directory, 'relay-123.state.json') - const state = JSON.parse(await readFile(statePath, 'utf8')) - state.completedAt = new Date(now - 300_001).toISOString() - await writeFile(statePath, `${JSON.stringify(state)}\n`) - await createEvidenceManifest(['--directory', directory, ...provenance]) - await assert.rejects( - verifyAuthority(), - /authority is incomplete or stale/ - ) - await assert.rejects( - verifyMutationEvidence( - [ - '--directory', - directory, - ...provenance, - '--mutation-mode', - 'execute', - '--source-cell-id', - 'c1', - '--director-origin', - 'https://relay.example' - ], - { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, - fetchImpl, - () => now - ), - /incomplete or stale/ - ) - state.completedAt = new Date(now - 60_000).toISOString() - state.lastSampleAt = new Date(now - 120_001).toISOString() - await writeFile(statePath, `${JSON.stringify(state)}\n`) - await createEvidenceManifest(['--directory', directory, ...provenance]) - await assert.rejects( - verifyMutationEvidence( - [ - '--directory', - directory, - ...provenance, - '--mutation-mode', - 'execute', - '--source-cell-id', - 'c1', - '--director-origin', - 'https://relay.example' - ], - { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, - fetchImpl, - () => now - ), - /incomplete or stale/ - ) - state.lastSampleAt = new Date(now - 60_007).toISOString() - state.startedAt = new Date(now - 26 * 60_000 - 1).toISOString() - await writeFile(statePath, `${JSON.stringify(state)}\n`) - await createEvidenceManifest(['--directory', directory, ...provenance]) - await assert.rejects( - verifyAuthority(), - /authority is incomplete or stale/ - ) - await assert.rejects( - verifyMutationEvidence( - [ - '--directory', - directory, - ...provenance, - '--mutation-mode', - 'execute', - '--source-cell-id', - 'c1', - '--director-origin', - 'https://relay.example' - ], - { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, - fetchImpl, - () => now - ), - /incomplete or stale/ - ) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -test('dry-run authority holds evidence to five minutes on its own clock', async () => { - const directory = await evidenceDirectory() - try { - const statePath = join(directory, 'relay-123.state.json') - const state = JSON.parse(await readFile(statePath, 'utf8')) - const authority = () => verifyDryRunAuthority( - ['--directory', directory, ...provenance, '--required-migration-policy', 'strict'], - () => now - ) - const ageState = async (ageMs) => { - state.completedAt = new Date(now - ageMs).toISOString() - state.lastSampleAt = new Date(now - ageMs - 7).toISOString() - state.startedAt = new Date(now - ageMs - 17 * 60_000).toISOString() - state.windowStartedAt = new Date(now - ageMs - 16 * 60_000).toISOString() - await writeFile(statePath, `${JSON.stringify(state)}\n`) - await createEvidenceManifest(['--directory', directory, ...provenance]) - } - await ageState(5 * 60_000) - await assert.doesNotReject(authority()) - await ageState(5 * 60_000 + 1) - await assert.rejects(authority(), /authority is incomplete or stale/) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -test('binds migration policies to their exact mutations', async () => { - const strictDirectory = await evidenceDirectory() - const recoveryDirectory = await evidenceDirectory('recover-forward') - const capacityDirectory = await evidenceDirectory('capacity-transition') - const fetchImpl = async () => Response.json({ selector }) - try { - await createEvidenceManifest(['--directory', strictDirectory, ...provenance]) - await createEvidenceManifest(['--directory', recoveryDirectory, ...provenance]) - await createEvidenceManifest(['--directory', capacityDirectory, ...provenance]) - await assert.rejects( - verifyDryRunAuthority( - [ - '--directory', - recoveryDirectory, - ...provenance, - '--required-migration-policy', - 'strict' - ], - () => now - ), - /authority is incomplete or stale/ - ) - const verify = (directory, mutationMode, sourceCellId = 'c1') => verifyMutationEvidence( - [ - '--directory', - directory, - ...provenance, - '--mutation-mode', - mutationMode, - '--source-cell-id', - sourceCellId, - '--director-origin', - 'https://relay.example' - ], - { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, - fetchImpl, - () => now - ) - await assert.doesNotReject(verify(strictDirectory, 'execute')) - await assert.doesNotReject( - verify(capacityDirectory, 'capacity-transition', 'c3') - ) - await assert.doesNotReject(verify(recoveryDirectory, 'recover-forward')) - await assert.doesNotReject(verify(recoveryDirectory, 'fence-source')) - await assert.doesNotReject( - verifyMutationEvidence( - [ - '--directory', - recoveryDirectory, - ...provenance, - '--mutation-mode', - 'execute', - '--source-cell-id', - 'c12', - '--scoped-recovery-source-cell-id', - 'c1', - '--director-origin', - 'https://relay.example' - ], - { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, - fetchImpl, - () => now - ) - ) - await assert.doesNotReject( - verifyMutationEvidence( - [ - '--directory', - recoveryDirectory, - ...provenance, - '--mutation-mode', - 'recover-forward', - '--source-cell-id', - 'c12', - '--scoped-recovery-source-cell-id', - 'c1', - '--director-origin', - 'https://relay.example' - ], - { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, - fetchImpl, - () => now - ) - ) - await assert.rejects( - verify(recoveryDirectory, 'execute'), - /migration policy does not match/ - ) - await assert.rejects( - verify(strictDirectory, 'recover-forward'), - /migration policy does not match/ - ) - await assert.rejects( - verify(strictDirectory, 'capacity-transition', 'c3'), - /migration policy does not match/ - ) - await assert.rejects( - verify(capacityDirectory, 'capacity-transition', 'c1'), - /capacity cell does not match/ - ) - await assert.rejects( - verifyMutationEvidence( - [ - '--directory', - recoveryDirectory, - ...provenance, - '--mutation-mode', - 'recover-forward', - '--source-cell-id', - 'c9', - '--director-origin', - 'https://relay.example' - ], - { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, - fetchImpl, - () => now - ), - /recovery source does not match/ - ) - await assert.rejects( - verifyMutationEvidence( - [ - '--directory', - recoveryDirectory, - ...provenance, - '--mutation-mode', - 'fence-source', - '--source-cell-id', - 'c1', - '--scoped-recovery-source-cell-id', - 'c1', - '--director-origin', - 'https://relay.example' - ], - { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, - fetchImpl, - () => now - ), - /scoped recovery evidence is invalid/ - ) - await assert.rejects( - verifyMutationEvidence( - [ - '--directory', - recoveryDirectory, - ...provenance, - '--mutation-mode', - 'execute', - '--source-cell-id', - 'c12', - '--scoped-recovery-source-cell-id', - 'c9', - '--director-origin', - 'https://relay.example' - ], - { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, - fetchImpl, - () => now - ), - /recovery source does not match/ - ) - } finally { - await rm(strictDirectory, { recursive: true, force: true }) - await rm(recoveryDirectory, { recursive: true, force: true }) - await rm(capacityDirectory, { recursive: true, force: true }) - } -}) - -test('workflow reruns restore the prior attempt into one stable incident', async () => { - const workflow = await readFile( - relayWorkflowUrl('monitor-relay-production-job.yml'), - 'utf8' - ) - assert.match(workflow, /INCIDENT_ID: relay-\$\{\{ github\.run_id \}\}-\$\{\{ inputs\.mode \}\}/) - assert.doesNotMatch(workflow, /INCIDENT_ID:.*run_attempt/) - assert.match(workflow, /actions\/download-artifact@v4/) - assert.match(workflow, /verify-restore/) - assert.match(workflow, /RESTART_FLAG=--restart/) - assert.equal(workflow.match(/--capacity-cell-id/g)?.length, 3) - const dispatchWorkflow = await readFile( - relayWorkflowUrl('monitor-relay-production.yml'), - 'utf8' - ) - assert.match(dispatchWorkflow, /- capacity-transition/) - assert.match(dispatchWorkflow, /capacity-cell-id: \$\{\{ inputs\.capacity-cell-id \}\}/) -}) - -test('rehome enable requires complete strict dry-run authority', async () => { - const workflow = await readFile( - relayWorkflowUrl('operate-relay-production-rehome-job.yml'), - 'utf8' - ) - assert.match(workflow, /relay-monitor-evidence\.mjs verify-authority/) - assert.match(workflow, /--required-migration-policy strict/) - assert.doesNotMatch(workflow, /relay-monitor-evidence\.mjs verify-restore/) -}) - -test('production mutation workflows consume and live-recheck dry-run evidence', async () => { - for (const name of [ - 'deploy-relay-production.yml', - 'deploy-relay-production-multi-target.yml' - ]) { - const workflow = await readFile( - relayWorkflowUrl(name), - 'utf8' - ) - assert.match(workflow, /actions\/download-artifact@v4/) - assert.match(workflow, /verify-mutation/) - assert.match(workflow, /--mutation-mode "\$\{DEPLOY_MODE\}"/) - assert.match(workflow, /--source-cell-id "\$\{SOURCE_CELL_ID\}"/) - assert.match(workflow, /incident:relay-preflight/) - assert.match(workflow, /Reject previously consumed dry-run evidence/) - assert.match(workflow, /actions\/upload-artifact@v4/) - assert.match(workflow, /relay-monitor-consumed-/) - assert.match(workflow, /ORCA_RELAY_ADMIN_ID_TOKEN/) - assert.match(workflow, /github\.ref == 'refs\/heads\/main'/) - assert.ok( - workflow.indexOf('pnpm install --frozen-lockfile') < - workflow.indexOf('id: google-auth') - ) - } -}) - -test('monitor and mutation workflows share the production Cloud SQL rollout lock', async () => { - for (const name of [ - 'monitor-relay-production.yml', - 'deploy-relay-production.yml', - 'deploy-relay-production-multi-target.yml', - 'deploy-relay-production-capacity.yml' - ]) { - const workflow = await readFile( - relayWorkflowUrl(name), - 'utf8' - ) - assert.match(workflow, /group: production-cloud-sql-rollout/) - } -}) - -test('monitor uses a reusable job so exact job_workflow_ref is present', async () => { - const wrapper = await readFile( - relayWorkflowUrl('monitor-relay-production.yml'), - 'utf8' - ) - assert.ok(wrapper.includes(`uses: ./${relayWorkflowPath('monitor-relay-production-job.yml')}`)) - const job = await readFile( - relayWorkflowUrl('monitor-relay-production-job.yml'), - 'utf8' - ) - assert.match(job, /workflow_call:/) - assert.match(job, /environment: production/) -}) - -function gitIn(root, ...args) { - return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim() -} - -// A real repository shaped like main under unrelated merge traffic: one sealed commit, a -// descendant that only touched untrusted files, a descendant that touched the monitor, and a -// sibling that never descended from the seal. -async function trustedCodeRepository() { - const root = await mkdtemp(join(tmpdir(), 'relay-evidence-repository-')) - gitIn(root, 'init', '--quiet') - gitIn(root, 'config', 'user.email', 'relay@example.test') - gitIn(root, 'config', 'user.name', 'Relay Evidence Test') - gitIn(root, 'config', 'commit.gpgsign', 'false') - const commit = async (path, body, message) => { - await mkdir(dirname(join(root, path)), { recursive: true }) - await writeFile(join(root, path), body) - gitIn(root, 'add', '--all') - gitIn(root, 'commit', '--quiet', '--no-verify', '--message', message) - return gitIn(root, 'rev-parse', 'HEAD') - } - const base = await commit( - 'cloud/apps/relay-ops/src/incident-monitor.ts', - 'export const v = 1\n', - 'monitor' - ) - const sealed = await commit('README.md', 'base\n', 'base') - const sameCode = await commit('README.md', 'an unrelated merge\n', 'unrelated') - const changedCode = await commit( - 'cloud/apps/relay-ops/src/incident-monitor.ts', - 'export const v = 2\n', - 'monitor change' - ) - // Branches before the seal, so the seal is not in its history even though its code matches. - gitIn(root, 'checkout', '--quiet', '--detach', base) - const sibling = await commit('README.md', 'a divergent line\n', 'divergent') - return { root, sealed, sameCode, changedCode, sibling } -} - -const authorityAt = (directory, commitSha, repositoryRoot) => verifyDryRunAuthority( - [ - '--directory', - directory, - ...provenanceFor(commitSha), - '--required-migration-policy', - 'strict' - ], - () => now, - repositoryRoot -) - -test('accepts dry-run evidence sealed by identical code at an ancestor commit', async () => { - const repository = await trustedCodeRepository() - const directory = await evidenceDirectory() - try { - await createEvidenceManifest([ - '--directory', - directory, - ...provenanceFor(repository.sealed) - ]) - // An exact match never consults git: a root with no checkout at all still verifies. - await assert.doesNotReject(authorityAt(directory, repository.sealed, directory)) - await assert.doesNotReject(authorityAt(directory, repository.sameCode, repository.root)) - } finally { - await rm(repository.root, { recursive: true, force: true }) - await rm(directory, { recursive: true, force: true }) - } -}) - -test('rejects dry-run evidence whose monitor code or lineage differs', async () => { - const repository = await trustedCodeRepository() - const directory = await evidenceDirectory() - try { - await createEvidenceManifest([ - '--directory', - directory, - ...provenanceFor(repository.sealed) - ]) - await assert.rejects( - authorityAt(directory, repository.changedCode, repository.root), - /code changed after it was sealed: cloud\/apps\/relay-ops\/src\/incident-monitor\.ts/ - ) - await assert.rejects( - authorityAt(directory, repository.sibling, repository.root), - /is not an ancestor of/ - ) - // Fails closed: a shallow clone that never fetched the sealed commit proves nothing. - await assert.rejects( - authorityAt(directory, 'f'.repeat(40), repository.root), - /unknown to this checkout/ - ) - // Fails closed: no checkout to compare against. - await assert.rejects( - authorityAt(directory, repository.sameCode, directory), - /cannot be compared without a git checkout/ - ) - } finally { - await rm(repository.root, { recursive: true, force: true }) - await rm(directory, { recursive: true, force: true }) - } -}) - -test('keeps restore and mutation bound to the exact sealing commit', async () => { - const repository = await trustedCodeRepository() - const directory = await evidenceDirectory() - try { - await createEvidenceManifest([ - '--directory', - directory, - ...provenanceFor(repository.sealed) - ]) - await assert.rejects( - verifyRestoredEvidence([ - '--directory', - directory, - ...provenanceFor(repository.sameCode) - ]), - /provenance does not match/ - ) - await assert.rejects( - verifyMutationEvidence( - [ - '--directory', - directory, - ...provenanceFor(repository.sameCode), - '--mutation-mode', - 'execute', - '--source-cell-id', - 'c1', - '--director-origin', - 'https://relay.example' - ], - { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, - async () => Response.json({ selector }), - () => now - ), - /provenance does not match/ - ) - } finally { - await rm(repository.root, { recursive: true, force: true }) - await rm(directory, { recursive: true, force: true }) - } -}) - -// A trusted path that no longer exists silently stops being compared, so the same-code rule would -// pass over code it was written to pin. -test('every trusted provenance path exists in this checkout', async () => { - for (const path of TRUSTED_EVIDENCE_CODE_PATHS) { - await assert.doesNotReject( - stat(new URL(path, RELAY_REPOSITORY_ROOT)), - `${path} is missing` - ) - } -}) diff --git a/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json b/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json deleted file mode 100644 index 0593c68ab63..00000000000 --- a/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json +++ /dev/null @@ -1 +0,0 @@ -{"source":"production cell runtime metrics, read-only, 2026-10-18T06:00Z to 2026-10-19T08:00Z; per-minute sums of authFailuresDelta and clientAcceptCompletedDelta by the minute each 30 s line ends in","cells":{"production-gce-c7":{"authFailuresPerMinute":[0,1,0,1,2,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,1,1,0,2,1,1,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,2,1,0,1,1,0,2,2,1,0,1,0,0,0,0,0,0,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,1,0,0,2,1,1,0,3,2,2,1,3,5,1,0,1,2,1,2,1,2,0,5,4,1,1,4,0,1,0,1,3,2,4,2,1,3,2,5,1,3,1,2,1,1,2,2,1,1,3,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,4,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,2,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,1,0,0,0,1,2,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,2,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,2,1,0,2,0,0,0,2,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,1,0,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,1,1,0,1,0,2,2,2,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,2,3,0,0,0,0,0,1,0,0,1,0,0,1,0,0,1,0,0,2,0,1,0,1,0,0,0,0,0,1,0,3,0,1,1,1,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,4,1,1,1,3,1,1,1,0,1,1,0,1,2,2,2,0,0,1,6,0,1,1,1,1,2,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,1,0,4,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0],"completedAcceptsPerMinute":[2,0,1,2,0,2,1,5,1,1,2,3,1,4,0,0,2,4,1,0,0,0,1,1,2,2,2,1,2,2,1,1,0,0,2,0,0,2,1,1,1,2,0,0,1,2,1,0,2,1,0,1,1,0,1,0,1,0,1,0,0,0,1,1,4,0,1,0,1,0,2,0,0,0,0,0,2,2,0,1,1,1,1,1,0,0,0,1,2,1,0,0,0,1,0,1,3,0,0,3,0,0,1,1,0,2,1,1,2,1,2,0,1,1,2,1,3,2,1,1,0,1,0,1,2,4,2,3,4,0,1,1,3,2,0,1,2,1,2,2,2,3,4,2,3,1,3,1,0,2,2,0,0,3,1,1,2,1,3,2,2,2,2,3,1,3,1,3,3,1,1,2,0,0,3,0,4,2,3,3,1,2,2,4,3,3,2,3,4,4,0,4,4,3,2,1,0,2,1,2,3,2,3,3,1,4,2,2,2,3,3,1,4,2,1,4,2,3,4,2,0,1,4,2,1,0,0,2,0,1,4,2,1,4,3,3,3,0,2,3,3,3,2,3,5,2,5,3,1,3,0,5,2,1,4,1,6,1,3,7,4,4,1,4,3,1,1,2,2,3,3,4,0,3,0,2,2,3,2,1,2,3,0,2,0,2,2,1,6,2,4,1,2,4,1,1,3,1,1,3,3,5,3,1,2,2,2,3,2,2,3,2,5,6,2,3,3,2,5,3,2,1,5,3,2,3,6,8,6,1,2,2,1,5,6,8,1,2,7,6,4,3,5,4,6,2,5,5,2,3,2,1,5,1,2,3,6,4,1,4,1,1,5,2,1,3,2,4,3,2,1,1,4,4,2,3,4,2,3,3,1,2,0,4,1,4,3,2,0,4,2,2,1,2,2,1,1,6,2,4,3,1,0,1,1,4,0,1,1,4,1,4,4,1,1,3,2,2,3,3,1,0,1,2,0,2,2,1,2,1,2,2,1,7,4,1,3,0,0,2,2,1,1,1,0,3,7,2,2,0,0,3,0,2,2,1,1,1,4,2,1,3,2,4,0,0,1,5,3,4,2,5,4,2,1,1,2,7,4,6,1,4,2,1,5,0,2,5,2,0,2,3,4,4,1,0,1,0,4,1,4,3,2,2,3,3,2,3,3,3,2,0,1,3,4,3,2,5,3,1,6,2,4,2,3,2,2,2,3,1,2,2,0,6,2,0,2,4,5,1,1,0,2,5,1,2,4,3,4,2,0,2,1,3,4,3,3,3,2,3,5,2,6,2,3,1,3,4,3,1,2,1,3,5,4,0,2,1,1,0,1,1,3,3,3,2,3,2,2,0,2,2,1,4,1,2,4,2,3,1,2,2,3,1,5,0,4,6,1,0,2,5,3,3,5,0,3,1,3,3,7,3,3,1,5,4,4,6,1,6,1,5,2,1,1,1,1,1,4,2,3,5,2,2,1,4,1,1,2,2,1,2,0,0,3,4,2,0,2,4,5,3,3,1,1,1,2,3,1,2,1,4,2,1,3,0,0,2,2,0,0,0,1,1,0,0,0,0,4,4,3,1,1,1,2,1,0,1,1,1,2,4,2,3,3,1,1,2,4,0,1,1,1,2,2,4,2,2,3,2,2,3,1,1,1,5,4,3,4,1,3,3,1,1,1,0,4,3,5,3,1,4,2,3,1,2,3,5,2,3,2,2,4,1,2,5,2,1,4,1,2,3,4,3,2,3,2,2,3,1,0,0,0,1,2,2,0,3,1,2,1,0,0,4,1,0,4,0,0,1,3,0,0,1,1,1,2,0,3,2,0,2,1,3,3,0,2,0,1,2,4,4,3,1,3,5,1,3,0,1,1,3,4,1,0,3,1,3,4,1,3,2,3,3,1,3,3,3,4,7,2,4,3,4,1,2,0,2,3,1,2,3,2,3,0,2,2,2,7,2,3,1,1,3,2,1,3,3,1,1,1,2,4,1,2,6,1,0,0,3,0,4,0,0,2,2,3,3,1,2,0,3,1,1,3,1,5,1,2,4,2,1,1,3,6,5,2,0,4,2,5,0,2,1,4,2,2,3,2,1,7,3,5,0,4,3,0,2,1,3,2,2,1,1,0,2,1,4,2,2,1,1,2,4,3,1,0,1,2,1,2,0,1,5,0,3,3,2,2,1,0,2,1,5,0,5,1,3,2,3,2,1,3,2,3,0,0,2,2,1,1,2,0,0,3,3,2,2,0,5,2,5,2,2,4,1,2,5,0,3,4,2,1,5,3,1,5,3,0,1,3,4,5,2,2,2,4,1,1,1,3,0,2,1,5,1,1,1,1,1,4,3,1,1,1,1,4,3,0,2,5,2,4,3,1,0,3,2,4,5,1,4,2,2,4,2,3,3,4,3,1,3,1,1,4,1,3,1,2,0,4,4,3,3,2,1,2,1,1,2,0,3,1,2,3,2,1,1,0,1,1,0,1,2,3,1,3,3,1,0,2,4,1,2,3,2,1,1,2,2,1,0,1,1,3,0,0,0,3,3,3,1,0,4,0,3,1,3,1,1,4,2,3,0,1,2,2,1,0,0,4,1,3,0,3,2,2,2,0,1,1,1,1,2,3,2,0,1,2,0,2,2,1,0,1,0,2,3,2,2,1,3,3,0,2,3,2,2,6,3,1,1,0,2,2,1,0,2,3,1,0,0,1,2,1,1,0,0,1,0,1,0,2,2,3,1,2,0,1,0,0,3,0,0,2,1,2,0,2,0,2,1,3,0,0,2,3,0,1,0,0,0,0,2,1,0,4,2,0,1,0,3,3,2,0,1,2,2,1,1,1,0,1,2,2,1,3,2,1,3,0,1,1,2,2,0,2,0,2,2,1,1,2,1,0,0,1,1,2,1,0,2,1,3,2,0,2,4,2,1,0,2,1,1,1,0,2,2,1,2,2,3,0,1,2,2,3,3,2,1,2,2,5,0,0,3,1,3,0,1,1,3,3,2,2,1,0,2,3,4,2,4,2,1,1,2,1,0,2,1,0,1,2,1,3,1,4,2,4,1,3,3,2,1,3,0,3,1,3,0,1,3,4,1,5,0,4,2,4,6,3,1,1,1,4,2,2,5,3,0,0,2,1,1,0,1,2,3,0,3,0,1,2,0,4,1,2,2,2,4,1,1,0,4,1,1,1,3,0,1,2,3,2,2,4,1,2,1,1,2,3,2,2,2,1,0,0,0,0,3,0,1,2,0,1,2,1,2,1,0,2,4,3,4,4,3,4,3,3,3,2,0,0,0,1,3,1,2,4,0,4,4,1,1,3,1,1,1,0,1,1,2,2,3,0,1,0,3,0,0,1,2,1,2,2,4,1,1,3,1,0,0,3,1,2,1,1,2,3,2,3,0,0,7,2,2,0,3,1,0,3,1,1,1,2,1,3,0,4,1,1,0,2,4,1,2,1,0,4,2,2,0,1,1,2,1,1,0,1,1,1,3,0,1,1,1,4,2,0,2,2,2,2,4,0,2,1,1,3,0]},"production-gce-c28":{"authFailuresPerMinute":[2,0,0,0,0,0,6,4,3,3,5,3,2,5,5,3,2,1,0,1,1,0,0,1,1,3,2,1,0,0,0,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,2,0,1,0,0,0,1,0,0,0,1,1,0,2,0,1,0,0,0,0,1,1,3,0,1,1,2,1,1,0,0,0,1,7,3,5,4,5,6,3,0,1,3,2,2,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,2,0,0,0,0,0,3,0,0,1,1,0,0,2,0,0,1,3,1,4,5,1,5,6,3,5,2,0,0,0,0,0,8,4,4,8,8,10,7,7,4,7,0,1,1,1,0,0,0,0,0,0,1,0,1,0,0,1,1,0,1,1,5,12,8,5,4,3,5,5,3,3,0,1,3,3,0,0,1,1,2,2,1,0,0,0,1,0,2,0,1,0,1,0,0,0,0,0,0,1,1,0,0,4,2,0,2,0,0,0,0,0,0,0,0,0,3,0,0,1,0,0,1,0,0,2,5,3,0,0,0,0,1,0,0,0,1,1,2,2,2,1,0,0,0,3,0,0,1,0,0,0,0,0,0,0,1,0,2,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,1,0,1,0,2,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,0,2,0,1,0,1,0,0,0,0,0,0,0,0,4,3,2,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,1,2,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,2,8,4,3,2,1,1,0,1,0,0,2,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,1,0,0,1,1,0,7,2,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,2,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,1,1,1,0,0,0,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,2,1,0,0,2,0,1,0,0,0,0,1,0,1,0,1,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,1,2,1,0,0,0,0,1,0,0,0,0,2,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,3,7,3,2,0,0,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,1,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,5,0,0,0,2,0,0,1,1,0,0,0,0,0,0,2,4,2,6,2,3,2,3,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,1,1,0,1,1,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,6,2,0,0,0,1,1,0,0,0,0,0,0,1,0,1,2,0,0,1,1,1,0,2,1,0,0,1,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,1,1,2,0,1,1,2,0,0,0,2,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,3,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,2,0,0,0,0,1,1,0,0,0,0,1,0,1,1,2,1,1,1,0,1,0,0,1,0,0,1,0,0,1,0,1,1,0,2,1,0,0,0,1,0,0,2,0,0,1,0,0,0,0,2,2,0,0,0,1,0,0,1,2,0,0,0,2,0,0,1,0,2,0,1,3,0,0,0,1,0,1,0,0,0,0,1,2,3,0,0,0,0,0,1,0,0,1,0,1,0,0,6,2,0,5,2,0,0,0,4,5,3,1,0,5,4,0,0,3,0,1,0,1,0,5,0,2,3,6,3,6,2,0,5,3,9,0,0,3,0,1,6,3,0,2,0,5,1,1,1,4,1,1,1,0,0,0,0,1,0,1,2,1,0,5,4,5,2,0,1,0,0,0,1,1,6,5,3,6,8,6,4,9,3,6,3,3,0,3,14,5,11,6,5,11,8,5,6,10,4,1,9,14,7,4,14,6,5,4,4,1,9,9,11,6,1,0,0,2,0,0,0,1,1,0,0,0,1,0,0,1,3,3,1,0,2,2,1,1,0,0,0,0,3,4,0,3,0,2,0,3,1,2,0,0,0,1,1,0,0,1,2,1,2,1,11,5,3,0,0,1,8,8,1,4,4,0,4,0,1,6,7,6,5,9,5,3,3,5,2,7,1,2,3,1,0,1,0,1,0,2,1,5,4,5,2,1,0,0,4,2,0,0,1,0,1,0,0,0,1,1,1,0,0,0,0,0,5,6,0,1,5,2,1,0,0,0,1,2,2,1,0,0,0,2,5,2,0,0,0,0,0,0,0,0,1,2,0,0,0,1,1,0,0,0,2,0,0,0,1,0,4,2,0,1,0,2,1,1,9,2,0,0,0,1,1,0,0,1,3,2,4,3,6,2,2,1,1,0,0,2,1,0,3,6,0,2,1,0,0,0,2,1,2,5,1,2,0,3,0,0,1,0,0,0,1,0,2,2,0,0,0,1,1,0,0,0,0,0,0,1,0,1],"completedAcceptsPerMinute":[17,10,8,8,13,10,8,10,16,16,31,20,15,17,14,14,17,19,22,20,20,12,15,23,12,16,11,20,21,19,23,17,19,25,18,19,13,12,16,8,11,10,15,15,19,6,9,7,7,11,14,16,15,14,9,10,11,7,10,5,12,12,11,12,6,7,10,6,10,4,10,7,10,7,7,13,14,5,11,4,9,7,9,9,10,17,11,14,16,9,15,12,12,1,8,8,9,7,10,17,14,13,7,10,16,13,8,21,18,13,14,7,8,14,10,14,12,14,16,19,16,22,21,15,19,18,16,16,9,8,14,9,10,14,10,12,14,8,7,13,9,9,10,8,9,7,14,16,15,11,17,6,8,18,6,12,10,5,11,11,8,10,8,7,13,16,8,9,13,9,15,8,18,14,13,9,9,5,7,9,3,8,8,6,7,13,15,17,12,8,9,7,10,11,15,16,12,15,16,16,14,9,6,6,16,11,10,12,16,14,18,11,11,16,11,8,8,2,12,7,10,8,8,13,3,8,8,15,11,6,9,7,11,14,7,15,7,6,6,10,15,8,10,10,10,10,8,7,5,5,11,9,18,12,10,19,16,12,14,8,9,6,10,20,11,10,12,22,11,20,10,17,6,14,19,12,14,7,18,12,9,8,8,11,10,9,7,1,6,6,11,6,6,9,8,13,5,8,16,17,10,8,12,11,8,16,8,9,12,10,7,5,6,8,13,12,8,7,13,6,5,12,11,8,10,14,6,13,11,11,7,10,8,2,2,6,4,9,9,9,7,8,13,6,6,10,6,6,6,5,2,9,7,4,9,10,5,9,7,9,7,4,6,7,13,10,8,6,10,7,8,14,7,9,6,5,7,13,9,14,9,5,12,17,13,19,21,8,15,12,10,18,13,12,11,13,9,8,9,13,8,12,7,6,8,3,4,4,4,9,5,10,8,4,12,13,19,10,8,9,13,4,5,4,4,1,8,9,2,5,7,7,12,10,6,7,4,9,5,9,8,13,11,10,6,11,10,8,6,15,14,9,20,24,17,12,10,6,10,8,10,6,8,5,4,3,12,9,3,7,9,5,5,4,11,6,8,7,6,7,13,4,5,5,9,6,8,4,14,9,11,12,9,5,10,6,6,7,9,7,8,11,5,6,6,9,5,8,11,7,3,5,6,7,7,9,5,12,8,8,7,5,8,5,5,10,4,10,6,6,6,2,3,7,5,12,4,3,7,6,5,5,7,10,2,7,8,12,5,7,11,5,6,8,6,12,10,4,10,7,9,7,12,14,11,14,7,11,4,9,5,7,5,4,6,13,9,3,5,6,6,12,3,4,5,9,11,5,5,4,9,11,16,9,9,13,7,4,5,9,4,5,5,7,4,4,4,1,3,4,11,7,4,3,4,3,5,2,4,3,4,4,3,3,5,4,5,6,7,3,3,2,5,2,6,4,4,3,2,2,4,6,4,1,6,7,2,2,3,4,4,1,2,3,3,4,3,3,6,2,2,4,3,3,1,6,3,1,3,0,7,3,3,2,1,1,1,3,4,1,0,1,1,4,1,1,2,3,4,3,2,2,3,1,4,4,2,0,2,1,0,2,3,1,3,1,1,3,5,3,3,3,2,1,2,6,1,0,1,4,0,5,6,3,2,0,5,1,4,1,0,0,1,4,2,2,4,0,4,0,1,1,1,1,0,1,2,2,2,1,0,1,2,2,5,3,0,1,3,0,3,2,2,4,1,2,1,1,1,2,1,1,2,1,2,4,1,2,0,1,1,0,1,0,1,3,1,1,1,0,2,0,1,0,1,0,0,0,0,3,0,3,3,0,2,1,0,1,0,2,4,6,1,0,3,1,0,0,2,0,2,1,1,2,2,1,0,0,4,1,0,4,2,2,3,1,0,3,1,2,4,2,3,3,1,1,3,5,0,2,3,4,2,0,0,1,1,0,2,1,2,1,0,1,0,1,0,1,1,0,0,2,1,2,2,1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,3,0,1,4,0,1,2,0,0,2,0,0,0,1,0,0,0,1,1,0,1,5,1,0,0,0,0,1,2,1,1,1,1,1,1,0,1,1,3,2,1,1,2,0,2,0,3,0,2,1,0,1,1,5,4,3,5,1,0,2,4,1,1,3,5,0,2,0,2,0,2,3,1,1,3,2,2,3,4,3,2,1,4,2,1,2,2,2,2,1,4,1,3,4,2,2,4,2,3,3,3,0,3,2,2,3,3,6,5,7,8,3,2,3,3,2,2,4,2,4,5,1,1,2,2,4,3,4,2,2,4,6,7,5,4,2,3,2,3,6,4,5,7,5,2,9,4,5,7,7,6,5,8,9,7,6,6,6,5,9,9,7,8,4,6,7,4,2,2,5,5,9,9,4,4,3,6,2,8,4,6,8,3,2,3,6,6,11,6,8,1,5,2,3,11,7,0,6,5,3,5,8,6,6,8,6,5,5,9,11,6,7,6,5,5,5,6,7,6,2,5,7,6,7,5,9,2,6,19,4,2,4,6,7,8,6,11,5,11,7,5,7,10,7,7,1,10,10,5,7,12,8,9,12,11,9,7,11,8,8,5,12,10,8,5,8,5,11,3,7,7,5,5,5,7,3,8,5,7,11,9,9,5,8,10,8,11,20,16,16,20,15,14,13,16,13,14,15,12,16,13,13,12,8,6,6,6,7,6,8,6,12,11,5,5,7,10,3,5,9,8,4,8,5,9,4,11,7,10,11,12,7,7,14,9,14,8,6,10,7,7,10,6,11,11,6,5,11,5,9,11,10,9,8,10,10,10,7,8,10,11,10,10,7,12,9,13,11,12,9,13,8,16,10,10,5,5,7,13,10,7,10,6,10,3,9,9,14,6,9,11,4,14,12,11,8,7,15,6,16,12,9,13,14,10,21,13,8,16,11,10,13,4,8,12,7,16,14,9,12,11,10,7,13,13,11,11,13,6,7,9,13,8,9,7,11,12,7,9,9,10,13,9,12,6,8,8,8,12,8,12,2,6,1,10,9,13,13,10,2,11,12,11,7,13,10,3,8,11,7,12,8,10,7,16,17,11,7,14,7,5,8,3,8,7,11,8,6,9,11,11,7,7,5,10,9,14,4,8,10,8,17,9,6,6,5,9,11,5,10,11,10,11,10,12,16,8,8,2,9,4,9,6,13,6,7,4,11,10,4,9,9,11,6,13,10,6,10,12,12,13,16,6,8,5,7,6,4,7,13,5,10,6,5,5,9,7,6,7,11,10,5,6,7,9,3,8,8,14,7,10,11,5,8,10,14,15,16,17,9,9,9,14,14,20,13,14,8,9,5,8,14,13,19,24,13,14,15,14,11,14,13,15,14,7,7,10,9,8,17,16,14,20,12,20,14,15,15,9,5,10,5,10,11,7,9,8,4,5,6,5,9,11,5,6,2,5,6,8,13,4,9,4,8,4,4,8,5,10,9,9,5,14,14,11,11,8,8,4,12,10,13,13,9,10,10,9,5,4,7]},"production-gce-c29":{"authFailuresPerMinute":[0,1,0,1,0,0,4,3,2,5,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,9,6,0,0,4,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,1,3,0,0,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,2,4,0,0,4,5,0,0,0,0,1,5,1,5,2,4,4,5,2,3,1,1,6,3,4,4,1,0,0,0,0,3,7,1,0,0,0,0,0,6,1,1,7,0,6,5,2,0,0,1,0,0,0,0,0,0,3,4,0,1,4,0,0,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,1,0,0,0,1,1,0,2,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,3,0,1,0,0,0,2,0,0,3,4,5,3,1,0,0,0,0,0,2,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,4,3,0,0,2,2,0,0,2,2,0,0,0,1,0,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,2,0,2,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,1,0,0,2,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,2,0,0,0,0,1,0,0,1,0,0,0,0,3,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,4,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,1,0,1,0,0,1,1,0,0,1,0,0,0,0,2,0,6,4,2,0,0,1,0,2,1,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,1,0,0,1,4,1,0,0,3,1,0,1,0,0,0,6,2,6,5,8,4,0,0,0,6,4,8,4,3,0,1,0,0,1,0,1,0,1,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,2,3,0,0,0,0,0,0,2,0,1,0,1,0,1,0,0,1,0,2,0,0,2,0,0,1,0,0,0,0,0,0,2,6,3,7,0,1,0,0,0,1,0,2,2,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,6,4,0,0,0,0,0,3,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,2,0,0,0,2,0,0,1,1,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,2,1,1,3,0,2,0,0,0,1,2,0,0,0,1,0,0,1,2,0,0,0,0,3,0,0,0,0,3,0,0,0,2,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,3,2,0,0,2,0,0,0,1,1,0,0,1,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,2,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,1,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,0,0,1,0,0,1,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,2,1,0,0,0,0,1,0,0,1,0,0,2,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,2,0,0,0,2,0,1,0,1,0,0,1,0,0,0,2,0,0,0],"completedAcceptsPerMinute":[3,7,7,8,8,7,4,8,12,4,9,8,10,8,6,5,11,8,10,12,6,7,11,7,12,7,6,11,10,7,9,8,3,10,10,9,6,4,11,9,9,10,16,4,11,11,7,7,9,8,7,11,10,9,5,6,6,7,12,12,7,6,12,6,11,18,3,9,7,8,8,4,10,5,13,10,8,12,8,13,14,3,7,5,7,8,8,4,4,8,11,3,9,9,8,3,7,4,3,7,5,5,9,8,6,7,6,3,9,9,6,10,5,7,15,5,9,11,10,14,2,7,8,10,12,7,19,8,9,9,5,7,5,5,8,11,9,10,8,5,4,11,7,8,13,10,15,12,5,11,15,9,12,8,9,3,11,9,11,9,7,15,11,8,5,8,6,8,8,1,7,5,10,9,7,7,7,9,11,7,13,10,7,9,9,4,6,7,9,7,7,5,10,13,8,6,4,11,5,9,9,7,2,9,1,8,5,8,10,12,10,12,4,8,4,8,7,9,7,3,4,4,3,9,3,8,12,5,6,12,15,8,16,14,9,9,12,5,15,7,8,6,7,11,9,12,3,6,6,7,6,7,3,2,7,10,9,16,2,14,21,11,15,12,10,6,13,1,12,7,15,5,7,11,15,6,2,10,9,10,15,13,14,12,17,12,11,13,8,4,14,10,7,3,4,6,3,5,9,8,10,10,3,7,8,6,8,6,8,12,7,4,9,7,9,5,8,6,10,5,3,6,6,5,3,8,12,5,8,7,12,4,14,13,12,9,14,16,13,14,11,7,8,4,3,8,9,3,5,5,12,7,8,6,11,10,12,13,7,14,17,16,13,15,17,9,14,7,6,9,9,9,11,8,8,7,8,2,12,7,10,6,7,5,5,7,11,7,11,7,3,5,4,9,8,7,9,5,7,8,7,9,5,9,16,15,23,20,14,11,10,5,4,13,14,13,10,16,15,20,16,21,16,17,20,15,16,21,19,13,12,9,11,10,7,12,12,4,9,7,4,14,9,14,15,16,14,9,7,3,6,8,9,6,12,7,12,5,12,10,13,6,10,8,10,6,5,4,8,6,13,6,10,9,6,8,7,7,12,6,5,6,10,12,15,14,9,12,13,17,13,4,3,4,6,8,6,8,9,8,13,3,2,3,6,6,10,6,3,10,4,8,13,5,2,4,2,12,5,6,5,9,8,7,5,7,4,4,5,9,12,14,15,9,8,2,4,6,6,2,6,4,6,1,6,5,4,3,3,6,7,1,3,2,6,1,5,9,5,3,1,4,4,5,6,11,5,4,4,4,4,4,6,4,3,6,5,6,4,4,2,7,6,2,7,4,7,6,4,3,3,4,8,5,0,3,5,4,1,5,1,2,2,5,1,8,3,2,6,1,2,3,5,3,2,4,6,5,3,2,3,2,1,3,2,5,4,3,1,3,2,5,4,2,5,1,2,3,3,2,0,3,0,1,3,5,3,1,4,1,6,3,4,2,2,4,2,2,4,2,3,2,5,4,3,2,2,1,6,10,9,10,7,4,1,2,6,8,18,14,15,16,15,19,15,17,13,18,14,8,3,1,1,0,5,1,3,1,0,0,1,0,3,2,0,1,1,6,6,3,3,3,1,2,2,2,4,2,0,1,4,2,0,1,1,0,1,1,2,1,4,1,4,1,6,3,1,1,2,3,1,0,1,2,0,0,1,0,4,2,1,2,1,1,1,1,3,2,0,1,4,0,0,5,0,1,1,2,1,1,3,2,2,3,3,3,2,0,3,2,1,3,0,2,4,3,3,5,5,3,1,0,2,1,2,0,0,1,4,2,0,1,2,2,2,1,1,1,1,2,2,4,2,3,1,5,1,2,3,3,1,0,1,1,1,0,0,2,2,0,1,3,3,4,2,0,0,2,1,2,2,0,1,1,0,0,0,0,1,1,1,2,0,2,0,2,0,1,0,1,3,1,2,1,0,0,2,1,0,1,1,2,0,1,1,1,0,0,0,1,2,1,2,2,0,0,1,1,1,4,0,1,0,1,5,2,0,2,0,1,1,2,3,1,1,1,2,2,2,3,2,0,2,1,1,0,0,2,4,2,3,0,0,0,0,2,8,2,1,1,1,1,1,1,2,0,2,0,1,1,0,0,0,1,2,1,3,2,3,1,1,3,4,5,0,3,1,2,0,3,2,4,1,3,3,3,2,4,1,5,5,6,1,3,1,2,2,0,4,13,5,8,3,2,2,1,4,2,3,2,4,0,4,5,2,7,5,0,6,5,6,4,3,8,7,4,2,5,3,12,11,12,7,5,11,17,15,12,11,7,6,4,3,8,3,3,4,2,2,2,5,8,4,7,5,4,1,5,3,2,6,2,5,7,16,3,5,4,11,6,12,7,8,6,8,3,7,2,3,6,8,2,5,11,6,6,7,7,4,7,7,11,4,5,3,7,7,0,3,8,5,9,11,4,7,4,15,5,4,2,6,5,7,4,5,10,3,5,10,6,10,6,8,5,8,6,10,9,12,5,7,12,6,8,5,7,9,8,6,6,15,12,17,13,8,14,5,5,9,7,4,8,9,7,10,6,8,7,10,9,10,6,8,10,4,3,9,4,11,4,5,6,9,9,10,8,6,8,5,8,4,4,5,11,3,2,5,3,4,3,5,7,4,8,1,5,6,7,6,14,11,9,11,7,6,8,12,4,10,6,9,13,12,11,2,7,11,7,10,7,8,9,8,9,8,12,18,12,9,14,13,14,16,12,13,8,8,14,14,3,12,5,15,9,12,9,6,8,15,11,9,5,8,11,12,8,8,6,4,10,6,7,17,12,8,9,11,11,8,10,11,6,13,9,11,15,11,12,6,8,9,11,14,7,10,7,10,6,7,13,5,10,14,16,15,20,9,7,7,10,9,13,10,6,10,10,15,16,18,16,12,6,7,10,9,6,7,15,10,6,13,14,25,10,10,8,9,10,8,7,12,13,6,5,6,8,7,12,8,12,11,14,6,10,10,13,11,14,14,9,16,14,10,5,6,8,6,12,8,8,7,8,9,7,7,14,7,7,8,8,4,7,5,6,8,9,24,10,12,14,10,12,9,6,10,14,6,13,7,16,8,9,13,10,15,11,14,14,11,10,16,9,12,9,9,11,6,10,13,4,13,11,10,6,9,11,10,6,7,8,9,7,7,6,7,7,8,10,12,6,12,10,10,6,7,9,11,5,3,6,6,6,10,9,8,9,7,7,14,5,7,7,3,4,11,8,5,10,3,6,12,4,6,4,9,7,7,4,13,6,7,10,7,9,5,8,2,9,8,9,7,6,6,10,10,13,15,8,7,7,11,4,6,9,4,17,12,17,15,12,13,5,13,12,6,18,10,12,8,8,3,6,12,7,4,9,6,8,5,5,18,9,13,15,8,6,7,6,6,8,9,9,11,6,7,3,8,5,4,2,2,9,7,4,6,8,6,12,3,7,11,7,4,10,11,8,4,9,12,15,10,8,8,5,6,7,5,3,7,6,3,3,10,8,3,7,12,9,6,4]}}} diff --git a/cloud/dev/scripts/relay-production-capacity-wave.mjs b/cloud/dev/scripts/relay-production-capacity-wave.mjs deleted file mode 100644 index e76c264e5ed..00000000000 --- a/cloud/dev/scripts/relay-production-capacity-wave.mjs +++ /dev/null @@ -1,172 +0,0 @@ -import { readFile, writeFile } from 'node:fs/promises' -import { pathToFileURL } from 'node:url' -import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs' - -const APPROVED_CELLS = new Set(PRODUCTION_CAPACITY_CELL_IDS) - -function argumentsByName(argv, allowed) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const argument = argv[index] - const value = argv[index + 1] - if (!argument?.startsWith('--') || !value || value.startsWith('--')) { - throw new Error('capacity wave arguments are invalid') - } - const name = argument.slice(2) - if (!allowed.has(name) || name in values) { - throw new Error(`capacity wave argument --${name} is invalid`) - } - values[name] = value - } - return values -} - -export function parseCapacityWave(waveCellIds, confirmation) { - const cells = waveCellIds.split(',') - if ( - cells.length < 2 || - cells.length > 4 || - cells.some((cell) => !APPROVED_CELLS.has(cell)) || - new Set(cells).size !== cells.length || - cells.join(',') !== waveCellIds - ) { - throw new Error('capacity wave must contain two to four unique approved cells') - } - if (confirmation !== `RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}`) { - throw new Error('capacity wave confirmation does not match the selected cells') - } - return cells -} - -function exactMembership(membership) { - const keys = ['existingOnly', 'migrationOnly', 'general'] - if (!keys.every((key) => Array.isArray(membership?.[key]))) return false - const cells = keys.flatMap((key) => membership[key]) - return cells.every((cell) => typeof cell === 'string') && new Set(cells).size === cells.length -} - -export function capacityWavePreflightState(state, waveCellIds, waveIndex, targetCellId) { - const cells = parseCapacityWave( - waveCellIds, - `RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}` - ) - const index = Number(waveIndex) - const membership = state.expectedSelector?.membership - if ( - !Number.isSafeInteger(index) || - index < 0 || - index >= cells.length || - targetCellId !== cells[index] || - state.schemaVersion !== 4 || - state.environment !== 'production' || - state.preDrainDryRun !== true || - state.migrationPolicy !== 'capacity-transition' || - state.recoverySourceCellId !== null || - state.capacityCellId !== cells[0] || - !Number.isSafeInteger(state.expectedSelector?.generation) || - !exactMembership(membership) || - !cells.every((cell) => membership.general.includes(cell)) || - state.sampleCount < 16 || - state.frozenAt !== null || - typeof state.completedAt !== 'string' - ) { - throw new Error('capacity wave evidence does not match this step') - } - return { - schemaVersion: 4, - environment: 'production', - expectedSelector: { - generation: state.expectedSelector.generation + index * 2, - membership - }, - migrationPolicy: 'capacity-transition', - recoverySourceCellId: null, - capacityCellId: targetCellId - } -} - -export function capacityWaveResumePreflightState(state, waveCellIds, targetCellId) { - const cells = parseCapacityWave( - waveCellIds, - `RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}` - ) - const index = cells.indexOf(targetCellId) - const base = capacityWavePreflightState( - state, - waveCellIds, - String(index), - targetCellId - ) - const membership = base.expectedSelector.membership - const general = membership.general.filter((cell) => cell !== targetCellId) - const capacityCellId = general.includes(state.capacityCellId) - ? state.capacityCellId - : cells.find((cell) => general.includes(cell)) - if (!capacityCellId) throw new Error('capacity wave resume has no general evidence cell') - return { - ...base, - expectedSelector: { - generation: base.expectedSelector.generation + 1, - membership: { - existingOnly: membership.existingOnly, - migrationOnly: [...membership.migrationOnly, targetCellId].sort(), - general - } - }, - capacityCellId - } -} - -async function main(argv) { - const [command, ...arguments_] = argv - if (command === 'validate') { - const values = argumentsByName( - arguments_, - new Set(['wave-cell-ids', 'confirmation']) - ) - process.stdout.write(`${JSON.stringify(parseCapacityWave( - values['wave-cell-ids'] ?? '', - values.confirmation ?? '' - ))}\n`) - return - } - if (command === 'build-preflight' || command === 'build-resume-preflight') { - const values = argumentsByName( - arguments_, - new Set([ - 'state-file', - 'wave-cell-ids', - ...(command === 'build-preflight' ? ['wave-index'] : []), - 'target-cell-id', - 'output-file' - ]) - ) - const state = JSON.parse(await readFile(values['state-file'] ?? '', 'utf8')) - const preflight = command === 'build-preflight' - ? capacityWavePreflightState( - state, - values['wave-cell-ids'] ?? '', - values['wave-index'] ?? '', - values['target-cell-id'] ?? '' - ) - : capacityWaveResumePreflightState( - state, - values['wave-cell-ids'] ?? '', - values['target-cell-id'] ?? '' - ) - await writeFile( - values['output-file'] ?? '', - `${JSON.stringify(preflight)}\n`, - { mode: 0o600 } - ) - return - } - throw new Error('capacity wave command is invalid') -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - main(process.argv.slice(2)).catch((error) => { - console.error(error instanceof Error ? error.message : 'capacity wave failed') - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/relay-production-capacity-wave.test.mjs b/cloud/dev/scripts/relay-production-capacity-wave.test.mjs deleted file mode 100644 index 2cb8ebfa538..00000000000 --- a/cloud/dev/scripts/relay-production-capacity-wave.test.mjs +++ /dev/null @@ -1,129 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { - capacityWavePreflightState, - capacityWaveResumePreflightState, - parseCapacityWave -} from './relay-production-capacity-wave.mjs' - -const wave = [ - 'production-gce-c22', - 'production-gce-c21', - 'production-gce-c20', - 'production-gce-c19' -] - -function evidence(overrides = {}) { - return { - schemaVersion: 4, - environment: 'production', - preDrainDryRun: true, - migrationPolicy: 'capacity-transition', - recoverySourceCellId: null, - capacityCellId: wave[0], - expectedSelector: { - generation: 39, - membership: { - existingOnly: ['production-gce-c1'], - migrationOnly: ['production-gce-c17', 'production-gce-c18'], - general: [...wave, 'production-gce-c23'] - } - }, - sampleCount: 16, - frozenAt: null, - completedAt: '2026-08-11T21:00:00.000Z', - ...overrides - } -} - -test('accepts only exact confirmed waves of two to four approved cells', () => { - for (const cells of [wave.slice(0, 2), wave]) { - const value = cells.join(',') - assert.deepEqual( - parseCapacityWave(value, `RAISE_SELECTED_WAVE_TO_1000 ${value}`), - cells - ) - } - for (const [cells, confirmation] of [ - [[wave[0]], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]}`], - [[...wave, 'production-gce-c16'], `RAISE_SELECTED_WAVE_TO_1000 ${wave.join(',')}`], - [[wave[0], wave[0]], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]},${wave[0]}`], - [[wave[0], 'production-gce-c17'], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]},production-gce-c17`], - [[wave[0], ` ${wave[1]}`], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]}, ${wave[1]}`], - [wave, 'RAISE_SELECTED_WAVE_TO_1000 production-gce-c22'] - ]) { - assert.throws(() => parseCapacityWave(cells.join(','), confirmation)) - } -}) - -test('derives each continuation preflight from the sealed selector generation', () => { - for (const [index, cell] of wave.entries()) { - const state = capacityWavePreflightState( - evidence(), - wave.join(','), - String(index), - cell - ) - assert.equal(state.expectedSelector.generation, 39 + index * 2) - assert.equal(state.capacityCellId, cell) - assert.deepEqual(state.expectedSelector.membership, evidence().expectedSelector.membership) - } -}) - -test('derives an exact isolated-cell resume state from sealed wave evidence', () => { - const state = capacityWaveResumePreflightState( - evidence(), - wave.join(','), - wave[3] - ) - assert.equal(state.expectedSelector.generation, 46) - assert.equal(state.capacityCellId, wave[0]) - assert.deepEqual(state.expectedSelector.membership, { - existingOnly: ['production-gce-c1'], - migrationOnly: ['production-gce-c17', 'production-gce-c18', wave[3]].sort(), - general: [wave[0], wave[1], wave[2], 'production-gce-c23'] - }) -}) - -test('resume rejects a target outside the exact sealed wave', () => { - assert.throws( - () => capacityWaveResumePreflightState( - evidence(), - wave.join(','), - 'production-gce-c16' - ), - /does not match/ - ) -}) - -test('resume rebinds first-cell evidence to another general wave cell', () => { - const state = capacityWaveResumePreflightState( - evidence(), - wave.join(','), - wave[0] - ) - assert.equal(state.capacityCellId, wave[1]) - assert.equal(state.expectedSelector.generation, 40) - assert.ok(state.expectedSelector.membership.migrationOnly.includes(wave[0])) - assert.ok(state.expectedSelector.membership.general.includes(wave[1])) -}) - -test('rejects reordered, incomplete, frozen, or mismatched wave evidence', () => { - const calls = [ - () => capacityWavePreflightState(evidence(), wave.join(','), '1', wave[0]), - () => capacityWavePreflightState(evidence({ capacityCellId: wave[1] }), wave.join(','), '0', wave[0]), - () => capacityWavePreflightState(evidence({ sampleCount: 15 }), wave.join(','), '0', wave[0]), - () => capacityWavePreflightState(evidence({ frozenAt: '2026-08-11T20:59:00.000Z' }), wave.join(','), '0', wave[0]), - () => capacityWavePreflightState(evidence({ completedAt: null }), wave.join(','), '0', wave[0]), - () => capacityWavePreflightState(evidence({ - expectedSelector: { - ...evidence().expectedSelector, - membership: { - ...evidence().expectedSelector.membership, - general: wave.slice(1) - } - } - }), wave.join(','), '0', wave[0]) - ] - for (const call of calls) assert.throws(call, /does not match/) -}) diff --git a/cloud/dev/scripts/relay-production-capacity-workflow.test.mjs b/cloud/dev/scripts/relay-production-capacity-workflow.test.mjs deleted file mode 100644 index d482d8cd856..00000000000 --- a/cloud/dev/scripts/relay-production-capacity-workflow.test.mjs +++ /dev/null @@ -1,440 +0,0 @@ -import assert from 'node:assert/strict' -import { spawnSync } from 'node:child_process' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { readRelayWorkflow, relayWorkflowPath } from './relay-repository.mjs' -import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs' - -const dispatchWorkflow = readRelayWorkflow('deploy-relay-production-capacity.yml') -const workflow = readRelayWorkflow('deploy-relay-production-capacity-job.yml') -const terraform = source('infra/terraform/relay-github-actions.tf') -const production = source('infra/terraform/environments/production.tfvars') -const capacityCells = PRODUCTION_CAPACITY_CELL_IDS - -function source(path) { - return readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8') -} - -function resource(type, name) { - const start = terraform.indexOf(`resource "${type}" "${name}"`) - assert.notEqual(start, -1, `${type}.${name} is missing`) - const next = terraform.indexOf('\nresource "', start + 1) - return terraform.slice(start, next === -1 ? undefined : next) -} - -function ordered(...markers) { - let previous = -1 - for (const marker of markers) { - const current = workflow.indexOf(marker) - assert.ok(current > previous, `${marker} is missing or out of order`) - previous = current - } -} - -function mutationConfirmation(mode, targetCellId, confirmation) { - const stepStart = workflow.indexOf(' - name: Require exact mutation confirmation') - const runMarker = ' run: |\n' - const runStart = workflow.indexOf(runMarker, stepStart) + runMarker.length - const runEnd = workflow.indexOf('\n - name:', runStart) - const script = workflow.slice(runStart, runEnd).replace(/^ {10}/gm, '') - return spawnSync('bash', ['-euo', 'pipefail', '-c', script], { - env: { - ...process.env, - DEPLOY_MODE: mode, - TARGET_CELL_ID: targetCellId, - CONFIRMATION: confirmation - } - }).status -} - -function imageCompatibility(activeImage, desiredImage) { - const start = workflow.indexOf(' ACTIVE_IMAGE_DIGEST="${ACTIVE_IMAGE##*@}"') - const end = workflow.indexOf('\n CURRENT_CELLS_JSON=', start) - const script = workflow.slice(start, end).replace(/^ {10}/gm, '') - return spawnSync('bash', ['-euo', 'pipefail', '-c', script], { - env: { - ...process.env, - ACTIVE_IMAGE: activeImage, - DESIRED_IMAGE: desiredImage, - DESIRED_IMAGE_DIGEST: desiredImage.split('@').at(-1), - COMPATIBLE_DIRECTOR_IMAGE_DIGEST: 'sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73', - COMPATIBLE_CELL_IMAGE_DIGEST: 'sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f' - } - }).status -} - -test('production capacity mutation is restricted to the exact serving rollout set', () => { - assert.match(workflow, /TARGET_CELL_ID: \$\{\{ inputs\.target-cell-id \}\}/) - const targetInput = dispatchWorkflow.slice( - dispatchWorkflow.indexOf(' target-cell-id:'), - dispatchWorkflow.indexOf(' wave-cell-ids:') - ) - assert.deepEqual( - [...targetInput.matchAll(/^\s+- (production-gce-c\d+)$/gm)].map((match) => match[1]), - capacityCells - ) - assert.match(workflow, new RegExp(`CAPACITY_CELL_IDS: ${capacityCells.join(',')}`)) - for (const cellId of capacityCells) { - assert.match(dispatchWorkflow, new RegExp(`^\\s+- ${cellId}$`, 'm')) - } - assert.match(workflow, /CELL_ORIGIN="https:\/\/\$\{TARGET_HOSTNAME\}\.relay\.onorca\.dev"/) - assert.match(workflow, /echo "TARGET_HOSTNAME=\$\{TARGET_HOSTNAME\}"/) - assert.match(workflow, /\} >> "\$\{GITHUB_ENV\}"/) - assert.match(workflow, /RAISE_SELECTED_CELL_TO_1000/) - assert.match(workflow, /ROLL_BACK_SELECTED_CELL_TO_600/) - assert.match(workflow, /TARGET_HARD_CAP=600[\s\S]*?else[\s\S]*?TARGET_HARD_CAP=1000/) -}) - -test('rollback confirmation is bound to the exact selected cell', () => { - assert.equal( - mutationConfirmation('apply', 'production-gce-c25', 'RAISE_SELECTED_CELL_TO_1000'), - 0 - ) - assert.equal( - mutationConfirmation( - 'rollback', - 'production-gce-c25', - 'ROLL_BACK_SELECTED_CELL_TO_600 production-gce-c25' - ), - 0 - ) - assert.notEqual( - mutationConfirmation('rollback', 'production-gce-c25', 'ROLL_BACK_SELECTED_CELL_TO_600'), - 0 - ) - assert.notEqual( - mutationConfirmation( - 'rollback', - 'production-gce-c25', - 'ROLL_BACK_SELECTED_CELL_TO_600 production-gce-c26' - ), - 0 - ) -}) - -test('production configuration selects only serving cells for 1,000 and the compatible image', () => { - const cell = (cellId) => production.slice( - production.indexOf(`"${cellId}"`), - production.indexOf('\n }', production.indexOf(`"${cellId}"`)) - ) - for (const cellId of capacityCells) { - assert.match(cell(cellId), /connection_hard_cap\s+= 1000/) - assert.match( - cell(cellId), - /sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563/ - ) - } - for (const cellId of ['production-gce-c17', 'production-gce-c18']) { - assert.match(cell(cellId), /connection_hard_cap\s+= 600/) - assert.doesNotMatch(cell(cellId), /connection_hard_cap\s+= 1000/) - assert.match(cell(cellId), /sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d/) - } - assert.equal(production.match(/connection_hard_cap\s+= 1000/g)?.length, capacityCells.length) - // Asia cells legitimately share this digest, so scope the uniqueness check to the capacity set. - assert.equal( - new Set(capacityCells.map((cellId) => cell(cellId).match(/sha256:[0-9a-f]{64}/)[0])).size, - 1 - ) - assert.match( - workflow, - /COMPATIBLE_DIRECTOR_IMAGE_DIGEST: sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73/ - ) - assert.match( - workflow, - /test "\$\{ACTIVE_IMAGE_DIGEST\}" = "\$\{COMPATIBLE_DIRECTOR_IMAGE_DIGEST\}"/ - ) - assert.match( - workflow, - /test "\$\{DESIRED_IMAGE_DIGEST\}" = "\$\{COMPATIBLE_CELL_IMAGE_DIGEST\}"/ - ) - assert.match(workflow, /\.\[\$cell\]\.connection_hard_cap = \$cap/) - assert.match(workflow, /baseCells:\$baseCells/) - assert.match(workflow, /capacityCellIds:\(\$capacityCellIds \| split\(","\)\)/) - assert.match( - workflow, - /Verify current selected-cell capacity[\s\S]*?TOPOLOGY_PHASE.*predecessor[\s\S]*?CURRENT_CAP=600[\s\S]*?DESIRED_IMAGE_DIGEST.*PREDECESSOR_IMAGE_DIGEST/ - ) -}) - -test('director and cell image compatibility is an exact reviewed pair', () => { - const repository = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@' - const director = `${repository}sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73` - const cell = `${repository}sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f` - const other = `${repository}sha256:${'a'.repeat(64)}` - assert.equal(imageCompatibility(cell, cell), 0) - assert.equal(imageCompatibility(director, cell), 0) - assert.notEqual(imageCompatibility(director, other), 0) - assert.notEqual(imageCompatibility(other, cell), 0) -}) - -// The matching check on google_project_service.required lives with the foundation root, which -// stays in the private repository. - -test('apply consumes fresh evidence before arming mutation cleanup', () => { - for (const marker of [ - 'Require fresh dry-run evidence reference', - 'Verify dry-run artifact before cloud authentication', - 'Reject previously consumed dry-run evidence', - 'Verify fresh dry-run evidence against the live selector', - 'Recheck every live safety signal', - 'Publish the consumed-evidence marker' - ]) { - assert.match(workflow, new RegExp(marker)) - } - assert.match(workflow, /--mutation-mode capacity-transition/) - assert.match(workflow, /--source-cell-id "\$\{TARGET_CELL_ID\}"/) - ordered( - 'Publish the consumed-evidence marker', - 'Arm fail-closed mutation cleanup', - 'Reversibly isolate only the selected cell', - 'Deploy only the reviewed director topology', - 'Plan and apply only the empty selected cell', - 'Restore only the selected cell to general admission', - 'Verify the live general selected cell' - ) -}) - -test('wave apply consumes one proof and runs fail-closed cells sequentially', () => { - assert.match(dispatchWorkflow, /- wave-apply/) - assert.match(dispatchWorkflow, /group: production-cloud-sql-rollout/) - assert.match(dispatchWorkflow, /Validate the exact wave request/) - assert.match(dispatchWorkflow, /Verify wave evidence against the live selector/) - assert.match(dispatchWorkflow, /Require exact 600\/60 predecessor wave cells/) - assert.match( - dispatchWorkflow, - /COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f/ - ) - assert.match( - dispatchWorkflow, - /Require exact 600\/60 predecessor wave cells[\s\S]*?--expected-image-digests \\\n\s+"\$\{PREDECESSOR_IMAGE_DIGEST\},\$\{COMPATIBLE_CELL_IMAGE_DIGEST\}"/ - ) - assert.match(dispatchWorkflow, /Publish the consumed-evidence marker/) - assert.match( - dispatchWorkflow, - /OUTPUT_DIRECTORY: \$\{\{ github\.workspace \}\}\/relay-monitor-evidence/ - ) - assert.match( - dispatchWorkflow, - /path: \$\{\{ github\.workspace \}\}\/relay-monitor-evidence/ - ) - assert.doesNotMatch(dispatchWorkflow, /strategy:/) - for (const [index, dependency] of [ - [1, 'wave_gate'], - [2, 'wave_cell_1'], - [3, 'wave_cell_2'], - [4, 'wave_cell_3'] - ]) { - const start = dispatchWorkflow.indexOf(` wave_cell_${index}:`) - const end = dispatchWorkflow.indexOf(`\n wave_cell_${index + 1}:`, start) - const job = dispatchWorkflow.slice(start, end === -1 ? undefined : end) - assert.match(job, new RegExp(`needs: (?:\\[wave_gate, )?${dependency}`)) - assert.match(job, /evidence-mode: continuation/) - assert.match(job, new RegExp(`wave-index: '${index - 1}'`)) - } - assert.match(workflow, /Download this workflow's wave authority/) - assert.match(workflow, /run-id: \$\{\{ github\.run_id \}\}/) - assert.match(workflow, /relay-production-capacity-wave\.mjs build-preflight/) - assert.match(workflow, /Require the exact wave predecessor topology/) - assert.match(workflow, /test "\$\{TOPOLOGY_PHASE\}" = predecessor/) - assert.match(workflow, /Recheck exact wave state and every live safety signal/) - assert.match(workflow, /if test "\$\{WAVE_INDEX\}" != 0; then RETRY_ARGS=\(--retry-freshness\); fi/) - assert.equal(workflow.match(/--retry-freshness/g)?.length, 2) - ordered( - 'Recheck exact wave state and every live safety signal', - 'Arm fail-closed mutation cleanup', - 'Reversibly isolate only the selected cell', - 'Verify the live general selected cell' - ) -}) - -test('Terraform mutation targets only the selected cell and has fail-closed recovery', () => { - assert.match( - workflow, - /google_compute_instance_template\.relay_gce_cell\[\\"\$\{TARGET_CELL_ID\}\\"\]/ - ) - assert.match( - workflow, - /google_compute_instance_group_manager\.relay_gce_cell\[\\"\$\{TARGET_CELL_ID\}\\"\]/ - ) - assert.doesNotMatch(workflow, /relay_gce_cell\["production-gce-c26"\]/) - assert.doesNotMatch(workflow, /target=google_cloud_run_v2_service\.relay/) - assert.match(workflow, /validate-relay-capacity-plan\.mjs/) - assert.match(workflow, /--mode bootstrap-cell/) - assert.match(workflow, /--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/) - assert.match(workflow, /failure\(\) && inputs\.mode != 'verify'/) - assert.match(workflow, /test "\$\{MUTATION_STARTED:-false\}" = true \|\| exit 0/) - assert.match(workflow, /--mode isolate/) - assert.doesNotMatch(workflow, /rolling-action restart/) - assert.doesNotMatch(workflow, /manage_artifact_dns/) - assert.match(workflow, /OFFLINE_ROLLBACK=true/) - assert.match(workflow, /--runtime unavailable/) - assert.equal(workflow.match(/--expected-image-digests/g)?.length, 7) - assert.match(workflow, /PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b/) - assert.match(workflow, /classify-relay-production-capacity-director\.mjs/) - assert.match(workflow, /CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON/) - assert.match(workflow, /if test "\$\{DIRECTOR_READY\}" = true; then exit 0; fi/) - assert.match( - workflow, - /Keep the selected cell isolated after a failed mutation[\s\S]*?--mode isolate[\s\S]*?--mode drain/ - ) - const cleanup = workflow.slice(workflow.indexOf('id: cleanup-auth')) - assert.match(cleanup, /Keep the selected cell isolated after a failed mutation/) - assert.match(cleanup, /steps\.cleanup-auth\.outputs\.id_token/) - assert.doesNotMatch(cleanup, /steps\.deploy-auth\.outputs\.id_token/) - ordered( - 'Reversibly isolate only the selected cell', - 'Drain the selected cell or prove an offline rollback', - 'id: restart-auth-one', - 'Require restart-safe selected-cell activity', - 'id: restart-auth-two', - 'Require extended restart-safe selected-cell activity', - 'Deploy only the reviewed director topology', - 'id: director-transition-auth', - 'Require fail-closed director transition', - 'id: capacity-auth', - 'Plan and apply only the empty selected cell', - 'id: capacity-transition-auth', - 'Verify fresh exact selected-cell heartbeat before admission' - ) - const restartGate = workflow.slice( - workflow.indexOf('id: restart-auth-one'), - workflow.indexOf('Deploy only the reviewed director topology') - ) - assert.equal(restartGate.match(/--timeout-ms 450000/g)?.length, 2) - assert.match(restartGate, /steps\.restart-auth-one\.outputs\.id_token/) - assert.match(restartGate, /steps\.restart-auth-two\.outputs\.id_token/) - assert.match(restartGate, /capacity transition verification timed out:/) - assert.match(workflow, /timeout-minutes: 75/) -}) - -test('wave resume is bound to the failed run and exact isolated selector state', () => { - assert.match(dispatchWorkflow, /- wave-resume/) - assert.match(dispatchWorkflow, /evidence-mode: resume/) - assert.match(dispatchWorkflow, /source-wave-run-id: \$\{\{ inputs\.source-wave-run-id \}\}/) - assert.match(workflow, /Download the failed wave authority for resume/) - assert.match(workflow, /test "\$\{SOURCE_SHA\}" = "\$\{MONITOR_SHA\}"/) - assert.match(workflow, /test "\$\{SOURCE_ATTEMPT\}" = "\$\{EXPECTED_SOURCE_ATTEMPT\}"/) - for (const boundary of [ - '31554591366:31555510376:production-gce-c16,production-gce-c15,production-gce-c14,production-gce-c13:production-gce-c13', - '31562760783:31563664692:production-gce-c10,production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c10', - '31571019947:31572080665:production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c8', - 'a917e8e1fc1a2654e8cb81ba39b57733ec56be9c', - '6082e9ca89a918ca51f0c87db003f5e8805b64b7', - 'e59958130c9d9b7a6cd805df2678d08997842c7c', - 'EXPECTED_SOURCE_ATTEMPT=2' - ]) { - assert.match(workflow, new RegExp(boundary)) - } - assert.match(workflow, /\*\) exit 1 ;;/) - assert.match(workflow, /test "\$\{MONITOR_SHA\}" = "\$\{EXPECTED_SHA\}"/) - assert.match(workflow, /\.head_branch == "main"/) - assert.match(workflow, /\.head_repository\.full_name == env\.GITHUB_REPOSITORY/) - assert.ok(workflow.includes(`.path == "${relayWorkflowPath('monitor-relay-production.yml')}"`)) - assert.ok(workflow.includes(`.path == "${relayWorkflowPath('deploy-relay-production-capacity.yml')}"`)) - assert.match(workflow, /build-resume-preflight/) - assert.match(workflow, /RESUME_SELECTED_CELL_TO_1000 \$\{TARGET_CELL_ID\}/) - assert.match( - workflow, - /Recheck exact isolated resume state[\s\S]*?--hard-cap 600[\s\S]*?--admission migration-only[\s\S]*?--draining required/ - ) -}) - -test('GCE capacity identity is exact-workflow and narrowly permissioned', () => { - const provider = resource( - 'google_iam_workload_identity_pool_provider', - 'github_production_relay_capacity' - ) - assert.match(provider, /concat\(local\.relay_github_leading_repository_claims, \[/) - for (const boundary of [ - "assertion.ref == 'refs/heads/main'", - "assertion.environment == 'production'", - 'local.relay_github_workflow_conditions["github_production_relay_capacity"]' - ]) { - assert.match(provider, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'))) - } - // The workflow pair itself is pinned in the clause the provider renders, once per accepted - // repository, and each repository supplies its own workflow-ref head. - for (const boundary of [ - "assertion.workflow_ref == '${prefix}${local.github_production_relay_capacity_workflow_file}@refs/heads/main'", - "assertion.job_workflow_ref == '${prefix}${local.github_production_relay_capacity_job_workflow_file}@refs/heads/main'" - ]) { - assert.match(terraform, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'))) - } - const role = resource( - 'google_project_iam_custom_role', - 'github_production_relay_capacity_mutation' - ) - assert.match(role, /compute\.instanceGroupManagers\.update/) - assert.match(role, /compute\.instanceTemplates\.create/) - assert.doesNotMatch( - role, - /compute\.(?:disks\.delete|instances\.(?:delete|start|stop|update))|cloudsql|secretmanager/ - ) - const state = resource( - 'google_storage_bucket_iam_member', - 'github_production_relay_capacity_state' - ) - assert.match(state, /objects\/terraform\/state\/default\.tfstate/) - assert.match(state, /objects\/terraform\/state\/default\.tflock/) -}) - -test('deploy and capacity identities are used in their intended phases', () => { - const jobStart = workflow.indexOf(' capacity:') - const stepsStart = workflow.indexOf(' steps:', jobStart) - const jobHeader = workflow.slice(jobStart, stepsStart) - assert.deepEqual(jobHeader.match(/^\s+if:.*$/gm), [ - " if: ${{ github.ref == 'refs/heads/main' }}" - ]) - const configurationStart = workflow.indexOf('Require production workflow configuration') - const configurationEnd = workflow.indexOf('- uses: actions/checkout@v4', configurationStart) - assert.ok(configurationStart >= 0) - assert.ok(configurationEnd > configurationStart) - const configurationStep = workflow.slice(configurationStart, configurationEnd) - for (const name of [ - 'GCP_REGION', - 'DEPLOY_WORKLOAD_IDENTITY_PROVIDER', - 'DEPLOY_SERVICE_ACCOUNT', - 'CAPACITY_WORKLOAD_IDENTITY_PROVIDER', - 'CAPACITY_SERVICE_ACCOUNT' - ]) { - assert.match(configurationStep, new RegExp(`test -n "\\$\\{${name}\\}"`)) - } - ordered('Require production workflow configuration', 'id: deploy-auth') - ordered('id: deploy-auth', 'Reversibly isolate only the selected cell', 'id: capacity-auth') - assert.match(workflow, /steps\.deploy-auth\.outputs\.id_token/) - assert.doesNotMatch(workflow, /steps\.capacity-auth\.outputs\.id_token/) - assert.equal( - workflow.match(/steps\.capacity-transition-auth\.outputs\.id_token/g)?.length, - 3 - ) - assert.match(workflow, /PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/) - assert.match(workflow, /PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/) - assert.match(workflow, /read-relay-production-capacity-identity\.mjs/) - assert.match(workflow, /\(\.directorReady \| type\) == "boolean"/) - assert.match(workflow, /\(\.directorReady \| tostring\)/) -}) - -test('director readiness extraction preserves only JSON booleans', { - skip: spawnSync('jq', ['--version']).status !== 0 -}, () => { - const filter = `if (.directorReady | type) == "boolean" then - (.directorReady | tostring) - else error("invalid directorReady classification") end` - const extract = (input) => spawnSync('jq', ['-er', filter], { - encoding: 'utf8', - input: JSON.stringify(input) - }) - for (const value of [true, false]) { - const result = extract({ directorReady: value }) - assert.equal(result.status, 0) - assert.equal(result.stdout.trim(), String(value)) - } - for (const input of [ - { directorReady: 'true' }, - { directorReady: 'false' }, - { directorReady: null }, - {} - ]) { - assert.notEqual(extract(input).status, 0) - } -}) diff --git a/cloud/dev/scripts/relay-production-identity-boundaries.test.mjs b/cloud/dev/scripts/relay-production-identity-boundaries.test.mjs deleted file mode 100644 index a256fd7139a..00000000000 --- a/cloud/dev/scripts/relay-production-identity-boundaries.test.mjs +++ /dev/null @@ -1,180 +0,0 @@ -import assert from 'node:assert/strict' -import { readFile } from 'node:fs/promises' -import test from 'node:test' -import { readRelayWorkflow, relayWorkflowFile } from './relay-repository.mjs' -import { readWorkflow, workflowFiles } from './cloud-sql-rollout-lock-census.mjs' - -async function source(path) { - return await readFile(new URL(`../../${path}`, import.meta.url), 'utf8') -} - -// Why: the shared deploy identity is relay-owned in production and moves with the relay -// extraction, so it needs a name the public repo can carry without touching the app pair. The -// generic names are retired; a workflow that still reads them would silently resolve to nothing. -test('no workflow names the retired generic production deploy identity', async () => { - const files = workflowFiles() - assert.ok(files.length > 20) - const relayReaders = [] - for (const file of files) { - const workflow = readWorkflow(file) - assert.doesNotMatch(workflow, /PRODUCTION_GCP_WORKLOAD_IDENTITY_PROVIDER\b/, file) - assert.doesNotMatch(workflow, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT\b/, file) - if (/PRODUCTION_GCP_RELAY_DEPLOY_/.test(workflow)) relayReaders.push(file) - } - assert.deepEqual(relayReaders.sort(), [ - 'deploy-relay-fence-broker.yml', - 'deploy-relay-production-capacity-job.yml', - 'deploy-relay-production-capacity.yml', - 'deploy-relay-production-director.yml', - 'deploy-relay-production-multi-target.yml', - 'deploy-relay-production-same-cap-job.yml', - 'deploy-relay-production-same-cap.yml', - 'deploy-relay-production.yml', - 'operate-relay-asia-admission.yml', - 'operate-relay-production-cell-flags.yml', - 'operate-relay-production-rehome-job.yml', - 'operate-relay-production-reserve-watch.yml', - 'publish-relay-production.yml' - ].map((name) => relayWorkflowFile(name)).sort()) -}) - -test('monitor workflow has no shared deploy identity fallback', async () => { - const workflow = readRelayWorkflow('monitor-relay-production-job.yml') - assert.match(workflow, /PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER/) - assert.match(workflow, /PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT/) - assert.doesNotMatch(workflow, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/) - assert.doesNotMatch(workflow, /PRODUCTION_GCP_WORKLOAD_IDENTITY_PROVIDER/) -}) - -test('relay fencing uses the dedicated requester and private broker', async () => { - const workflow = readRelayWorkflow('deploy-relay-production-multi-target.yml') - assert.match(workflow, /Reject direct-runner Terraform fence aborts/) - assert.match(workflow, /inputs\.mode == 'fence-source'/) - assert.match(workflow, /inputs\.mode == 'abort-fence-source'/) - assert.match(workflow, /inputs\.mode == 'supersede-target'/) - assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER/) - assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT/) - assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_BROKER_URI/) - assert.match(workflow, /Invoke private target-supersession broker/) - assert.match(workflow, /Invoke private source-fence broker/) - assert.match(workflow, /Require exact broker cell contract/) - assert.match(workflow, /Require exact source-fence broker contract/) - assert.match(workflow, /Require private fence-broker environment/) - assert.match( - workflow, - /DEPLOY_MODE\}" = "execute" \|\|\s+"\$\{DEPLOY_MODE\}" = "recover-forward"\) &&\s+"\$\{SOURCE_CELL_ID\}" = "production-gce-c12"/ - ) - assert.match( - workflow, - /--scoped-recovery-source-cell-id\s+production-gce-c3/ - ) - assert.match( - workflow, - /test "\$\{FAILED_TARGET_CELL_ID\}" = "production-gce-c12"/ - ) - assert.match( - workflow, - /test "\$\{REPLACEMENT_TARGET_CELL_ID\}" = "production-gce-c13"/ - ) - assert.match( - workflow, - /test "\$\{TARGET_CELL_IDS\}" = "production-gce-c12,production-gce-c13"/ - ) - assert.match( - workflow, - /test "\$\{TARGET_CELL_IDS\}" = "production-gce-c7,production-gce-c8,production-gce-c10,production-gce-c13,production-gce-c17,production-gce-c18"/ - ) - const jobGate = workflow.slice( - workflow.indexOf('jobs:'), - workflow.indexOf('runs-on:') - ) - assert.doesNotMatch(jobGate, /PRODUCTION_GCP_RELAY_FENCE_/) - const brokerStep = workflow.slice( - workflow.indexOf('- name: Invoke private target-supersession broker'), - workflow.indexOf('- name: Preflight or run multi-target evacuation') - ) - assert.match(brokerStep, /steps\.google-fence-broker-auth\.outputs\.id_token/) - assert.doesNotMatch(brokerStep, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/) - const sourceFenceStep = workflow.slice( - workflow.indexOf('- name: Invoke private source-fence broker'), - workflow.indexOf('- name: Preflight or run multi-target evacuation') - ) - assert.match(sourceFenceStep, /steps\.google-fence-broker-auth\.outputs\.id_token/) - assert.match(sourceFenceStep, /\/v1\/fence-source/) - assert.doesNotMatch(sourceFenceStep, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/) -}) - -test('Terraform binds dedicated identities to exact OIDC and resource boundaries', async () => { - const terraform = await source('infra/terraform/relay-github-actions.tf') - for (const claim of ['job_workflow_ref', 'workflow_ref', 'ref', 'environment']) { - assert.match(terraform, new RegExp(`assertion\\.${claim}`)) - } - assert.match(terraform, /github_monitor_workflow_file/) - assert.match(terraform, /github_fence_workflow_file/) - assert.match(terraform, /github_production_relay_capacity_job_workflow_file/) - assert.match(terraform, /google_service_account" "github_monitor"/) - assert.match(terraform, /google_service_account" "github_fence"/) - assert.match(terraform, /google_service_account\.github_monitor\[0\]\.member/) - assert.match(terraform, /service_account_id = google_service_account\.github_fence\[0\]\.name/) - assert.match(terraform, /attribute\.relay_ops_identity\/monitor/) - assert.match(terraform, /attribute\.relay_ops_identity\/fence/) - assert.doesNotMatch(terraform, /github_relay_fence_operator/) - assert.doesNotMatch(terraform, /github_terraform_fence_state_writer/) - const broker = await source('infra/terraform/relay-fence-broker.tf') - assert.match(broker, /max_instance_request_concurrency = 1/) - assert.match(broker, /max_instance_count = 1/) - assert.match(broker, /roles\/run\.invoker/) - assert.match(broker, /google_service_account\.github_fence\[0\]\.member/) - assert.doesNotMatch(broker, /allUsers/) - const brokerDeploy = readRelayWorkflow('deploy-relay-fence-broker.yml') - assert.match(brokerDeploy, /sha-\$\{GITHUB_SHA\}/) - assert.match(brokerDeploy, /gcloud run services update/) - assert.match(brokerDeploy, /\.status\.traffic/) - assert.doesNotMatch(brokerDeploy, /latestReadyRevisionName/) - assert.doesNotMatch(brokerDeploy, /--set-env-vars/) -}) - -test('Terraform exposes the audited production environment values', async () => { - const outputs = await source('infra/terraform/outputs.tf') - for (const output of [ - 'github_relay_monitor_workload_identity_provider', - 'github_relay_monitor_service_account', - 'github_relay_fence_workload_identity_provider', - 'github_relay_fence_service_account' - ]) { - assert.match(outputs, new RegExp(`output "${output}"`)) - } -}) - -test('production mutations pass the minted admin token to live preflight', async () => { - const workflow = readRelayWorkflow('deploy-relay-production.yml') - const recheck = workflow.slice( - workflow.indexOf('- name: Recheck all live safety signals'), - workflow.indexOf('- name: Create single-use dry-run marker') - ) - assert.match( - recheck, - /ORCA_RELAY_ADMIN_ID_TOKEN: \$\{\{ steps\.google-auth\.outputs\.id_token \}\}/ - ) - const multiTarget = readRelayWorkflow('deploy-relay-production-multi-target.yml') - const multiTargetRecheck = multiTarget.slice( - multiTarget.indexOf('- name: Recheck all live safety signals'), - multiTarget.indexOf('- name: Create single-use dry-run marker') - ) - assert.match(multiTargetRecheck, /steps\.google-auth\.outputs\.id_token/) - assert.match(multiTargetRecheck, /inputs\.mode != 'supersede-target'/) -}) - -test('fence broker pins the security-updated Terraform planner', async () => { - const dockerfile = await source('apps/relay-fence-broker/Dockerfile') - assert.match(dockerfile, /FROM hashicorp\/terraform:1\.17\.0@sha256:[a-f0-9]{64} AS terraform/) -}) - - test('push uses its dedicated identity and rollout lease', () => { - const workflow = readRelayWorkflow('push-deploy.yml') - assert.match(workflow, /PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER/) - assert.match(workflow, /PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT/) - assert.doesNotMatch(workflow, /PRODUCTION_GCP_RELAY_DEPLOY_/) - assert.match(workflow, /group: production-push-rollout/) - assert.match(workflow, /object: terraform\/state\/push-rollout\/production.lock/) -}) diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.mjs deleted file mode 100644 index 3389910ccfc..00000000000 --- a/cloud/dev/scripts/relay-production-same-cap-wave.mjs +++ /dev/null @@ -1,322 +0,0 @@ -import { readFileSync } from 'node:fs' -import { pathToFileURL } from 'node:url' -import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs' - -// Migration-only by policy: zero hosts and no reservation, so a wave rolls one without -// displacing anybody. It enters and must leave migration-only, never general. -export const SAME_CAP_MIGRATION_ONLY_CELLS = ['production-gce-c17', 'production-gce-c18'] - -export const SAME_CAP_CELLS = [ - 'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10', - 'production-gce-c13', 'production-gce-c14', 'production-gce-c15', 'production-gce-c16', - 'production-gce-c19', 'production-gce-c20', 'production-gce-c21', 'production-gce-c22', - 'production-gce-c23', 'production-gce-c24', 'production-gce-c25', 'production-gce-c26', - 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30', - 'production-gce-c31', 'production-gce-c32', 'production-gce-c33', 'production-gce-c34', - 'production-gce-c35', - ...SAME_CAP_MIGRATION_ONLY_CELLS -] - -// A general cell's wave isolates and restores it, advancing the selector twice; a -// migration-only cell's isolate and restore are both no-ops, so its wave advances nothing. -export function selectorWaveDelta(cellId) { - return SAME_CAP_MIGRATION_ONLY_CELLS.includes(cellId) ? 0 : 2 -} - -// The cell spreads its drain sends evenly over this window (host-session-registry.ts drain), so -// it sets the re-placement arrival rate: hosts / window. A closed set, stepped down one rung at a -// time per cloud/docs/relay-workflows.md; the first entry is the default every wave used before. -// The slower windows are for cells whose 300 s drain browned out the database (c28, 10-07). -export const SAME_CAP_DRAIN_PACE_WINDOWS_MS = [300_000, 60_000, 30_000, 900_000, 1_200_000] -export const DEFAULT_SAME_CAP_DRAIN_PACE_WINDOW_MS = SAME_CAP_DRAIN_PACE_WINDOWS_MS[0] - -// Only US general cells may drain faster than the default. An Asia drain is bounded by its -// targets' own accept rate (~4-6 hosts/s per cell over 176 ms round trips), and a migration-only -// cell carries no hosts, so neither has anything to gain from a shorter window. -export const SAME_CAP_FAST_DRAIN_PACE_CELLS = [ - 'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10', - 'production-gce-c13', 'production-gce-c14', 'production-gce-c15', 'production-gce-c16', - 'production-gce-c19', 'production-gce-c20', 'production-gce-c21', 'production-gce-c22', - 'production-gce-c23', 'production-gce-c24', 'production-gce-c25', 'production-gce-c26', - 'production-gce-c32', 'production-gce-c33' -] - -export function drainPaceWindowMs(value, cellIds) { - const parsed = /^[1-9][0-9]*$/.test(value ?? '') ? Number(value) : Number.NaN - if (!SAME_CAP_DRAIN_PACE_WINDOWS_MS.includes(parsed)) { - throw new Error( - `drain pace window must be one of ${SAME_CAP_DRAIN_PACE_WINDOWS_MS.join(', ')} ms` - ) - } - if (parsed < DEFAULT_SAME_CAP_DRAIN_PACE_WINDOW_MS) { - const slow = cellIds.filter((cell) => !SAME_CAP_FAST_DRAIN_PACE_CELLS.includes(cell)) - if (slow.length > 0) { - throw new Error( - `drain pace window ${parsed} ms is for US general cells only, not ${slow.join(',')}` - ) - } - } - return parsed -} - -// The default keeps the confirmation every earlier wave typed; any other window must be named -// in it, so a dispatch cannot run a pace its confirmation did not state. -function confirmationPaceSuffix(paceWindowMs) { - return paceWindowMs === DEFAULT_SAME_CAP_DRAIN_PACE_WINDOW_MS - ? '' - : ` drain-pace-window-ms=${paceWindowMs}` -} - -export function entryAdmission(cellId) { - return SAME_CAP_MIGRATION_ONLY_CELLS.includes(cellId) ? 'migration-only' : 'general' -} - -function digest(value, name) { - if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`) - return value -} - -function cells(value) { - const parsed = value.split(',').map((cell) => cell.trim()).filter(Boolean) - if ( - parsed.length < 1 || - parsed.length > 10 || - new Set(parsed).size !== parsed.length || - parsed.some((cell) => !SAME_CAP_CELLS.includes(cell)) - ) throw new Error('same-cap wave cells are invalid') - // Every later cell offsets from one per-wave selector delta, and the two classes - // have different ones, so a mixed wave has no single offset any cell could use. - if (new Set(parsed.map(selectorWaveDelta)).size > 1) { - throw new Error('same-cap wave cells must be all general or all migration-only') - } - return parsed -} - -export function validateSameCapWave(input) { - if (!['verify', 'canary-apply', 'batch-apply', 'rollback'].includes(input.mode)) { - throw new Error('same-cap wave mode is invalid') - } - const selected = cells(input.cellIds) - const targetDigest = digest(input.targetDigest, 'target digest') - const rollbackDigest = digest(input.rollbackDigest, 'rollback digest') - if (targetDigest === rollbackDigest) throw new Error('target and rollback digests must differ') - if (input.mode === 'canary-apply' && selected.length !== 1) { - throw new Error('canary mode requires exactly one cell') - } - // Ten is the wave workflow's statically declared serial cell-job chain, cell_1..cell_10. - if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 10)) { - throw new Error('batch mode requires two to ten cells') - } - // Later waves expect the selector to advance by exactly 2 per predecessor, - // which a resumed rollback cell (isolate skipped, +1) violates. - if (input.mode === 'rollback' && selected.length !== 1) { - throw new Error('rollback mode requires exactly one cell') - } - const paceWindowMs = drainPaceWindowMs(input.drainPaceWindowMs, selected) - const mutation = input.mode !== 'verify' - const expectedConfirmation = (input.mode === 'rollback' - ? `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} ${selected.join(',')}` - : `ROLL_RELAY_SAME_CAP ${targetDigest} ${selected.join(',')}`) + - confirmationPaceSuffix(paceWindowMs) - if (mutation && input.confirmation !== expectedConfirmation) { - throw new Error('same-cap confirmation does not match the exact digest and cells') - } - if (!mutation && input.confirmation) throw new Error('verify does not accept confirmation') - if (input.mode === 'batch-apply' && !/^[1-9][0-9]*$/.test(input.canaryRunId ?? '')) { - throw new Error('batch mode requires a canary run ID') - } - if (input.mode !== 'batch-apply' && input.canaryRunId) { - throw new Error('only batch mode accepts a canary run ID') - } - return { cells: selected, targetDigest, rollbackDigest, drainPaceWindowMs: paceWindowMs } -} - -const CANARY_PACE_VERDICTS = ['PASS', 'WARN', 'WOULD_BLOCK', 'UNVERIFIED'] - -// A pace is a host arrival rate (hosts / window), so a canary proves it only with a real cohort: -// at least about half the 692-782 hosts a US general cell carried on 10-02, keeping any batch -// cell within ~2x of the rate the canary actually drained at. -export const CANARY_MIN_DRAINED_HOSTS = 400 - -// The canary cell's own pace checks, trusted only from a report on this cell that drained enough -// hosts at this pace; a cell whose image fell back to an unpaced drain proved nothing about it. -export function canaryPaceVerdict(report, cellId, paceWindowMs) { - if ( - report?.cellId !== cellId || - report.drain?.paceWindowMs !== paceWindowMs || - report.drain?.appliedPaceWindowMs !== paceWindowMs || - !(report.drain?.targetHosts >= CANARY_MIN_DRAINED_HOSTS) || - !CANARY_PACE_VERDICTS.includes(report.paceVerdict) - ) return 'UNVERIFIED' - return report.paceVerdict -} - -export function canaryAuthority(input) { - const wave = validateSameCapWave({ ...input, mode: 'canary-apply', canaryRunId: '' }) - if (!/^[0-9a-f]{40}$/.test(input.commitSha ?? '')) throw new Error('commit SHA is invalid') - if (!/^[1-9][0-9]*$/.test(input.runId ?? '')) throw new Error('run ID is invalid') - const selectorGeneration = Number(input.selectorGeneration) - const rehomeGeneration = Number(input.rehomeGeneration) - if (!Number.isSafeInteger(selectorGeneration) || selectorGeneration < 0) { - throw new Error('selector generation is invalid') - } - if (!Number.isSafeInteger(rehomeGeneration) || rehomeGeneration < 0) { - throw new Error('rehome generation is invalid') - } - return { - v: 2, - commitSha: input.commitSha, - runId: input.runId, - cellId: wave.cells[0], - targetDigest: wave.targetDigest, - rollbackDigest: wave.rollbackDigest, - drainPaceWindowMs: wave.drainPaceWindowMs, - paceVerdict: canaryPaceVerdict(input.shadowReport, wave.cells[0], wave.drainPaceWindowMs), - selectorGeneration: selectorGeneration + selectorWaveDelta(wave.cells[0]), - rehomeGeneration - } -} - -export function verifyCanaryAuthority(authority, expected, repositoryRoot) { - const selectorGeneration = Number(expected.selectorGeneration) - // A mixed wave is already rejected, so the batch's first cell names the whole batch's class. - const batchCells = cells(expected.cellIds ?? '') - const batchAdmission = entryAdmission(batchCells[0]) - const batchPaceWindowMs = drainPaceWindowMs(expected.drainPaceWindowMs, batchCells) - if ( - authority?.v !== 2 || - !/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') || - authority.runId !== expected.runId || - authority.targetDigest !== expected.targetDigest || - authority.rollbackDigest !== expected.rollbackDigest || - !Number.isSafeInteger(authority.selectorGeneration) || - authority.selectorGeneration < 0 || - !Number.isSafeInteger(selectorGeneration) || - selectorGeneration < authority.selectorGeneration || - authority.rehomeGeneration !== Number(expected.rehomeGeneration) || - !SAME_CAP_CELLS.includes(authority.cellId) || - !SAME_CAP_DRAIN_PACE_WINDOWS_MS.includes(authority.drainPaceWindowMs) || - !CANARY_PACE_VERDICTS.includes(authority.paceVerdict) - ) throw new Error('canary authority does not match this batch') - // A canary proves its own pace and every slower one; a faster batch needs its own canary, and - // falling back to a slower pace mid-ladder never does. - if (batchPaceWindowMs < authority.drainPaceWindowMs) { - throw new Error( - `canary authority drained over ${authority.drainPaceWindowMs} ms, ` + - `so it cannot authorize a batch draining over ${batchPaceWindowMs} ms` - ) - } - // A batch rolls up to ten cells back to back, so a faster one needs a canary whose own drain - // passed; the default and anything slower stay available to any canary. - if ( - batchPaceWindowMs < DEFAULT_SAME_CAP_DRAIN_PACE_WINDOW_MS && - authority.paceVerdict !== 'PASS' - ) { - throw new Error( - `canary authority pace checks were ${authority.paceVerdict}, ` + - `so it cannot authorize a batch draining over ${batchPaceWindowMs} ms` - ) - } - // A migration-only cell carries no hosts and a different cap, so rolling it proves nothing - // about a general batch, and its wave advances a different selector delta. - if (entryAdmission(authority.cellId) !== batchAdmission) { - throw new Error( - `canary authority cell ${authority.cellId} is ${entryAdmission(authority.cellId)}, ` + - `but this batch is ${batchAdmission}` - ) - } - // Each cell checks exact live selector state; later batches may reuse this control epoch's canary. - requireSameEvidenceCode({ - sealedSha: authority.commitSha, - currentSha: expected.commitSha, - label: 'relay same-cap canary authority', - repositoryRoot - }) - return authority -} - -// A missing or unreadable report seals UNVERIFIED rather than failing the seal: the default pace -// never needed one. -function readShadowReport(path) { - try { - return JSON.parse(readFileSync(path, 'utf8')) - } catch { - return null - } -} - -function values(argv) { - const result = {} - for (let index = 0; index < argv.length; index += 2) { - if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) { - throw new Error('invalid arguments') - } - result[argv[index].slice(2)] = argv[index + 1] - } - return result -} - -export function main(argv = process.argv.slice(2)) { - const command = argv.shift() - const input = values(argv) - if (command === 'validate') { - const wave = validateSameCapWave({ - mode: input.mode, - cellIds: input['cell-ids'], - targetDigest: input['target-digest'], - rollbackDigest: input['rollback-digest'], - confirmation: input.confirmation, - canaryRunId: input['canary-run-id'], - drainPaceWindowMs: input['drain-pace-window-ms'] - }) - process.stdout.write(`${JSON.stringify(wave.cells)}\n`) - return - } - if (command === 'create-canary') { - process.stdout.write(`${JSON.stringify(canaryAuthority({ - mode: 'canary-apply', - cellIds: input['cell-id'], - targetDigest: input['target-digest'], - rollbackDigest: input['rollback-digest'], - confirmation: input.confirmation, - drainPaceWindowMs: input['drain-pace-window-ms'], - shadowReport: readShadowReport(input['shadow-report']), - commitSha: input['commit-sha'], - runId: input['run-id'], - selectorGeneration: input['selector-generation'], - rehomeGeneration: input['rehome-generation'] - }))}\n`) - return - } - if (command === 'cell-class') { - const cellId = input['cell-id'] - if (!SAME_CAP_CELLS.includes(cellId)) throw new Error('same-cap wave cells are invalid') - process.stdout.write(`${JSON.stringify({ - entryAdmission: entryAdmission(cellId), - selectorWaveDelta: selectorWaveDelta(cellId), - drainPaceWindowMs: drainPaceWindowMs(input['drain-pace-window-ms'], [cellId]) - })}\n`) - return - } - if (command === 'verify-canary') { - verifyCanaryAuthority(JSON.parse(readFileSync(input.file, 'utf8')), { - commitSha: input['commit-sha'], - runId: input['run-id'], - cellIds: input['cell-ids'], - targetDigest: input['target-digest'], - rollbackDigest: input['rollback-digest'], - selectorGeneration: input['selector-generation'], - rehomeGeneration: input['rehome-generation'], - drainPaceWindowMs: input['drain-pace-window-ms'] - }) - return - } - throw new Error('unknown same-cap wave command') -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - try { main() } catch (error) { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - } -} diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs deleted file mode 100644 index 3e7ae952a47..00000000000 --- a/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs +++ /dev/null @@ -1,946 +0,0 @@ -import assert from 'node:assert/strict' -import { execFileSync, spawnSync } from 'node:child_process' -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { dirname, join } from 'node:path' -import { test } from 'node:test' -import { - CANARY_MIN_DRAINED_HOSTS, - DEFAULT_SAME_CAP_DRAIN_PACE_WINDOW_MS, - SAME_CAP_CELLS, - SAME_CAP_DRAIN_PACE_WINDOWS_MS, - SAME_CAP_FAST_DRAIN_PACE_CELLS, - SAME_CAP_MIGRATION_ONLY_CELLS, - canaryAuthority, - entryAdmission, - main, - selectorWaveDelta, - validateSameCapWave, - verifyCanaryAuthority -} from './relay-production-same-cap-wave.mjs' -import { readRelayWorkflow } from './relay-repository.mjs' - -const targetDigest = `sha256:${'a'.repeat(64)}` -const rollbackDigest = `sha256:${'b'.repeat(64)}` - -test('requires one canary or a bounded reviewed batch', () => { - assert.deepEqual(validateSameCapWave({ - mode: 'canary-apply', - cellIds: 'production-gce-c7', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7` - }).cells, ['production-gce-c7']) - assert.throws(() => validateSameCapWave({ - mode: 'canary-apply', - cellIds: 'production-gce-c7,production-gce-c8', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: 'wrong' - }), /canary/) - assert.deepEqual(validateSameCapWave({ - mode: 'batch-apply', - cellIds: 'production-gce-c8,production-gce-c9', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c8,production-gce-c9`, - canaryRunId: '42' - }).cells, ['production-gce-c8', 'production-gce-c9']) - assert.deepEqual(validateSameCapWave({ - mode: 'canary-apply', - cellIds: 'production-gce-c28', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c28` - }).cells, ['production-gce-c28']) - assert.deepEqual(validateSameCapWave({ - mode: 'canary-apply', - cellIds: 'production-gce-c30', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c30` - }).cells, ['production-gce-c30']) - assert.deepEqual(validateSameCapWave({ - mode: 'canary-apply', - cellIds: 'production-gce-c31', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c31` - }).cells, ['production-gce-c31']) - for (const cellId of ['production-gce-c32', 'production-gce-c33', 'production-gce-c34']) { - assert.deepEqual(validateSameCapWave({ - mode: 'canary-apply', - cellIds: cellId, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}` - }).cells, [cellId]) - } - assert.throws(() => validateSameCapWave({ - mode: 'canary-apply', - cellIds: 'production-gce-c36', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c36` - }), /cells/) -}) - -// The bound is the wave workflow's static cell_1..cell_10 chain: a batch longer than the -// chain would silently drop its tail cells, so it is refused before any mutation. -test('a batch fills the serial cell chain and never overflows it', () => { - const general = SAME_CAP_CELLS.filter((cell) => entryAdmission(cell) === 'general') - const batch = (count) => { - const cellIds = general.slice(0, count).join(',') - return validateSameCapWave({ - mode: 'batch-apply', - cellIds, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds}`, - canaryRunId: '42' - }) - } - assert.equal(batch(10).cells.length, 10) - assert.throws(() => batch(11), /same-cap wave cells are invalid/) - assert.throws(() => batch(1), /batch mode requires two to ten cells/) -}) - -// The validator's ten-cell bound is only true if the workflow really declares ten strictly -// serial cell jobs and frees the lease after all of them. -test('the wave workflow chains exactly ten serial cell jobs', () => { - const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml') - for (let index = 0; index < 10; index += 1) { - const job = index + 1 - assert.match(dispatch, new RegExp(`\n cell_${job}:\n`), `cell_${job} is missing`) - assert.match(dispatch, new RegExp(`fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\]`)) - assert.match(dispatch, new RegExp(`wave-index: '${index}'`)) - if (index > 0) { - assert.match(dispatch, new RegExp(`needs: \\[gate, cell_${index}\\]`)) - assert.match( - dispatch, - new RegExp(`if: \\$\\{\\{ needs\\.cell_${index}\\.result == 'success' && ` + - `fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\] != null \\}\\}`) - ) - } - assert.match(dispatch, new RegExp(`\n - cell_${job}\n`), `release_lease must need cell_${job}`) - } - assert.doesNotMatch(dispatch, /\n cell_11:/) -}) - -test('lists only C17 and C18 as migration-only now that C35 is promoted, and C30-C35 as general', () => { - assert.deepEqual(SAME_CAP_MIGRATION_ONLY_CELLS, ['production-gce-c17', 'production-gce-c18']) - for (const cellId of [ - 'production-gce-c30', 'production-gce-c31', 'production-gce-c32', 'production-gce-c33', - 'production-gce-c34', 'production-gce-c35' - ]) { - assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId) - } -}) - -test('rolls the migration-only cells but never mixes the two classes in one wave', () => { - for (const cellId of SAME_CAP_MIGRATION_ONLY_CELLS) { - assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId) - assert.equal(entryAdmission(cellId), 'migration-only', cellId) - assert.deepEqual(validateSameCapWave({ - mode: 'canary-apply', - cellIds: cellId, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}` - }).cells, [cellId]) - } - const cellIds = 'production-gce-c17,production-gce-c18' - assert.deepEqual(validateSameCapWave({ - mode: 'batch-apply', - cellIds, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds}`, - canaryRunId: '42' - }).cells, ['production-gce-c17', 'production-gce-c18']) - // C30 is general since its 2026-09-23 promotion, so it cannot share a wave with C17/C18. - assert.equal(entryAdmission('production-gce-c30'), 'general') - assert.equal(selectorWaveDelta('production-gce-c30'), 2) - const asiaGeneral = 'production-gce-c29,production-gce-c30' - assert.deepEqual(validateSameCapWave({ - mode: 'batch-apply', - cellIds: asiaGeneral, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asiaGeneral}`, - canaryRunId: '42' - }).cells, ['production-gce-c29', 'production-gce-c30']) - const asiaMixed = 'production-gce-c30,production-gce-c17' - assert.throws(() => validateSameCapWave({ - mode: 'batch-apply', - cellIds: asiaMixed, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asiaMixed}`, - canaryRunId: '42' - }), /all general or all migration-only/) - // C31 is general since its 2026-10-01 promotion, so it rolls beside C30 but never C17/C18. - assert.equal(entryAdmission('production-gce-c31'), 'general') - assert.equal(selectorWaveDelta('production-gce-c31'), 2) - const asiaPromoted = 'production-gce-c30,production-gce-c31' - assert.deepEqual(validateSameCapWave({ - mode: 'batch-apply', - cellIds: asiaPromoted, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asiaPromoted}`, - canaryRunId: '42' - }).cells, ['production-gce-c30', 'production-gce-c31']) - // C34 is general once its Asia canary promotes it, so a rollback restores it general. - assert.equal(entryAdmission('production-gce-c34'), 'general') - assert.equal(selectorWaveDelta('production-gce-c34'), 2) - const c34Mixed = 'production-gce-c34,production-gce-c17' - assert.throws(() => validateSameCapWave({ - mode: 'batch-apply', - cellIds: c34Mixed, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${c34Mixed}`, - canaryRunId: '42' - }), /all general or all migration-only/) - // C35 is general once its Asia canary promotes it, so a rollback restores it general. - assert.equal(entryAdmission('production-gce-c35'), 'general') - assert.equal(selectorWaveDelta('production-gce-c35'), 2) - const c35Mixed = 'production-gce-c35,production-gce-c17' - assert.throws(() => validateSameCapWave({ - mode: 'batch-apply', - cellIds: c35Mixed, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${c35Mixed}`, - canaryRunId: '42' - }), /all general or all migration-only/) - const c31Mixed = 'production-gce-c31,production-gce-c18' - assert.throws(() => validateSameCapWave({ - mode: 'batch-apply', - cellIds: c31Mixed, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${c31Mixed}`, - canaryRunId: '42' - }), /all general or all migration-only/) - // C32 and C33 are general since their 2026-10-01 promotions: they roll beside US 1k cells, - // isolate and restore (delta 2), and never share a wave with C17/C18. - for (const cellId of ['production-gce-c32', 'production-gce-c33']) { - assert.equal(entryAdmission(cellId), 'general', cellId) - assert.equal(selectorWaveDelta(cellId), 2, cellId) - } - const usPromoted = 'production-gce-c26,production-gce-c32,production-gce-c33' - assert.deepEqual(validateSameCapWave({ - mode: 'batch-apply', - cellIds: usPromoted, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usPromoted}`, - canaryRunId: '42' - }).cells, ['production-gce-c26', 'production-gce-c32', 'production-gce-c33']) - const usMixed = 'production-gce-c32,production-gce-c17' - assert.throws(() => validateSameCapWave({ - mode: 'batch-apply', - cellIds: usMixed, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usMixed}`, - canaryRunId: '42' - }), /all general or all migration-only/) - // A mixed wave has no single selector delta for its later cells to offset from. - const mixed = 'production-gce-c7,production-gce-c17' - assert.throws(() => validateSameCapWave({ - mode: 'batch-apply', - cellIds: mixed, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${mixed}`, - canaryRunId: '42' - }), /all general or all migration-only/) -}) - -test('seals a migration-only canary at the generation its wave leaves behind', () => { - const seal = (cellId) => canaryAuthority({ - cellIds: cellId, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`, - commitSha: 'c'.repeat(40), - runId: '42', - selectorGeneration: '11', - rehomeGeneration: '4' - }) - // Isolate and restore are both no-ops on a migration-only cell, so nothing advances. - assert.equal(seal('production-gce-c17').selectorGeneration, 11) - assert.equal(seal('production-gce-c7').selectorGeneration, 13) - // That canary still authorizes a later batch of its own class; it is evidence about the image. - assert.equal(verifyCanaryAuthority(seal('production-gce-c17'), { - commitSha: 'c'.repeat(40), - runId: '42', - cellIds: 'production-gce-c17,production-gce-c18', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - selectorGeneration: '11', - rehomeGeneration: '4' - }).cellId, 'production-gce-c17') -}) - -test('reports each approved cell\'s class and selector delta', () => { - const printed = [] - const write = process.stdout.write.bind(process.stdout) - process.stdout.write = (chunk) => printed.push(String(chunk)) - try { - main(['cell-class', '--cell-id', 'production-gce-c17', '--drain-pace-window-ms', '300000']) - main(['cell-class', '--cell-id', 'production-gce-c7', '--drain-pace-window-ms', '60000']) - } finally { - process.stdout.write = write - } - assert.deepEqual(printed.map((line) => JSON.parse(line)), [ - { entryAdmission: 'migration-only', selectorWaveDelta: 0, drainPaceWindowMs: 300000 }, - { entryAdmission: 'general', selectorWaveDelta: 2, drainPaceWindowMs: 60000 } - ]) - assert.throws(() => main([ - 'cell-class', '--cell-id', 'production-gce-c12', '--drain-pace-window-ms', '300000' - ]), /cells are invalid/) - // The job re-checks the pace per cell, so a cell outside the fast list cannot drain fast. - assert.throws(() => main([ - 'cell-class', '--cell-id', 'production-gce-c28', '--drain-pace-window-ms', '30000' - ]), /US general cells only, not production-gce-c28/) - assert.throws(() => main(['cell-class', '--cell-id', 'production-gce-c7']), /drain pace window/) -}) - -test('binds rollback confirmation to the exact digest and ordered cells', () => { - assert.throws(() => validateSameCapWave({ - mode: 'rollback', - cellIds: 'production-gce-c7', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_BACK_RELAY_SAME_CAP ${targetDigest} production-gce-c7` - }), /confirmation/) -}) - -test('rollback rolls exactly one cell so later waves stay unreachable', () => { - const cellIds = 'production-gce-c7,production-gce-c8' - assert.throws(() => validateSameCapWave({ - mode: 'rollback', - cellIds, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} ${cellIds}` - }), /rollback mode requires exactly one cell/) - assert.deepEqual(validateSameCapWave({ - mode: 'rollback', - cellIds: 'production-gce-c7', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} production-gce-c7` - }).cells, ['production-gce-c7']) -}) - -test('seals and verifies canary authority for later batches', () => { - const authority = canaryAuthority({ - cellIds: 'production-gce-c7', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`, - commitSha: 'c'.repeat(40), - runId: '42', - selectorGeneration: '11', - rehomeGeneration: '4' - }) - assert.equal(verifyCanaryAuthority(authority, { - commitSha: 'c'.repeat(40), - runId: '42', - cellIds: 'production-gce-c8,production-gce-c9', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - selectorGeneration: '13', - rehomeGeneration: '4' - }).cellId, 'production-gce-c7') - assert.throws(() => verifyCanaryAuthority(authority, { - commitSha: 'd'.repeat(40), - runId: '42', - cellIds: 'production-gce-c8,production-gce-c9', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - selectorGeneration: '11', - rehomeGeneration: '4' - }), /does not match/) -}) - -test('reuses a canary across selector advances only within the same control epoch', () => { - const authority = canaryAuthority({ - cellIds: 'production-gce-c7', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`, - commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4' - }) - const expected = { - commitSha: 'c'.repeat(40), runId: '42', cellIds: 'production-gce-c8,production-gce-c9', - targetDigest, rollbackDigest, drainPaceWindowMs: '300000', selectorGeneration: '21', rehomeGeneration: '4' - } - for (const generation of ['13', '14', '21', '29']) { - assert.equal(verifyCanaryAuthority(authority, { - ...expected, selectorGeneration: generation - }), authority) - } - for (const generation of ['12', '-1', 'NaN', 'Infinity', '13.5', '9007199254740992']) { - assert.throws(() => verifyCanaryAuthority(authority, { - ...expected, selectorGeneration: generation - }), /does not match/) - } - for (const generation of [-1, NaN, Infinity, 13.5, '13', Number.MAX_SAFE_INTEGER + 1]) { - assert.throws(() => verifyCanaryAuthority({ - ...authority, selectorGeneration: generation - }, expected), /does not match/) - } - for (const mismatch of [ - { rehomeGeneration: '3' }, { rehomeGeneration: '5' }, - { targetDigest: rollbackDigest }, { rollbackDigest: targetDigest }, { runId: '43' } - ]) { - assert.throws(() => verifyCanaryAuthority(authority, { - ...expected, ...mismatch - }), /does not match/) - } -}) - -function gitIn(root, ...args) { - return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim() -} - -async function canaryRepository() { - const root = await mkdtemp(join(tmpdir(), 'relay-same-cap-canary-')) - gitIn(root, 'init', '--quiet') - gitIn(root, 'config', 'user.email', 'relay@example.test') - gitIn(root, 'config', 'user.name', 'Relay Wave Test') - gitIn(root, 'config', 'commit.gpgsign', 'false') - const commit = async (path, body, message) => { - await mkdir(dirname(join(root, path)), { recursive: true }) - await writeFile(join(root, path), body) - gitIn(root, 'add', '--all') - gitIn(root, 'commit', '--quiet', '--no-verify', '--message', message) - return gitIn(root, 'rev-parse', 'HEAD') - } - const sealed = await commit( - 'cloud/dev/scripts/relay-production-same-cap-wave.mjs', - 'export const v = 1\n', - 'wave' - ) - const sameCode = await commit('README.md', 'an unrelated merge\n', 'unrelated') - const changedCode = await commit( - 'cloud/dev/scripts/relay-production-same-cap-wave.mjs', - 'export const v = 2\n', - 'wave change' - ) - return { root, sealed, sameCode, changedCode } -} - -test('a batch trusts a canary sealed by identical code at an ancestor commit', async () => { - const repository = await canaryRepository() - try { - const authority = canaryAuthority({ - cellIds: 'production-gce-c7', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`, - commitSha: repository.sealed, - runId: '42', - selectorGeneration: '11', - rehomeGeneration: '4' - }) - const verifyAt = (commitSha, repositoryRoot) => verifyCanaryAuthority(authority, { - commitSha, - runId: '42', - cellIds: 'production-gce-c8,production-gce-c9', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - selectorGeneration: '21', - rehomeGeneration: '4' - }, repositoryRoot) - assert.equal(verifyAt(repository.sameCode, repository.root).cellId, 'production-gce-c7') - assert.throws( - () => verifyAt(repository.changedCode, repository.root), - /code changed after it was sealed/ - ) - assert.throws(() => verifyAt('f'.repeat(40), repository.root), /unknown to this checkout/) - } finally { - await rm(repository.root, { recursive: true, force: true }) - } -}) - - -function sealedCanary(cellId) { - return canaryAuthority({ - cellIds: cellId, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`, - commitSha: 'c'.repeat(40), - runId: '42', - selectorGeneration: '11', - rehomeGeneration: '4' - }) -} - -// Why: a migration-only cell holds zero hosts at a different cap and its wave advances no -// selector, so rolling one is no evidence for a general batch, and the reverse is no evidence -// either. Nothing but the sealed cell id says which class a canary actually proved. -test('refuses a canary sealed on a cell of the other admission class', () => { - const expected = { - commitSha: 'c'.repeat(40), - runId: '42', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '300000', - selectorGeneration: '99', - rehomeGeneration: '4' - } - const general = 'production-gce-c8,production-gce-c9' - const migrationOnly = SAME_CAP_MIGRATION_ONLY_CELLS.join(',') - assert.throws( - () => verifyCanaryAuthority(sealedCanary('production-gce-c17'), { - ...expected, cellIds: general - }), - /canary authority cell production-gce-c17 is migration-only, but this batch is general/ - ) - assert.throws( - () => verifyCanaryAuthority(sealedCanary('production-gce-c7'), { - ...expected, cellIds: migrationOnly - }), - /canary authority cell production-gce-c7 is general, but this batch is migration-only/ - ) - assert.equal( - verifyCanaryAuthority(sealedCanary('production-gce-c7'), { - ...expected, cellIds: general - }).cellId, - 'production-gce-c7' - ) - assert.equal( - verifyCanaryAuthority(sealedCanary('production-gce-c17'), { - ...expected, cellIds: migrationOnly - }).cellId, - 'production-gce-c17' - ) - // A caller that names no batch at all gets no verdict, rather than an unchecked class. - assert.throws( - () => verifyCanaryAuthority(sealedCanary('production-gce-c7'), expected), - /same-cap wave cells are invalid/ - ) -}) - -// The dispatch workflow is the only caller, so the class check only binds anything if that -// step actually hands the batch over; run the step's own shell exactly as written. -function verifyCanaryStepScript() { - const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml') - const first = ' node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \\\n' - const start = dispatch.indexOf(first) - assert.notEqual(start, -1, 'the dispatch workflow has no verify-canary step') - const last = ' --rehome-generation "${REHOME_GENERATION}"\n' - const end = dispatch.indexOf(last, start) - assert.notEqual(end, -1, 'the verify-canary step does not end at the rehome generation') - return dispatch.slice(start, end + last.length).replace(/^ {10}/gm, '') -} - -async function runVerifyCanaryStep(authority, cellIds, drainPaceWindowMs = '300000') { - const temporary = await mkdtemp(join(tmpdir(), 'relay-same-cap-verify-')) - try { - await mkdir(join(temporary, 'relay-same-cap-canary'), { recursive: true }) - await writeFile( - join(temporary, 'relay-same-cap-canary', 'authority.json'), - JSON.stringify(authority) - ) - return spawnSync('bash', ['-euo', 'pipefail', '-c', verifyCanaryStepScript()], { - cwd: new URL('../..', import.meta.url), - env: { - ...process.env, - RUNNER_TEMP: temporary, - GITHUB_SHA: authority.commitSha, - CANARY_RUN_ID: authority.runId, - CELL_IDS: cellIds, - TARGET_DIGEST: targetDigest, - ROLLBACK_DIGEST: rollbackDigest, - SELECTOR_GENERATION: '99', - REHOME_GENERATION: '4', - DRAIN_PACE_WINDOW_MS: drainPaceWindowMs - }, - encoding: 'utf8' - }) - } finally { - await rm(temporary, { recursive: true, force: true }) - } -} - -test('the batch gate hands its own cells to the canary check', async () => { - const accepted = await runVerifyCanaryStep( - sealedCanary('production-gce-c7'), - 'production-gce-c8,production-gce-c9' - ) - assert.equal(accepted.status, 0, accepted.stderr) - const crossed = await runVerifyCanaryStep( - sealedCanary('production-gce-c17'), - 'production-gce-c8,production-gce-c9' - ) - assert.equal(crossed.status, 1, crossed.stdout) - assert.match( - crossed.stderr, - /canary authority cell production-gce-c17 is migration-only, but this batch is general/ - ) - const migrationOnly = await runVerifyCanaryStep( - sealedCanary('production-gce-c17'), - SAME_CAP_MIGRATION_ONLY_CELLS.join(',') - ) - assert.equal(migrationOnly.status, 0, migrationOnly.stderr) - // The batch's own pace reaches the check, so a 300 s canary cannot authorize a faster batch. - const faster = await runVerifyCanaryStep( - sealedCanary('production-gce-c7'), - 'production-gce-c8,production-gce-c9', - '60000' - ) - assert.equal(faster.status, 1, faster.stdout) - assert.match(faster.stderr, /drained over 300000 ms, so it cannot authorize a batch draining over 60000 ms/) -}) - -const usGeneral = 'production-gce-c8,production-gce-c32' - -function fastWave(overrides = {}) { - return { - mode: 'batch-apply', - cellIds: usGeneral, - targetDigest, - rollbackDigest, - drainPaceWindowMs: '60000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usGeneral} drain-pace-window-ms=60000`, - canaryRunId: '42', - ...overrides - } -} - -test('admits only the reviewed drain pace windows', () => { - assert.deepEqual( - SAME_CAP_DRAIN_PACE_WINDOWS_MS, - [300_000, 60_000, 30_000, 900_000, 1_200_000] - ) - assert.equal(validateSameCapWave(fastWave()).drainPaceWindowMs, 60_000) - assert.equal(validateSameCapWave(fastWave({ - drainPaceWindowMs: '30000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usGeneral} drain-pace-window-ms=30000` - })).drainPaceWindowMs, 30_000) - for (const pace of [ - undefined, '', '0', '120000', '299999', '600000', '60000.0', '060000', ' 60000', - '1200001', '1800000' - ]) { - assert.throws( - () => validateSameCapWave(fastWave({ drainPaceWindowMs: pace })), - /drain pace window must be one of 300000, 60000, 30000, 900000, 1200000 ms/, - String(pace) - ) - } - // Verify reads production, never drains, but still states the pace it would roll at. - assert.equal(validateSameCapWave(fastWave({ - mode: 'verify', confirmation: '', canaryRunId: '' - })).drainPaceWindowMs, 60_000) -}) - -test('keeps Asia and migration-only cells on the default pace', () => { - assert.deepEqual( - SAME_CAP_FAST_DRAIN_PACE_CELLS.filter((cell) => !SAME_CAP_CELLS.includes(cell)), - [] - ) - for (const cellId of SAME_CAP_CELLS) { - const fast = SAME_CAP_FAST_DRAIN_PACE_CELLS.includes(cellId) - const asia = ['c27', 'c28', 'c29', 'c30', 'c31', 'c34', 'c35'].some( - (cell) => cellId === `production-gce-${cell}` - ) - assert.equal(fast, entryAdmission(cellId) === 'general' && !asia, cellId) - } - const asia = 'production-gce-c30,production-gce-c31' - assert.throws(() => validateSameCapWave(fastWave({ - cellIds: asia, - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asia} drain-pace-window-ms=60000` - })), /US general cells only, not production-gce-c30,production-gce-c31/) - // One Asia cell in an otherwise US batch is enough to refuse the whole batch. - const mixed = 'production-gce-c8,production-gce-c29' - assert.throws(() => validateSameCapWave(fastWave({ - cellIds: mixed, - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${mixed} drain-pace-window-ms=60000` - })), /not production-gce-c29$/) - assert.throws(() => validateSameCapWave(fastWave({ - mode: 'canary-apply', - cellIds: 'production-gce-c17', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c17 drain-pace-window-ms=60000`, - canaryRunId: '' - })), /not production-gce-c17/) -}) - -test('admits the slower paces on every cell class, Asia and migration-only included', () => { - for (const pace of ['900000', '1200000']) { - for (const cellIds of [ - usGeneral, - 'production-gce-c28,production-gce-c29', - 'production-gce-c8,production-gce-c29', - 'production-gce-c17' - ]) { - const single = !cellIds.includes(',') - const wave = validateSameCapWave(fastWave({ - cellIds, - ...(single ? { mode: 'canary-apply', canaryRunId: '' } : {}), - drainPaceWindowMs: pace, - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds} drain-pace-window-ms=${pace}` - })) - assert.equal(wave.drainPaceWindowMs, Number(pace), `${cellIds} ${pace}`) - } - assert.throws(() => validateSameCapWave(fastWave({ - mode: 'canary-apply', - canaryRunId: '', - cellIds: 'production-gce-c29', - drainPaceWindowMs: pace, - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c29` - })), /confirmation does not match/) - } -}) - -test('binds a non-default pace into the confirmation, in both directions', () => { - // A confirmation that does not name the pace is a confirmation of the default. - assert.throws(() => validateSameCapWave(fastWave({ - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usGeneral}` - })), /confirmation does not match/) - assert.throws(() => validateSameCapWave(fastWave({ - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usGeneral} drain-pace-window-ms=30000` - })), /confirmation does not match/) - assert.throws(() => validateSameCapWave(fastWave({ - drainPaceWindowMs: '300000' - })), /confirmation does not match/) - assert.throws(() => validateSameCapWave(fastWave({ - drainPaceWindowMs: '300000', - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usGeneral} drain-pace-window-ms=300000` - })), /confirmation does not match/) - assert.deepEqual(validateSameCapWave(fastWave({ - mode: 'rollback', - cellIds: 'production-gce-c8', - confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} production-gce-c8 drain-pace-window-ms=60000`, - canaryRunId: '' - })).cells, ['production-gce-c8']) -}) - -function shadowReport(pace, paceVerdict = 'PASS', overrides = {}) { - return { - cellId: 'production-gce-c7', - paceVerdict, - drain: { paceWindowMs: Number(pace), appliedPaceWindowMs: Number(pace), targetHosts: 692 }, - ...overrides - } -} - -test('a canary authorizes batches at its own pace or slower, never faster', () => { - const seal = (pace, report = shadowReport(pace)) => canaryAuthority({ - cellIds: 'production-gce-c7', - targetDigest, - rollbackDigest, - drainPaceWindowMs: pace, - shadowReport: report, - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7` + - (pace === '300000' ? '' : ` drain-pace-window-ms=${pace}`), - commitSha: 'c'.repeat(40), - runId: '42', - selectorGeneration: '11', - rehomeGeneration: '4' - }) - assert.equal(seal('60000').v, 2) - assert.equal(seal('60000').drainPaceWindowMs, 60_000) - const verify = (authority, pace) => verifyCanaryAuthority(authority, { - commitSha: 'c'.repeat(40), - runId: '42', - cellIds: usGeneral, - targetDigest, - rollbackDigest, - drainPaceWindowMs: pace, - selectorGeneration: '13', - rehomeGeneration: '4' - }) - // Stepping back to the default after a bad rung needs no new canary. - for (const pace of ['60000', '300000']) assert.equal(verify(seal('60000'), pace).cellId, 'production-gce-c7') - assert.throws(() => verify(seal('60000'), '30000'), /drained over 60000 ms/) - assert.throws(() => verify(seal('300000'), '60000'), /drained over 300000 ms/) - // Slower than the default needs no PASS, and a slow canary authorizes only slower-or-equal. - for (const pace of ['900000', '1200000']) { - assert.equal(verify(seal('300000'), pace).cellId, 'production-gce-c7') - assert.equal(verify(seal('300000', shadowReport('300000', 'WARN')), pace).cellId, 'production-gce-c7') - assert.equal(verify(seal('60000'), pace).cellId, 'production-gce-c7') - } - assert.equal(seal('1200000').drainPaceWindowMs, 1_200_000) - assert.equal(verify(seal('900000'), '1200000').cellId, 'production-gce-c7') - assert.equal(verify(seal('1200000'), '1200000').cellId, 'production-gce-c7') - assert.throws(() => verify(seal('900000'), '300000'), /drained over 900000 ms/) - assert.throws(() => verify(seal('1200000'), '900000'), /drained over 1200000 ms/) - // An authority sealed before the pace was recorded proves nothing about it. - const { drainPaceWindowMs: _dropped, ...unpaced } = seal('300000') - assert.throws(() => verify({ ...unpaced, v: 1 }, '300000'), /does not match/) - assert.throws(() => verify({ ...seal('300000'), drainPaceWindowMs: 0 }, '300000'), /does not match/) - assert.throws(() => verify({ ...seal('300000'), paceVerdict: 'pass' }, '300000'), /does not match/) -}) - -// A canary whose own drain looked bad still succeeds as a job, so the seal must carry what its pace -// checks said, and a faster batch must refuse anything but PASS. -test('only a canary whose pace checks passed authorizes a faster batch', () => { - const seal = (report) => canaryAuthority({ - cellIds: 'production-gce-c7', - targetDigest, - rollbackDigest, - drainPaceWindowMs: '60000', - shadowReport: report, - confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7 drain-pace-window-ms=60000`, - commitSha: 'c'.repeat(40), - runId: '42', - selectorGeneration: '11', - rehomeGeneration: '4' - }) - const verify = (authority, pace) => verifyCanaryAuthority(authority, { - commitSha: 'c'.repeat(40), - runId: '42', - cellIds: usGeneral, - targetDigest, - rollbackDigest, - drainPaceWindowMs: pace, - selectorGeneration: '13', - rehomeGeneration: '4' - }) - assert.equal(seal(shadowReport('60000')).paceVerdict, 'PASS') - assert.equal(CANARY_MIN_DRAINED_HOSTS, 400) - assert.equal(seal(shadowReport('60000', 'PASS', { - drain: { paceWindowMs: 60000, appliedPaceWindowMs: 60000, targetHosts: 400 } - })).paceVerdict, 'PASS') - for (const [report, sealed] of [ - [shadowReport('60000', 'WARN'), 'WARN'], - [shadowReport('60000', 'WOULD_BLOCK'), 'WOULD_BLOCK'], - [null, 'UNVERIFIED'], - // Another cell's report, another pace's, or a cell that fell back to an unpaced drain. - [shadowReport('60000', 'PASS', { cellId: 'production-gce-c8' }), 'UNVERIFIED'], - [shadowReport('300000'), 'UNVERIFIED'], - [shadowReport('60000', 'PASS', { drain: { paceWindowMs: 60000, appliedPaceWindowMs: 0, targetHosts: 692 } }), 'UNVERIFIED'], - // Too few hosts to have tested the pace, or no host count at all. - [shadowReport('60000', 'PASS', { drain: { paceWindowMs: 60000, appliedPaceWindowMs: 60000, targetHosts: 399 } }), 'UNVERIFIED'], - [shadowReport('60000', 'PASS', { drain: { paceWindowMs: 60000, appliedPaceWindowMs: 60000, targetHosts: 0 } }), 'UNVERIFIED'], - [shadowReport('60000', 'PASS', { drain: { paceWindowMs: 60000, appliedPaceWindowMs: 60000, targetHosts: null } }), 'UNVERIFIED'], - [shadowReport('60000', 'MAYBE'), 'UNVERIFIED'] - ]) { - const authority = seal(report) - assert.equal(authority.paceVerdict, sealed) - assert.throws(() => verify(authority, '60000'), new RegExp(`pace checks were ${sealed}`)) - // Falling back to the default never needs a passing canary. - assert.equal(verify(authority, '300000').cellId, 'production-gce-c7') - } -}) - -test('create-canary seals the verdict from the canary cell\'s shadow report file', async () => { - const temporary = await mkdtemp(join(tmpdir(), 'relay-same-cap-seal-')) - const printed = [] - const write = process.stdout.write.bind(process.stdout) - try { - const path = join(temporary, 'report.json') - await writeFile(path, JSON.stringify(shadowReport('60000'))) - process.stdout.write = (chunk) => printed.push(String(chunk)) - for (const report of [path, join(temporary, 'missing.json')]) { - main([ - 'create-canary', '--cell-id', 'production-gce-c7', - '--target-digest', targetDigest, '--rollback-digest', rollbackDigest, - '--confirmation', `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7 drain-pace-window-ms=60000`, - '--drain-pace-window-ms', '60000', '--shadow-report', report, - '--commit-sha', 'c'.repeat(40), '--run-id', '42', - '--selector-generation', '11', '--rehome-generation', '4' - ]) - } - } finally { - process.stdout.write = write - await rm(temporary, { recursive: true, force: true }) - } - assert.deepEqual(printed.map((line) => JSON.parse(line).paceVerdict), ['PASS', 'UNVERIFIED']) -}) - -// The dispatch choice list, the validator's closed set, and the job's transition wait have to -// agree, or the form offers a pace the gate refuses or the wait outlives the 20-min token. -test('the workflows offer exactly the closed set and scale the drain wait with it', () => { - const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml') - const input = dispatch.slice(dispatch.indexOf(' drain-pace-window-ms:\n')) - assert.match(input, /default: '300000'\n/) - assert.deepEqual( - JSON.parse(/options: (\[[^\]]+\])/.exec(input)[1].replaceAll("'", '"')).map(Number), - SAME_CAP_DRAIN_PACE_WINDOWS_MS - ) - const seal = dispatch.slice(dispatch.indexOf('\n seal_canary:')) - assert.match( - seal, - /name: relay-same-cap-shadow-gate-\$\{\{ fromJSON\(needs\.gate\.outputs\.cells\)\[0\] \}\}-\$\{\{ github\.run_id \}\}\.json\n/ - ) - assert.ok(seal.indexOf('download-artifact') < seal.indexOf('create-canary')) - assert.match(seal, /--shadow-report "\$\{RUNNER_TEMP\}\/relay-same-cap-shadow-gate\/relay-same-cap-shadow-gate-\$\{\{ fromJSON\(needs\.gate\.outputs\.cells\)\[0\] \}\}-\$\{GITHUB_RUN_ID\}\.json"/) - // The seal names the cell the gate normalized, never the raw input a padded form could carry. - assert.doesNotMatch(seal.slice(0, seal.indexOf('\n release_lease:')), /inputs\.cell-ids/) - // The job uploads under the same normalized cell, which cell_1 receives as target-cell-id. - assert.match(dispatch, /target-cell-id: \$\{\{ fromJSON\(needs\.gate\.outputs\.cells\)\[0\] \}\}/) - for (const command of ['validate', 'verify-canary', 'create-canary']) { - const at = dispatch.indexOf(`relay-production-same-cap-wave.mjs ${command}`) - assert.notEqual(at, -1, command) - const end = dispatch.indexOf('\n\n', at) - assert.match(dispatch.slice(at, end), /--drain-pace-window-ms "\$\{/, command) - } - assert.equal( - dispatch.match(/\n drain-pace-window-ms: \$\{\{ inputs\.drain-pace-window-ms \}\}\n/g).length, - 10 - ) - const job = readRelayWorkflow('deploy-relay-production-same-cap-job.yml') - assert.match(job, /\n DRAIN_PACE_WINDOW_MS: \$\{\{ inputs\.drain-pace-window-ms \}\}\n/) - assert.doesNotMatch(job, /DRAIN_PACE_WINDOW_MS: '/) - assert.match(job, /cell-class \\\n\s+--cell-id "\$\{TARGET_CELL_ID\}" --drain-pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/) - // The 15-min migration lease plus the window: restart-safe passes once the window has ended. - const timeout = /--timeout-ms "\$\(\((.+)\)\)"\n/.exec(job)[1] - const timeoutMs = (pace) => Number(execFileSync( - 'bash', ['-c', `DRAIN_PACE_WINDOW_MS=${pace}; echo $((${timeout}))`], { encoding: 'utf8' } - )) - for (const pace of SAME_CAP_DRAIN_PACE_WINDOWS_MS) { - assert.equal(timeoutMs(pace), 900_000 + pace, String(pace)) - } - const slowest = timeoutMs(Math.max(...SAME_CAP_DRAIN_PACE_WINDOWS_MS)) - // The rest of the job keeps the 70 min it had beside the old 20-min drain step in 90. - const jobMinutes = Number(/\n rollout:\n(?: [^\n]*\n)*? timeout-minutes: (\d+)\n/.exec(job)[1]) - assert.ok(jobMinutes * 60_000 >= slowest + 70 * 60_000) - // The drain step's fresh one-hour ID token must outlive its wait, with room for the drain call. - assert.ok(slowest + 5 * 60_000 <= 60 * 60_000) - // The pace check runs before the isolate and before the failsafe treats the cell as touched. - const drainStep = job.slice(job.indexOf('- name: Reversibly isolate and drain only the selected cell')) - const paceCheck = drainStep.indexOf('--mode pace-check') - assert.notEqual(paceCheck, -1) - assert.match(drainStep.slice(paceCheck, drainStep.indexOf('\n', paceCheck) + 80), /--pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/) - assert.ok(paceCheck < drainStep.indexOf('MUTATION_STARTED=true')) - assert.ok(paceCheck < drainStep.indexOf('--mode isolate')) -}) diff --git a/cloud/dev/scripts/relay-public-workflow-contract.test.mjs b/cloud/dev/scripts/relay-public-workflow-contract.test.mjs deleted file mode 100644 index 1d6befc616e..00000000000 --- a/cloud/dev/scripts/relay-public-workflow-contract.test.mjs +++ /dev/null @@ -1,82 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { - isEntrypoint, - jobIf, - jobNeeds, - jobs, - readWorkflow, - workflowFiles -} from './cloud-sql-rollout-lock-census.mjs' -import { relayWorkflowFile } from './relay-repository.mjs' - -// Why: this repository publishes the relay's operate surface next to the desktop app. Three -// invariants make that safe, and each of them is one careless edit away from being lost. -const OPERATIONS_GATE = "vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'" - -// Cloud Verify is the only cloud workflow that must run on every pull request. -const UNGATED = relayWorkflowFile('verify.yml') - -const relayWorkflows = () => workflowFiles().filter((file) => file !== UNGATED) - -test('the copy carries every relay workflow', () => { - assert.equal(relayWorkflows().length, 28) -}) - -// Why: workflow_run chains match by display name, not filename. Renaming a file is safe; renaming -// one of these silently breaks the recovery chain with no failing run to notice. -test('the recovery chain keeps the display names it is matched by', () => { - const names = Object.fromEntries( - ['prove-relay-staging-capacity.yml', 'recover-relay-staging-c4-image.yml', 'requeue-relay-staging-c4-recovery.yml'].map( - (name) => [name, /^name: (.+)$/m.exec(readWorkflow(relayWorkflowFile(name)))?.[1]] - ) - ) - assert.deepEqual(names, { - 'prove-relay-staging-capacity.yml': 'Prove Relay Staging Capacity', - 'recover-relay-staging-c4-image.yml': 'Recover Relay Staging C4 Image', - 'requeue-relay-staging-c4-recovery.yml': 'Requeue Relay Staging C4 Recovery' - }) - const recover = readWorkflow(relayWorkflowFile('recover-relay-staging-c4-image.yml')) - const requeue = readWorkflow(relayWorkflowFile('requeue-relay-staging-c4-recovery.yml')) - assert.ok(recover.includes(`workflows: [${names['prove-relay-staging-capacity.yml']}]`)) - assert.ok(requeue.includes(`workflows: [${names['recover-relay-staging-c4-image.yml']}]`)) -}) - -// Why: this repository holds none of the GCP credentials these workflows would need. Every one -// authenticates through Workload Identity read from a variable, so any repository secret other -// than the automatic token would be a credential the owner has to store here. -test('no cloud workflow reads a repository secret', () => { - for (const file of workflowFiles()) { - for (const [, name] of readWorkflow(file).matchAll(/secrets\.([A-Za-z_][A-Za-z0-9_]*)/g)) { - assert.equal(name, 'GITHUB_TOKEN', `${file} reads secrets.${name}`) - } - } -}) - -// Why: the operations gate is what makes the whole surface inert until the owner enables it. A -// job that can start without a gated dependency would run the moment someone dispatches it. -test('every job that can start on its own is gated on the operations variable', () => { - const reachable = [] - for (const file of relayWorkflows()) { - const text = readWorkflow(file) - if (!isEntrypoint(text)) continue - for (const job of jobs(text)) { - if (jobNeeds(job.text).length > 0) continue - reachable.push(`${file}:${job.id}`) - assert.ok(jobIf(job.text).includes(OPERATIONS_GATE), `${file}:${job.id} is not gated`) - } - } - assert.ok(reachable.length >= 20, `only ${reachable.length} root jobs were checked`) -}) - -// Why: reusable jobs inherit the caller's gate. Gating them again would be dead configuration -// that reads as protection, and every caller is already checked above. -test('reusable workflows carry no gate of their own', () => { - for (const file of relayWorkflows()) { - const text = readWorkflow(file) - if (isEntrypoint(text)) continue - for (const job of jobs(text)) { - assert.ok(!jobIf(job.text).includes(OPERATIONS_GATE), `${file}:${job.id} regates a reusable job`) - } - } -}) diff --git a/cloud/dev/scripts/relay-reconnect-herd-alerts.test.mjs b/cloud/dev/scripts/relay-reconnect-herd-alerts.test.mjs deleted file mode 100644 index 5cd3e5733bf..00000000000 --- a/cloud/dev/scripts/relay-reconnect-herd-alerts.test.mjs +++ /dev/null @@ -1,45 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { fileURLToPath } from 'node:url' - -// Why: both herd alerts key on text the relay writes (a log line and a runtime field). A rename on -// either side silences the alert without failing anything. - -const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8') -const terraform = read('../../infra/terraform/relay-observability.tf') - -const block = (kind, name) => { - const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1] - assert.ok(body, `${kind}.${name} not found in relay-observability.tf`) - return body -} - -const incidentMetric = (key) => { - const body = new RegExp(`\\n ${key} = \\{([\\s\\S]*?)\\n \\}`).exec(terraform)?.[1] - assert.ok(body, `relay_incident_metrics.${key} not found`) - return body -} - -test('the disconnect-burst metric matches the line a cell writes per control close', () => { - const emitter = read('../../apps/relay/src/host-session-registry.ts') - assert.match(emitter, /`\[orca-relay\] control closed host=\$\{/) - assert.match(emitter, /` code=\$\{code\} reason=\$\{/) - const metric = incidentMetric('cell_control_abnormal_closes') - assert.match(metric, /jsonPayload\.message:\\"\[orca-relay\] control closed \\"/) - assert.match(metric, /jsonPayload\.message:\\" code=1006 \\"/) - assert.match(metric, /logs\/cos_containers/) - const policy = block('google_monitoring_alert_policy', 'relay_cell_control_close_burst') - assert.match(policy, /orca_relay_cell_control_abnormal_closes/) - assert.match(policy, /group_by_fields\s*=\s*\["resource\.label\.\\"instance_id\\""\]/) -}) - -test('the pool-herd metric thresholds the interval maximum the relay emits', () => { - const emitter = read('../../apps/relay/src/postgres-pool-pressure.ts') - assert.match(emitter, /\bdatabasePoolWaitersMax: number\b/) - const metric = block('google_logging_metric', 'relay_cell_pool_herd') - // 200, not lower: single-cell stalls of 50-196 waiters recur daily; herds go past 200. - assert.match(metric, /jsonPayload\.databasePoolWaitersMax>200"/) - const policy = block('google_monitoring_alert_policy', 'relay_cell_pool_herd') - assert.match(policy, /threshold_value\s*=\s*0\n/) -}) diff --git a/cloud/dev/scripts/relay-recovery-wave-gate.mjs b/cloud/dev/scripts/relay-recovery-wave-gate.mjs deleted file mode 100644 index 08d53d3680f..00000000000 --- a/cloud/dev/scripts/relay-recovery-wave-gate.mjs +++ /dev/null @@ -1,517 +0,0 @@ -const EXPECTED_KEYS = { - report: ['schemaVersion', 'environment', 'load', 'outcomes'], - environment: [ - 'projectId', - 'directorOrigin', - 'databaseVcpu', - 'databasePoolMax', - 'publicConcurrentMax', - 'resolvePrioritySlots', - 'directorMinInstances', - 'directorMaxInstances', - 'cloudRunConcurrency', - 'rolloutOldPublicConcurrentMax', - 'rolloutOldResolvePrioritySlots', - 'rolloutNewPublicConcurrentMax', - 'rolloutNewResolvePrioritySlots' - ], - load: [ - 'drainingDesktops', - 'backgroundRequestsPerMinute', - 'backgroundAssignmentRequestsPerMinute', - 'backgroundAssignment503PerMinute', - 'targetConnectionCap', - 'targetCells' - ], - targetCell: ['cellId', 'peakConnections', 'recoveredControls'], - outcomes: [ - 'migrationExpirations', - 'migrationAborts', - 'transactionRetryExhaustions', - 'keyProvenTargetRegistrations', - 'oldestMigrationLeaseRemainingAtDrainMs', - 'targetRegistrationDurationMs', - 'assignmentSuccessesPerMinuteBaseline', - 'assignmentSuccessesPerMinuteMinimum', - 'eligibleResolveRequests', - 'resolve2xx', - 'resolveOverload', - 'readinessChecks', - 'readinessFailures', - 'maintenanceOperations', - 'maintenanceFailures', - 'directorPeakInstances', - 'rolloutOverlapPeakInstances', - 'rolloutOverlapPeakPublicOperations', - 'rolloutOverlapEligibleResolveRequests', - 'rolloutOverlapResolve2xx', - 'rolloutOverlapResolveOverload', - 'rolloutOverlapReadinessFailures', - 'rolloutOverlapPoolWaitP95Ms', - 'rolloutOverlapDatabaseCpuPercentMax', - 'poolWaitP95Ms', - 'poolWaitMaxMs', - 'databaseCpuPercentP95', - 'databaseCpuPercentMax', - 'recoveryDurationMs' - ] -} - -const LIMITS = { - drainingDesktopsMin: 760, - drainingDesktopsMax: 840, - backgroundRequestsPerMinuteMin: 10_450, - backgroundRequestsPerMinuteMax: 11_550, - backgroundAssignment503PerMinuteMin: 8_500, - backgroundAssignment503PerMinuteMax: 10_500, - targetCellCount: 2, - targetConnectionCap: 600, - databaseVcpu: 2, - databasePoolMax: 3, - publicConcurrentMax: 2, - resolvePrioritySlots: 1, - directorMinInstances: 1, - directorMaxInstances: 2, - directorPeakInstances: 2, - rolloutOverlapPeakInstances: 4, - rolloutOverlapPeakPublicOperations: 8, - cloudRunConcurrency: 80, - rolloutOldPublicConcurrentMax: 2, - rolloutOldResolvePrioritySlots: 0, - rolloutNewPublicConcurrentMax: 2, - rolloutNewResolvePrioritySlots: 1, - assignmentThroughputRetentionMin: 0.9, - resolveSuccessRateMin: 0.95, - resolveOverloadRateMaxExclusive: 0.01, - poolWaitP95MsMaxExclusive: 500, - poolWaitMaxMsMaxExclusive: 5_000, - databaseCpuPercentP95MaxExclusive: 70, - databaseCpuPercentMaxMaxExclusive: 85, - oldestMigrationLeaseRemainingAtDrainMsMin: 10 * 60_000, - targetRegistrationDurationMsMax: 5 * 60_000, - recoveryDurationMsMax: 14 * 60_000 -} - -export function evaluateRecoveryWaveReport(input) { - const report = parseReport(input) - const recoveredControls = report.load.targetCells.reduce( - (total, cell) => total + cell.recoveredControls, - 0 - ) - const peakTargetConnections = Math.max( - ...report.load.targetCells.map((cell) => cell.peakConnections) - ) - const assignmentThroughputRetention = ratio( - report.outcomes.assignmentSuccessesPerMinuteMinimum, - report.outcomes.assignmentSuccessesPerMinuteBaseline - ) - const resolveSuccessRate = ratio( - report.outcomes.resolve2xx, - report.outcomes.eligibleResolveRequests - ) - const resolveOverloadRate = ratio( - report.outcomes.resolveOverload, - report.outcomes.eligibleResolveRequests - ) - const rolloutOverlapResolveSuccessRate = ratio( - report.outcomes.rolloutOverlapResolve2xx, - report.outcomes.rolloutOverlapEligibleResolveRequests - ) - const rolloutOverlapResolveOverloadRate = ratio( - report.outcomes.rolloutOverlapResolveOverload, - report.outcomes.rolloutOverlapEligibleResolveRequests - ) - const thresholds = [ - equal('database_vcpu', report.environment.databaseVcpu, LIMITS.databaseVcpu), - equal('database_pool_max', report.environment.databasePoolMax, LIMITS.databasePoolMax), - equal( - 'public_concurrent_max', - report.environment.publicConcurrentMax, - LIMITS.publicConcurrentMax - ), - equal( - 'resolve_priority_slots', - report.environment.resolvePrioritySlots, - LIMITS.resolvePrioritySlots - ), - equal( - 'director_min_instances', - report.environment.directorMinInstances, - LIMITS.directorMinInstances - ), - equal( - 'director_max_instances', - report.environment.directorMaxInstances, - LIMITS.directorMaxInstances - ), - equal( - 'cloud_run_concurrency', - report.environment.cloudRunConcurrency, - LIMITS.cloudRunConcurrency - ), - equal( - 'rollout_old_public_concurrent_max', - report.environment.rolloutOldPublicConcurrentMax, - LIMITS.rolloutOldPublicConcurrentMax - ), - equal( - 'rollout_old_resolve_priority_slots', - report.environment.rolloutOldResolvePrioritySlots, - LIMITS.rolloutOldResolvePrioritySlots - ), - equal( - 'rollout_new_public_concurrent_max', - report.environment.rolloutNewPublicConcurrentMax, - LIMITS.rolloutNewPublicConcurrentMax - ), - equal( - 'rollout_new_resolve_priority_slots', - report.environment.rolloutNewResolvePrioritySlots, - LIMITS.rolloutNewResolvePrioritySlots - ), - between( - 'draining_desktops', - report.load.drainingDesktops, - LIMITS.drainingDesktopsMin, - LIMITS.drainingDesktopsMax - ), - between( - 'background_requests_per_minute', - report.load.backgroundRequestsPerMinute, - LIMITS.backgroundRequestsPerMinuteMin, - LIMITS.backgroundRequestsPerMinuteMax - ), - between( - 'background_assignment_requests_per_minute', - report.load.backgroundAssignmentRequestsPerMinute, - LIMITS.backgroundRequestsPerMinuteMin, - LIMITS.backgroundRequestsPerMinuteMax - ), - between( - 'background_assignment_503_per_minute', - report.load.backgroundAssignment503PerMinute, - LIMITS.backgroundAssignment503PerMinuteMin, - LIMITS.backgroundAssignment503PerMinuteMax - ), - atMost( - 'background_assignment_requests_within_total', - report.load.backgroundAssignmentRequestsPerMinute, - report.load.backgroundRequestsPerMinute - ), - atMost( - 'background_assignment_503_within_assignments', - report.load.backgroundAssignment503PerMinute, - report.load.backgroundAssignmentRequestsPerMinute - ), - equal('target_cell_count', report.load.targetCells.length, LIMITS.targetCellCount), - equal( - 'target_connection_cap', - report.load.targetConnectionCap, - LIMITS.targetConnectionCap - ), - atMost( - 'peak_target_connections', - peakTargetConnections, - report.load.targetConnectionCap - ), - equal('recovered_controls', recoveredControls, report.load.drainingDesktops), - equal('migration_expirations', report.outcomes.migrationExpirations, 0), - equal('migration_aborts', report.outcomes.migrationAborts, 0), - equal('transaction_retry_exhaustions', report.outcomes.transactionRetryExhaustions, 0), - equal( - 'key_proven_target_registrations', - report.outcomes.keyProvenTargetRegistrations, - report.load.drainingDesktops - ), - atLeast( - 'oldest_migration_lease_remaining_at_drain_ms', - report.outcomes.oldestMigrationLeaseRemainingAtDrainMs, - LIMITS.oldestMigrationLeaseRemainingAtDrainMsMin - ), - atMost( - 'target_registration_duration_ms', - report.outcomes.targetRegistrationDurationMs, - LIMITS.targetRegistrationDurationMsMax - ), - atLeast( - 'assignment_throughput_retention', - assignmentThroughputRetention, - LIMITS.assignmentThroughputRetentionMin - ), - atLeast('resolve_success_rate', resolveSuccessRate, LIMITS.resolveSuccessRateMin), - lessThan( - 'resolve_overload_rate', - resolveOverloadRate, - LIMITS.resolveOverloadRateMaxExclusive - ), - atLeast('eligible_resolve_requests', report.outcomes.eligibleResolveRequests, 100), - equal('readiness_failures', report.outcomes.readinessFailures, 0), - atLeast('readiness_checks', report.outcomes.readinessChecks, 1), - equal('maintenance_failures', report.outcomes.maintenanceFailures, 0), - atLeast('maintenance_operations', report.outcomes.maintenanceOperations, 1), - equal( - 'director_peak_instances', - report.outcomes.directorPeakInstances, - LIMITS.directorPeakInstances - ), - equal( - 'rollout_overlap_peak_instances', - report.outcomes.rolloutOverlapPeakInstances, - LIMITS.rolloutOverlapPeakInstances - ), - equal( - 'rollout_overlap_peak_public_operations', - report.outcomes.rolloutOverlapPeakPublicOperations, - LIMITS.rolloutOverlapPeakPublicOperations - ), - atLeast( - 'rollout_overlap_eligible_resolve_requests', - report.outcomes.rolloutOverlapEligibleResolveRequests, - 100 - ), - atLeast( - 'rollout_overlap_resolve_success_rate', - rolloutOverlapResolveSuccessRate, - LIMITS.resolveSuccessRateMin - ), - lessThan( - 'rollout_overlap_resolve_overload_rate', - rolloutOverlapResolveOverloadRate, - LIMITS.resolveOverloadRateMaxExclusive - ), - equal( - 'rollout_overlap_readiness_failures', - report.outcomes.rolloutOverlapReadinessFailures, - 0 - ), - lessThan( - 'rollout_overlap_pool_wait_p95_ms', - report.outcomes.rolloutOverlapPoolWaitP95Ms, - LIMITS.poolWaitP95MsMaxExclusive - ), - lessThan( - 'rollout_overlap_database_cpu_percent_max', - report.outcomes.rolloutOverlapDatabaseCpuPercentMax, - LIMITS.databaseCpuPercentMaxMaxExclusive - ), - lessThan( - 'pool_wait_p95_ms', - report.outcomes.poolWaitP95Ms, - LIMITS.poolWaitP95MsMaxExclusive - ), - lessThan( - 'pool_wait_max_ms', - report.outcomes.poolWaitMaxMs, - LIMITS.poolWaitMaxMsMaxExclusive - ), - lessThan( - 'database_cpu_percent_p95', - report.outcomes.databaseCpuPercentP95, - LIMITS.databaseCpuPercentP95MaxExclusive - ), - lessThan( - 'database_cpu_percent_max', - report.outcomes.databaseCpuPercentMax, - LIMITS.databaseCpuPercentMaxMaxExclusive - ), - atMost( - 'recovery_duration_ms', - report.outcomes.recoveryDurationMs, - LIMITS.recoveryDurationMsMax - ) - ] - return { - schemaVersion: 1, - status: thresholds.every(({ pass }) => pass) ? 'PASS' : 'FAIL', - environment: { - projectId: report.environment.projectId, - directorOrigin: report.environment.directorOrigin - }, - metrics: { - recoveredControls, - peakTargetConnections, - assignmentThroughputRetention, - resolveSuccessRate, - resolveOverloadRate, - rolloutOverlapResolveSuccessRate, - rolloutOverlapResolveOverloadRate - }, - thresholds - } -} - -function parseReport(input) { - const report = strictObject(input, EXPECTED_KEYS.report, 'report') - if (report.schemaVersion !== 1) throw new Error('unsupported report schemaVersion') - const environment = strictObject( - report.environment, - EXPECTED_KEYS.environment, - 'environment' - ) - assertSafeEnvironment(environment) - const load = strictObject(report.load, EXPECTED_KEYS.load, 'load') - if (!Array.isArray(load.targetCells)) throw new Error('load.targetCells must be an array') - const targetCells = load.targetCells.map((value, index) => { - const cell = strictObject(value, EXPECTED_KEYS.targetCell, `load.targetCells[${index}]`) - if (!/^[a-z0-9-]{1,128}$/.test(cell.cellId)) throw new Error('target cellId is invalid') - return { - cellId: cell.cellId, - peakConnections: nonnegativeNumber(cell.peakConnections, 'peakConnections'), - recoveredControls: nonnegativeNumber(cell.recoveredControls, 'recoveredControls') - } - }) - if (new Set(targetCells.map(({ cellId }) => cellId)).size !== targetCells.length) { - throw new Error('target cell IDs must be unique') - } - const outcomes = strictObject(report.outcomes, EXPECTED_KEYS.outcomes, 'outcomes') - return { - schemaVersion: 1, - environment: { - projectId: environment.projectId, - directorOrigin: environment.directorOrigin, - databaseVcpu: positiveNumber(environment.databaseVcpu, 'databaseVcpu'), - databasePoolMax: positiveNumber(environment.databasePoolMax, 'databasePoolMax'), - publicConcurrentMax: positiveNumber( - environment.publicConcurrentMax, - 'publicConcurrentMax' - ), - resolvePrioritySlots: positiveNumber( - environment.resolvePrioritySlots, - 'resolvePrioritySlots' - ), - directorMinInstances: positiveNumber( - environment.directorMinInstances, - 'directorMinInstances' - ), - directorMaxInstances: positiveNumber( - environment.directorMaxInstances, - 'directorMaxInstances' - ), - cloudRunConcurrency: positiveNumber( - environment.cloudRunConcurrency, - 'cloudRunConcurrency' - ), - rolloutOldPublicConcurrentMax: positiveNumber( - environment.rolloutOldPublicConcurrentMax, - 'rolloutOldPublicConcurrentMax' - ), - rolloutOldResolvePrioritySlots: nonnegativeNumber( - environment.rolloutOldResolvePrioritySlots, - 'rolloutOldResolvePrioritySlots' - ), - rolloutNewPublicConcurrentMax: positiveNumber( - environment.rolloutNewPublicConcurrentMax, - 'rolloutNewPublicConcurrentMax' - ), - rolloutNewResolvePrioritySlots: positiveNumber( - environment.rolloutNewResolvePrioritySlots, - 'rolloutNewResolvePrioritySlots' - ) - }, - load: { - drainingDesktops: positiveNumber(load.drainingDesktops, 'drainingDesktops'), - backgroundRequestsPerMinute: positiveNumber( - load.backgroundRequestsPerMinute, - 'backgroundRequestsPerMinute' - ), - backgroundAssignmentRequestsPerMinute: positiveNumber( - load.backgroundAssignmentRequestsPerMinute, - 'backgroundAssignmentRequestsPerMinute' - ), - backgroundAssignment503PerMinute: nonnegativeNumber( - load.backgroundAssignment503PerMinute, - 'backgroundAssignment503PerMinute' - ), - targetConnectionCap: positiveNumber(load.targetConnectionCap, 'targetConnectionCap'), - targetCells - }, - outcomes: Object.fromEntries( - EXPECTED_KEYS.outcomes.map((key) => [key, nonnegativeNumber(outcomes[key], `outcomes.${key}`)]) - ) - } -} - -function assertSafeEnvironment(environment) { - if (typeof environment.projectId !== 'string') throw new Error('projectId must be a string') - if ( - environment.projectId !== 'local' && - !environment.projectId.endsWith('-staging') && - !environment.projectId.endsWith('-test') - ) { - throw new Error('recovery-wave reports must come from an isolated non-production project') - } - if (typeof environment.directorOrigin !== 'string') { - throw new Error('directorOrigin must be a string') - } - const origin = new URL(environment.directorOrigin) - const loopback = ['localhost', '127.0.0.1', '::1', '[::1]'].includes(origin.hostname) - const isolatedHost = - loopback || origin.hostname.endsWith('.test') || origin.hostname.includes('staging') - if ( - origin.origin !== environment.directorOrigin || - origin.pathname !== '/' || - (!loopback && origin.protocol !== 'https:') || - !isolatedHost - ) { - throw new Error('directorOrigin must identify a canonical isolated non-production origin') - } -} - -function strictObject(value, keys, name) { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`${name} must be an object`) - } - const actual = Object.keys(value).sort() - const expected = [...keys].sort() - if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { - throw new Error(`${name} has unexpected or missing fields`) - } - return value -} - -function positiveNumber(value, name) { - const number = nonnegativeNumber(value, name) - if (number <= 0) throw new Error(`${name} must be positive`) - return number -} - -function nonnegativeNumber(value, name) { - if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { - throw new Error(`${name} must be a finite nonnegative number`) - } - return value -} - -function ratio(numerator, denominator) { - return denominator === 0 ? 0 : numerator / denominator -} - -function equal(name, observed, limit) { - return threshold(name, observed, '==', limit, observed === limit) -} - -function atLeast(name, observed, limit) { - return threshold(name, observed, '>=', limit, observed >= limit) -} - -function atMost(name, observed, limit) { - return threshold(name, observed, '<=', limit, observed <= limit) -} - -function lessThan(name, observed, limit) { - return threshold(name, observed, '<', limit, observed < limit) -} - -function between(name, observed, minimum, maximum) { - return threshold( - name, - observed, - 'between_inclusive', - [minimum, maximum], - observed >= minimum && observed <= maximum - ) -} - -function threshold(name, observed, operator, limit, pass) { - return { name, observed, operator, limit, pass } -} diff --git a/cloud/dev/scripts/relay-recovery-wave-gate.test.mjs b/cloud/dev/scripts/relay-recovery-wave-gate.test.mjs deleted file mode 100644 index 4ac1de83e72..00000000000 --- a/cloud/dev/scripts/relay-recovery-wave-gate.test.mjs +++ /dev/null @@ -1,214 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { evaluateRecoveryWaveReport } from './relay-recovery-wave-gate.mjs' - -test('passes a complete isolated production-shaped recovery report', () => { - const result = evaluateRecoveryWaveReport(passingReport()) - - assert.equal(result.status, 'PASS') - assert.equal(result.metrics.recoveredControls, 800) - assert.equal(result.metrics.peakTargetConnections, 425) - assert.equal(result.metrics.resolveSuccessRate, 0.99) - assert.equal(result.thresholds.every(({ pass }) => pass), true) - assert.equal( - result.thresholds.find(({ name }) => name === 'recovery_duration_ms')?.limit, - 840_000 - ) -}) - -for (const [name, mutate, failedThreshold] of [ - [ - 'target connection ceiling', - (report) => { - report.load.targetCells[0].peakConnections = 601 - }, - 'peak_target_connections' - ], - [ - 'migration expiration', - (report) => { - report.outcomes.migrationExpirations = 1 - }, - 'migration_expirations' - ], - [ - 'migration abort', - (report) => { - report.outcomes.migrationAborts = 1 - }, - 'migration_aborts' - ], - [ - 'full-wave registration deadline', - (report) => { - report.outcomes.targetRegistrationDurationMs = 300_001 - }, - 'target_registration_duration_ms' - ], - [ - 'legacy assignment background shape', - (report) => { - report.load.backgroundAssignment503PerMinute = 100 - }, - 'background_assignment_503_per_minute' - ], - [ - 'production director instance topology', - (report) => { - report.outcomes.directorPeakInstances = 1 - }, - 'director_peak_instances' - ], - [ - 'old/new rollout overlap topology', - (report) => { - report.outcomes.rolloutOverlapPeakInstances = 2 - }, - 'rollout_overlap_peak_instances' - ], - [ - 'old revision shared admission mode', - (report) => { - report.environment.rolloutOldResolvePrioritySlots = 1 - }, - 'rollout_old_resolve_priority_slots' - ], - [ - 'old/new rollout overlap resolve availability', - (report) => { - report.outcomes.rolloutOverlapResolveOverload = 2 - }, - 'rollout_overlap_resolve_overload_rate' - ], - [ - 'resolve availability', - (report) => { - report.outcomes.resolve2xx = 940 - report.outcomes.resolveOverload = 20 - }, - 'resolve_success_rate' - ], - [ - 'pool wait', - (report) => { - report.outcomes.poolWaitP95Ms = 500 - }, - 'pool_wait_p95_ms' - ], - [ - 'database CPU', - (report) => { - report.outcomes.databaseCpuPercentMax = 85 - }, - 'database_cpu_percent_max' - ], - [ - 'recovery deadline', - (report) => { - report.outcomes.recoveryDurationMs = 840_001 - }, - 'recovery_duration_ms' - ], - [ - 'non-public database maintenance', - (report) => { - report.outcomes.maintenanceFailures = 1 - }, - 'maintenance_failures' - ] -]) { - test(`fails closed on ${name}`, () => { - const report = passingReport() - mutate(report) - const result = evaluateRecoveryWaveReport(report) - - assert.equal(result.status, 'FAIL') - assert.equal( - result.thresholds.find(({ name: thresholdName }) => thresholdName === failedThreshold)?.pass, - false - ) - }) -} - -test('rejects production provenance and unexpected report fields', () => { - const productionProject = passingReport() - productionProject.environment.projectId = 'onorca-cloud' - assert.throws( - () => evaluateRecoveryWaveReport(productionProject), - /isolated non-production project/ - ) - - const productionOrigin = passingReport() - productionOrigin.environment.directorOrigin = 'https://relay.onorca.dev' - assert.throws( - () => evaluateRecoveryWaveReport(productionOrigin), - /isolated non-production origin/ - ) - - const extraField = passingReport() - extraField.environment.accessToken = 'must-not-be-accepted' - assert.throws(() => evaluateRecoveryWaveReport(extraField), /unexpected or missing fields/) -}) - -function passingReport() { - return { - schemaVersion: 1, - environment: { - projectId: 'onorca-cloud-staging', - directorOrigin: 'https://relay-staging.onorca.dev', - databaseVcpu: 2, - databasePoolMax: 3, - publicConcurrentMax: 2, - resolvePrioritySlots: 1, - directorMinInstances: 1, - directorMaxInstances: 2, - cloudRunConcurrency: 80, - rolloutOldPublicConcurrentMax: 2, - rolloutOldResolvePrioritySlots: 0, - rolloutNewPublicConcurrentMax: 2, - rolloutNewResolvePrioritySlots: 1 - }, - load: { - drainingDesktops: 800, - backgroundRequestsPerMinute: 11_000, - backgroundAssignmentRequestsPerMinute: 10_950, - backgroundAssignment503PerMinute: 9_500, - targetConnectionCap: 600, - targetCells: [ - { cellId: 'target-a', peakConnections: 425, recoveredControls: 400 }, - { cellId: 'target-b', peakConnections: 419, recoveredControls: 400 } - ] - }, - outcomes: { - migrationExpirations: 0, - migrationAborts: 0, - transactionRetryExhaustions: 0, - keyProvenTargetRegistrations: 800, - oldestMigrationLeaseRemainingAtDrainMs: 660_000, - targetRegistrationDurationMs: 240_000, - assignmentSuccessesPerMinuteBaseline: 1_400, - assignmentSuccessesPerMinuteMinimum: 1_330, - eligibleResolveRequests: 1_000, - resolve2xx: 990, - resolveOverload: 5, - readinessChecks: 180, - readinessFailures: 0, - maintenanceOperations: 30, - maintenanceFailures: 0, - directorPeakInstances: 2, - rolloutOverlapPeakInstances: 4, - rolloutOverlapPeakPublicOperations: 8, - rolloutOverlapEligibleResolveRequests: 100, - rolloutOverlapResolve2xx: 99, - rolloutOverlapResolveOverload: 0, - rolloutOverlapReadinessFailures: 0, - rolloutOverlapPoolWaitP95Ms: 180, - rolloutOverlapDatabaseCpuPercentMax: 78, - poolWaitP95Ms: 120, - poolWaitMaxMs: 900, - databaseCpuPercentP95: 55, - databaseCpuPercentMax: 72, - recoveryDurationMs: 360_000 - } - } -} diff --git a/cloud/dev/scripts/relay-region-hint-metrics.test.mjs b/cloud/dev/scripts/relay-region-hint-metrics.test.mjs deleted file mode 100644 index 8f331efce18..00000000000 --- a/cloud/dev/scripts/relay-region-hint-metrics.test.mjs +++ /dev/null @@ -1,84 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { fileURLToPath } from 'node:url' - -// Why: the region-skew alert compares asia-east2's share of assignment hints against its share of -// actual placements. Both shares are sums over one log-based metric per region, and the region -// list is written out by hand in Terraform. A region added to the contract without matching -// metrics would silently drop out of both denominators and move the ratio the alert fires on. - -const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8') -const collapse = (text) => text.replaceAll(/\s+/g, ' ') - -const contractRegions = (() => { - const source = read('../../packages/relay-contract/src/relay-regions.ts') - const literal = /export const RELAY_REGIONS = \[([^\]]*)\]/.exec(source) - assert.ok(literal, 'RELAY_REGIONS literal not found in relay-regions.ts') - return [...literal[1].matchAll(/'([^']+)'/g)].map((match) => match[1]) -})() - -const terraform = read('../../infra/terraform/relay-observability.tf') - -const terraformRegions = (() => { - const literal = /relay_region_keys = \[([^\]]*)\]/.exec(terraform) - assert.ok(literal, 'relay_region_keys not found in relay-observability.tf') - return [...literal[1].matchAll(/"([^"]+)"/g)].map((match) => match[1]) -})() - -// Both sides now spell the field-name segments out, so the test compares the two declared maps -// rather than two source expressions. Reformatting either file cannot break this, and a literal -// expected value below still catches an identical wrong edit made to both. -const declaredSegments = (source, open, close) => { - const body = source.slice(source.indexOf(open) + open.length, source.indexOf(close, source.indexOf(open))) - return Object.fromEntries( - [...body.matchAll(/'?"?([a-z0-9-]+)'?"?\s*[:=]\s*'?"?([A-Za-z0-9]+)'?"?/g)].map((match) => [ - match[1], - match[2] - ]) - ) -} - -const terraformSegments = declaredSegments(terraform, 'relay_region_field_segments = {', '}') -const contractSegments = declaredSegments( - read('../../packages/relay-contract/src/relay-regions.ts'), - 'RELAY_REGION_METRIC_SEGMENTS = {', - '}' -) - -test('terraform covers exactly the regions the contract can hint or select', () => { - assert.deepEqual([...terraformRegions].sort(), [...contractRegions].sort()) -}) - -test('terraform and the contract declare the same flat field segments', () => { - assert.deepEqual(terraformSegments, contractSegments) - // Pinned literally so the same wrong edit applied to both sides still fails. - assert.deepEqual(terraformSegments, { 'us-central1': 'UsCentral1', 'asia-east2': 'AsiaEast2' }) - assert.deepEqual(Object.keys(terraformSegments).sort(), [...contractRegions].sort()) -}) - -test('the skew query compares a catalogued region against itself', () => { - const columns = terraformRegions.map((region) => region.replaceAll('-', '_')) - const hint = /hint_share: req_([a-z0-9_]+) \//.exec(terraform) - const placement = /placement_share: sel_([a-z0-9_]+) \//.exec(terraform) - assert.ok(hint && placement, 'skew query share columns not found') - assert.equal(hint[1], placement[1], 'the two shares must be about the same region') - assert.ok(columns.includes(hint[1]), `${hint[1]} is not one of ${columns.join(', ')}`) -}) - -test('the skew condition never divides by the placement share', () => { - // A zero-placement hour is the worst skew there is; MQL drops the row on x/0, so the ratio form - // silences exactly the case the alert exists for. - assert.ok( - !/hint_share \/ placement_share/.test(terraform), - 'cross-multiply instead: hint_share > 2 * placement_share' - ) - assert.match(collapse(terraform), /condition hint_share > 2 \* placement_share/) -}) - -test('the unhinted bucket stays out of the skew denominators', () => { - assert.ok( - !terraformRegions.includes('unhinted'), - 'unhinted requests are a client-side choice, not a region; including them moves the share' - ) -}) diff --git a/cloud/dev/scripts/relay-region-observation-evidence.mjs b/cloud/dev/scripts/relay-region-observation-evidence.mjs deleted file mode 100644 index 1c23a9c9b00..00000000000 --- a/cloud/dev/scripts/relay-region-observation-evidence.mjs +++ /dev/null @@ -1,152 +0,0 @@ -import { createHash } from 'node:crypto' -import { readFileSync } from 'node:fs' -import { pathToFileURL } from 'node:url' - -const WINDOW_MS = 24 * 60 * 60_000 -const BUCKET_MS = 60 * 60_000 -const METRICS = [ - 'requestedRegionsDelta', - 'selectedRegionsDelta', - 'regionFallbacksDelta', - 'unavailableRegionsDelta' -] -const REGION_KEYS = new Set(['asia-east2', 'us-central1', 'unhinted']) - -function integer(value, name) { - const parsed = Number(value) - if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${name} is invalid`) - return parsed -} - -function digest(value, name) { - if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`) - return value -} - -function metric(value, name) { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`${name} is invalid`) - } - return Object.fromEntries(Object.entries(value).map(([key, count]) => { - if (!REGION_KEYS.has(key)) throw new Error(`${name} has an unknown aggregate key`) - return [key, integer(count, `${name}.${key}`)] - })) -} - -function sumMetric(total, value) { - for (const [key, count] of Object.entries(value)) total[key] = (total[key] ?? 0) + count -} - -function evidenceSha256(evidence) { - return createHash('sha256').update(JSON.stringify(evidence)).digest('hex') -} - -export function createRegionObservationEvidence(entries, bindings, now = Date.now()) { - if (!Array.isArray(entries)) throw new Error('runtime metrics response must be an array') - if (!/^[0-9a-f]{40}$/.test(bindings.commitSha ?? '')) throw new Error('commit SHA is invalid') - const directorImageDigest = digest(bindings.directorImageDigest, 'director digest') - const selectorGeneration = integer(bindings.selectorGeneration, 'selector generation') - const controlGeneration = integer(bindings.controlGeneration, 'control generation') - const start = now - WINDOW_MS - const buckets = Array.from({ length: 24 }, () => 0) - const totals = Object.fromEntries(METRICS.map((name) => [name, {}])) - let samples = 0 - for (const entry of entries) { - const timestamp = Date.parse(entry?.timestamp ?? '') - const payload = entry?.jsonPayload - if ( - !Number.isFinite(timestamp) || - timestamp < start || - timestamp > now + 60_000 || - payload?.event !== 'orca_relay_runtime_metrics' || - payload.role !== 'director' - ) continue - const bucket = Math.min(23, Math.floor((timestamp - start) / BUCKET_MS)) - buckets[bucket] += 1 - samples += 1 - for (const name of METRICS) sumMetric(totals[name], metric(payload[name], name)) - } - if (buckets.some((count) => count === 0)) { - throw new Error('24-hour region evidence has a missing hourly bucket') - } - if ( - !Number.isSafeInteger(totals.requestedRegionsDelta['asia-east2']) || - totals.requestedRegionsDelta['asia-east2'] < 1 || - !Number.isSafeInteger(totals.selectedRegionsDelta['asia-east2']) || - totals.selectedRegionsDelta['asia-east2'] < 1 - ) throw new Error('24-hour region evidence has no Asia request and selection activity') - const evidence = { - v: 1, - commitSha: bindings.commitSha, - directorImageDigest, - selectorGeneration, - controlGeneration, - windowStartedAt: start, - windowEndedAt: now, - hourlySampleCounts: buckets, - samples, - totals - } - return { evidence, sha256: evidenceSha256(evidence) } -} - -export function verifyRegionObservationEvidence(sealed, bindings) { - if ( - sealed?.sha256 !== evidenceSha256(sealed?.evidence) || - sealed.evidence?.commitSha !== bindings.commitSha || - sealed.evidence?.directorImageDigest !== bindings.directorImageDigest || - sealed.evidence?.selectorGeneration !== Number(bindings.selectorGeneration) || - sealed.evidence?.controlGeneration !== Number(bindings.controlGeneration) || - !Array.isArray(sealed.evidence?.hourlySampleCounts) || - sealed.evidence.hourlySampleCounts.length !== 24 || - sealed.evidence.hourlySampleCounts.some((count) => integer(count, 'bucket') < 1) || - integer(sealed.evidence?.totals?.requestedRegionsDelta?.['asia-east2'], 'Asia requests') < 1 || - integer(sealed.evidence?.totals?.selectedRegionsDelta?.['asia-east2'], 'Asia selections') < 1 - ) throw new Error('sealed 24-hour region evidence does not match enable authority') - return sealed.evidence -} - -function values(argv) { - const result = {} - for (let index = 0; index < argv.length; index += 2) { - if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) { - throw new Error('invalid arguments') - } - result[argv[index].slice(2)] = argv[index + 1] - } - return result -} - -async function stdinJson(input) { - const chunks = [] - for await (const chunk of input) chunks.push(chunk) - return JSON.parse(Buffer.concat(chunks).toString('utf8')) -} - -export async function main(argv = process.argv.slice(2), input = process.stdin) { - const command = argv.shift() - const args = values(argv) - const bindings = { - commitSha: args['commit-sha'], - directorImageDigest: args['director-image-digest'], - selectorGeneration: args['selector-generation'], - controlGeneration: args['control-generation'] - } - if (command === 'create') { - const sealed = createRegionObservationEvidence(await stdinJson(input), bindings) - process.stdout.write(`${JSON.stringify(sealed)}\n`) - return - } - if (command === 'verify') { - verifyRegionObservationEvidence(JSON.parse(readFileSync(args.file, 'utf8')), bindings) - return - } - throw new Error('unknown region observation evidence command') -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/relay-region-observation-evidence.test.mjs b/cloud/dev/scripts/relay-region-observation-evidence.test.mjs deleted file mode 100644 index 3426c5da5d6..00000000000 --- a/cloud/dev/scripts/relay-region-observation-evidence.test.mjs +++ /dev/null @@ -1,53 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { - createRegionObservationEvidence, - verifyRegionObservationEvidence -} from './relay-region-observation-evidence.mjs' - -const now = Date.parse('2026-08-14T12:00:00Z') -const bindings = { - commitSha: 'a'.repeat(40), - directorImageDigest: `sha256:${'b'.repeat(64)}`, - selectorGeneration: 11, - controlGeneration: 4 -} - -function entries() { - return Array.from({ length: 24 }, (_, index) => ({ - timestamp: new Date(now - (index * 60 + 30) * 60_000).toISOString(), - jsonPayload: { - event: 'orca_relay_runtime_metrics', - role: 'director', - requestedRegionsDelta: { 'asia-east2': index === 0 ? 2 : 0 }, - selectedRegionsDelta: { 'asia-east2': index === 0 ? 1 : 0 }, - regionFallbacksDelta: { 'asia-east2': index === 0 ? 1 : 0 }, - unavailableRegionsDelta: {} - } - })) -} - -test('seals all 24 hourly aggregate region buckets', () => { - const sealed = createRegionObservationEvidence(entries(), bindings, now) - assert.equal(sealed.evidence.hourlySampleCounts.length, 24) - assert.equal(sealed.evidence.totals.requestedRegionsDelta['asia-east2'], 2) - assert.equal(verifyRegionObservationEvidence(sealed, bindings).samples, 24) -}) - -test('rejects missing coverage, missing Asia activity, and changed bindings', () => { - assert.throws(() => createRegionObservationEvidence(entries().slice(1), bindings, now), /missing hourly/) - const noAsia = entries().map((entry) => ({ - ...entry, - jsonPayload: { - ...entry.jsonPayload, - requestedRegionsDelta: {}, - selectedRegionsDelta: {} - } - })) - assert.throws(() => createRegionObservationEvidence(noAsia, bindings, now), /no Asia/) - const sealed = createRegionObservationEvidence(entries(), bindings, now) - assert.throws(() => verifyRegionObservationEvidence(sealed, { - ...bindings, - commitSha: 'c'.repeat(40) - }), /does not match/) -}) diff --git a/cloud/dev/scripts/relay-rehome-aggregate-evidence.mjs b/cloud/dev/scripts/relay-rehome-aggregate-evidence.mjs deleted file mode 100644 index a0563ffa94a..00000000000 --- a/cloud/dev/scripts/relay-rehome-aggregate-evidence.mjs +++ /dev/null @@ -1,106 +0,0 @@ -import { pathToFileURL } from 'node:url' - -const INVENTORY_PREFIX = '[orca-relay] regional rehome inventory ' -// A counters line, so every field is a bare name and a non-negative integer or -// `none`. Pinning the whole line instead is what broke the enable workflow when -// `hostNotArrivedLast24Hours` shipped: the director grew a field and the parser -// read a healthy line as no evidence at all. Tolerating extra fields is safe -// only because the value shape stays fenced — `hostId=someone` is still not a -// counter, so an identity-bearing lookalike cannot slip through as an extra. -const FIELD = /^([A-Za-z][A-Za-z0-9]*)=(none|\d{1,15})$/ -// `oldestActiveAgeMs` is the one required field the director can report as -// `none`; a count that reads `none` is a line this parser does not recognise, -// not evidence worth failing the run over. -const REQUIRED_COUNTS = [ - 'active', - 'awaitingReceipt', - 'targetRegistered', - 'completedLast24Hours', - 'abortedLast24Hours' -] -const REQUIRED_FIELDS = [...REQUIRED_COUNTS, 'oldestActiveAgeMs'] - -function count(value, name) { - const parsed = Number(value) - if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${name} is invalid`) - return parsed -} - -// Returns the field map, or null for anything that is not this line. -export function readRegionalRehomeInventoryFields(textPayload) { - if (typeof textPayload !== 'string' || !textPayload.startsWith(INVENTORY_PREFIX)) return null - const fields = new Map() - for (const token of textPayload.slice(INVENTORY_PREFIX.length).split(' ')) { - const field = FIELD.exec(token) - if (!field || fields.has(field[1])) return null - fields.set(field[1], field[2]) - } - if (!REQUIRED_FIELDS.every((name) => fields.has(name))) return null - if (REQUIRED_COUNTS.some((name) => fields.get(name) === 'none')) return null - return fields -} - -// Absent is not zero: a director on an older image emits no such field, and -// reporting 0 would read as "no leaks" rather than "not measured". -function optionalCount(fields, name) { - const value = fields.get(name) - if (value === undefined || value === 'none') return null - return count(value, name) -} - -export function parseRegionalRehomeInventory(entries, options = {}) { - if (!Array.isArray(entries)) throw new Error('logging response must be an array') - const parsed = entries.flatMap((entry) => { - const fields = readRegionalRehomeInventoryFields(entry?.textPayload ?? '') - const timestamp = Date.parse(entry?.timestamp ?? '') - if (!fields || !Number.isFinite(timestamp)) return [] - return [{ - timestamp, - active: count(fields.get('active'), 'active'), - awaitingReceipt: count(fields.get('awaitingReceipt'), 'awaiting receipt'), - targetRegistered: count(fields.get('targetRegistered'), 'target registered'), - completedLast24Hours: count(fields.get('completedLast24Hours'), 'completed'), - abortedLast24Hours: count(fields.get('abortedLast24Hours'), 'aborted'), - hostNotArrivedLast24Hours: optionalCount(fields, 'hostNotArrivedLast24Hours'), - oldestActiveAgeMs: optionalCount(fields, 'oldestActiveAgeMs') - }] - }).sort((left, right) => right.timestamp - left.timestamp) - if (parsed.length === 0) throw new Error('no aggregate regional rehome inventory evidence') - const latest = parsed[0] - const now = options.now ?? Date.now() - const maxAgeMs = options.maxAgeMs ?? 15 * 60_000 - if (latest.timestamp > now + 60_000 || latest.timestamp < now - maxAgeMs) { - throw new Error('aggregate regional rehome inventory evidence is stale') - } - return latest -} - -function argumentsMap(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) { - throw new Error('invalid arguments') - } - values[argv[index].slice(2)] = argv[index + 1] - } - return values -} - -export async function main(argv = process.argv.slice(2), input = process.stdin) { - const values = argumentsMap(argv) - const maxAgeMs = count(values['max-age-ms'] ?? 900_000, '--max-age-ms') - const chunks = [] - for await (const chunk of input) chunks.push(chunk) - const evidence = parseRegionalRehomeInventory( - JSON.parse(Buffer.concat(chunks).toString('utf8')), - { maxAgeMs } - ) - process.stdout.write(`${JSON.stringify({ event: 'relay_rehome_aggregate_evidence', ...evidence })}\n`) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/relay-rehome-aggregate-evidence.test.mjs b/cloud/dev/scripts/relay-rehome-aggregate-evidence.test.mjs deleted file mode 100644 index e85c4d5606d..00000000000 --- a/cloud/dev/scripts/relay-rehome-aggregate-evidence.test.mjs +++ /dev/null @@ -1,135 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { parseRegionalRehomeInventory } from './relay-rehome-aggregate-evidence.mjs' -import { readRelayWorkflow } from './relay-repository.mjs' - -const now = Date.parse('2026-08-14T12:00:00Z') - -test('selects the newest fresh aggregate-only regional rehome inventory', () => { - const result = parseRegionalRehomeInventory([ - { - timestamp: '2026-08-14T11:58:00Z', - textPayload: '[orca-relay] regional rehome inventory active=2 awaitingReceipt=1 targetRegistered=1 completedLast24Hours=9 abortedLast24Hours=0 oldestActiveAgeMs=30000' - }, - { - timestamp: '2026-08-14T11:50:00Z', - textPayload: '[orca-relay] regional rehome inventory active=1 awaitingReceipt=0 targetRegistered=1 completedLast24Hours=8 abortedLast24Hours=0 oldestActiveAgeMs=none' - } - ], { now, maxAgeMs: 5 * 60_000 }) - assert.deepEqual(result, { - timestamp: Date.parse('2026-08-14T11:58:00Z'), - active: 2, - awaitingReceipt: 1, - targetRegistered: 1, - completedLast24Hours: 9, - abortedLast24Hours: 0, - hostNotArrivedLast24Hours: null, - oldestActiveAgeMs: 30_000 - }) -}) - -test('reads a line the director grew a field on, wherever the field sits', () => { - const result = parseRegionalRehomeInventory([{ - timestamp: '2026-08-14T11:58:00Z', - textPayload: - '[orca-relay] regional rehome inventory hostNotArrivedLast24Hours=4 active=2' + - ' awaitingReceipt=1 targetRegistered=1 completedLast24Hours=9 abortedLast24Hours=7' + - ' oldestActiveAgeMs=30000 someFieldFromALaterRelease=11' - }], { now, maxAgeMs: 5 * 60_000 }) - assert.equal(result.hostNotArrivedLast24Hours, 4) - assert.equal(result.abortedLast24Hours, 7) - assert.equal(result.oldestActiveAgeMs, 30_000) -}) - -test('reports an unmeasured host-not-arrived count as absent, not as zero', () => { - const [withField, withoutField] = ['4', null].map((value) => - parseRegionalRehomeInventory([{ - timestamp: '2026-08-14T11:58:00Z', - textPayload: - '[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0' + - ' completedLast24Hours=0 abortedLast24Hours=0 oldestActiveAgeMs=none' + - (value === null ? '' : ` hostNotArrivedLast24Hours=${value}`) - }], { now, maxAgeMs: 5 * 60_000 }) - ) - assert.equal(withField.hostNotArrivedLast24Hours, 4) - assert.equal(withoutField.hostNotArrivedLast24Hours, null) -}) - -test('rejects stale, malformed, and identity-bearing lookalikes', () => { - assert.throws(() => parseRegionalRehomeInventory([{ - timestamp: '2026-08-14T11:00:00Z', - textPayload: '[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0 completedLast24Hours=0 abortedLast24Hours=0 oldestActiveAgeMs=none' - }], { now, maxAgeMs: 5 * 60_000 }), /stale/) - assert.throws(() => parseRegionalRehomeInventory([{ - timestamp: '2026-08-14T11:59:00Z', - textPayload: '[orca-relay] regional rehome inventory active=0 hostId=secret' - }], { now }), /no aggregate/) -}) - -test('keeps out an identity-bearing field riding along on a complete line', () => { - const complete = - '[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0' + - ' completedLast24Hours=0 abortedLast24Hours=0 oldestActiveAgeMs=none' - for (const extra of [' hostId=secret', ' userId=someone@example.test', ' note=a b']) { - assert.throws( - () => parseRegionalRehomeInventory( - [{ timestamp: '2026-08-14T11:59:00Z', textPayload: complete + extra }], - { now } - ), - /no aggregate/, - extra - ) - } -}) - -test('refuses a line missing a required field, or repeating one', () => { - const missing = - '[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0' + - ' completedLast24Hours=0 oldestActiveAgeMs=none' - assert.throws( - () => parseRegionalRehomeInventory( - [{ timestamp: '2026-08-14T11:59:00Z', textPayload: missing }], - { now } - ), - /no aggregate/ - ) - assert.throws( - () => parseRegionalRehomeInventory( - [{ timestamp: '2026-08-14T11:59:00Z', textPayload: `${missing} abortedLast24Hours=0 abortedLast24Hours=1` }], - { now } - ), - /no aggregate/ - ) - assert.throws( - () => parseRegionalRehomeInventory( - [{ - timestamp: '2026-08-14T11:59:00Z', - textPayload: missing.replace('active=0', 'active=none') + ' abortedLast24Hours=0' - }], - { now } - ), - /no aggregate/ - ) -}) - -// The third edge of the chain the enable workflow depends on. The formatter is -// pinned against this parser in the relay package's inventory-line census; this -// pins the parser against the summary an operator reads, so a field that -// reaches the evidence JSON and stops there fails here. -test('publishes every parsed counter in the operator step summary', () => { - const job = readRelayWorkflow('operate-relay-production-rehome-job.yml') - // The jq program and the file it reads sit on separate continuation lines, so - // match the whole render rather than one line of it. - const summary = /jq -r '([^']*)' \\\n\s*"\$\{RUNNER_TEMP\}\/relay-rehome-inventory\.json"/.exec(job)?.[1] - assert.ok(summary, 'the rehome job no longer renders the inventory evidence') - const evidence = parseRegionalRehomeInventory([{ - timestamp: '2026-08-14T11:58:00Z', - textPayload: - '[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0' + - ' completedLast24Hours=0 abortedLast24Hours=0 hostNotArrivedLast24Hours=0 oldestActiveAgeMs=none' - }], { now, maxAgeMs: 5 * 60_000 }) - for (const key of Object.keys(evidence)) { - if (key === 'timestamp') continue - assert.ok(summary.includes(`.${key}`), `${key} is missing from the step summary`) - } -}) diff --git a/cloud/dev/scripts/relay-repository.mjs b/cloud/dev/scripts/relay-repository.mjs deleted file mode 100644 index 040acef5fe5..00000000000 --- a/cloud/dev/scripts/relay-repository.mjs +++ /dev/null @@ -1,51 +0,0 @@ -import { readFileSync } from 'node:fs' -import { relative } from 'node:path' -import { fileURLToPath } from 'node:url' - -// Single place naming the repository the Relay workflows live in and where their files sit. The -// public-repo copy moves this tree under cloud/, prefixes every workflow filename, and changes the -// owning repository, so only this module changes: nothing else may restate any of the three. -export const RELAY_GITHUB_REPOSITORY = 'stablyai/orca' - -export const RELAY_WORKFLOW_FILE_PREFIX = 'cloud-' - -// Where .github/workflows sits relative to this file. Workflows stay at the repository root while -// this tree moves under cloud/, so the depth changes at the copy even though the layout does not. -export const RELAY_WORKFLOW_DIRECTORY = new URL('../../../.github/workflows/', import.meta.url) - -// Repository root, derived from the one directory above that already tracks the copy's depth. -export const RELAY_REPOSITORY_ROOT = new URL('../../', RELAY_WORKFLOW_DIRECTORY) - -// Repository-relative path for a file in this tree. The prefix is 'cloud/' here and empty where -// the tree is the repository root, so callers naming git paths never restate the layout. -export function relayTreePath(suffix) { - const prefix = relative( - fileURLToPath(RELAY_REPOSITORY_ROOT), - fileURLToPath(new URL('../../', import.meta.url)) - ).split(/[\\/]/).filter(Boolean) - return [...prefix, suffix].join('/') -} - -export function relayWorkflowFile(name) { - return `${RELAY_WORKFLOW_FILE_PREFIX}${name}` -} - -// Repository-relative path for a repository that renames its copies with `prefix`. Terraform's -// trusted prefix is a variable and need not be this checkout's, so callers rendering a -// workflow_ref from Terraform pass it in rather than assuming the local one. -export function prefixedRelayWorkflowPath(prefix, name) { - return `.github/workflows/${prefix}${name}` -} - -// Repository-relative path, the shape GitHub reports in workflow_ref and evidence payloads. -export function relayWorkflowPath(name) { - return prefixedRelayWorkflowPath(RELAY_WORKFLOW_FILE_PREFIX, name) -} - -export function relayWorkflowUrl(name) { - return new URL(relayWorkflowFile(name), RELAY_WORKFLOW_DIRECTORY) -} - -export function readRelayWorkflow(name) { - return readFileSync(relayWorkflowUrl(name), 'utf8') -} diff --git a/cloud/dev/scripts/relay-repository.test.mjs b/cloud/dev/scripts/relay-repository.test.mjs deleted file mode 100644 index cf33f869773..00000000000 --- a/cloud/dev/scripts/relay-repository.test.mjs +++ /dev/null @@ -1,39 +0,0 @@ -import assert from 'node:assert/strict' -import { readdirSync, readFileSync } from 'node:fs' -import test from 'node:test' -import { fileURLToPath } from 'node:url' -import { - RELAY_GITHUB_REPOSITORY, - RELAY_WORKFLOW_FILE_PREFIX, - prefixedRelayWorkflowPath, - readRelayWorkflow, - relayWorkflowFile, - relayWorkflowPath, - relayWorkflowUrl -} from './relay-repository.mjs' - -const directory = fileURLToPath(new URL('.', import.meta.url)) -// The Relay copy takes the scripts named for it. Everything else stays with the applications. -const relayScripts = readdirSync(directory) - .filter((name) => name.includes('relay') && name.endsWith('.mjs')) - .filter((name) => !name.startsWith('relay-repository.')) - -test('workflow identity is derived, never restated', () => { - assert.equal(relayWorkflowFile('deploy-relay-staging.yml'), `${RELAY_WORKFLOW_FILE_PREFIX}deploy-relay-staging.yml`) - assert.equal(relayWorkflowPath('deploy-relay-staging.yml'), `.github/workflows/${relayWorkflowFile('deploy-relay-staging.yml')}`) - assert.ok(relayWorkflowUrl('deploy-relay-staging.yml').pathname.endsWith(relayWorkflowPath('deploy-relay-staging.yml'))) - // A caller rendering Terraform's trusted ref supplies that prefix instead of this checkout's. - assert.equal(prefixedRelayWorkflowPath('cloud-', 'deploy-relay-staging.yml'), '.github/workflows/cloud-deploy-relay-staging.yml') - assert.match(readRelayWorkflow('deploy-relay-staging.yml'), /^name:/m) - assert.match(RELAY_GITHUB_REPOSITORY, /^[\w.-]+\/[\w.-]+$/) -}) - -// Why: the public-repo copy changes the owning repository, the workflow filenames, and the depth -// this tree sits at. Each has to be one edit here, so no Relay script may restate any of them. -test('no Relay script restates the repository or the workflow directory', () => { - for (const name of relayScripts) { - const text = readFileSync(`${directory}${name}`, 'utf8') - assert.doesNotMatch(text, /stablyai\//, `${name} restates the GitHub repository`) - assert.doesNotMatch(text, /\.github\/workflows/, `${name} restates the workflow directory`) - } -}) diff --git a/cloud/dev/scripts/relay-reserve-cell-refusal.mjs b/cloud/dev/scripts/relay-reserve-cell-refusal.mjs deleted file mode 100644 index b91dcb5203d..00000000000 --- a/cloud/dev/scripts/relay-reserve-cell-refusal.mjs +++ /dev/null @@ -1,16 +0,0 @@ -// Step 5: the director refuses per-cell admin reads for a reserve cell with exactly this body; -// any other 404 stays a failure. -export const RESERVE_CELL_REFUSAL_ERROR = 'cell_in_reserve_mode' - -export function isReserveCellRefusal(status, body) { - return status === 404 && body?.error === RESERVE_CELL_REFUSAL_ERROR -} - -// Step-5 design default 4: a reserve cell is flipped back to db before a roll. -export function reserveCellRollRefusal(cellId) { - return new Error( - `${cellId ?? 'a cell this step reads'} is in reserve mode ` + - `(${RESERVE_CELL_REFUSAL_ERROR}): flip it to admitMode=db with the cell-flags ` + - 'workflow and wait for it to record db before rolling it, then flip it back to reserve' - ) -} diff --git a/cloud/dev/scripts/relay-reserve-dead-man-alert.test.mjs b/cloud/dev/scripts/relay-reserve-dead-man-alert.test.mjs deleted file mode 100644 index 71533ea61e8..00000000000 --- a/cloud/dev/scripts/relay-reserve-dead-man-alert.test.mjs +++ /dev/null @@ -1,109 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { fileURLToPath } from 'node:url' - -// Why: each metric keys on the event name and fields the relay writes. A rename on either side -// silences the alert without failing anything. - -const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8') -const terraform = read('../../infra/terraform/relay-observability.tf') -const deadMan = read('../../apps/relay/src/cell-reserve-dead-man.ts') -const registry = read('../../apps/relay/src/host-session-registry.ts') -const server = read('../../apps/relay/src/relay-server.ts') -const poller = read('../../apps/relay/src/shadow-seat-directory.ts') -const watch = read('./watch-relay-reserve-cell.mjs') -const githubActions = read('../../infra/terraform/relay-github-actions.tf') - -const block = (kind, name) => { - const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1] - assert.ok(body, `${kind}.${name} not found in relay-observability.tf`) - return body -} - -test('the metric counts the event and cell the dead-man writes', () => { - assert.match(deadMan, /event: 'orca_relay_cell_reserve_dead_man_tripped',\s*cellId: this\.cellId,/) - const metric = block('google_logging_metric', 'relay_cell_reserve_dead_man_tripped') - assert.ok(metric.includes('jsonPayload.event=\\"orca_relay_cell_reserve_dead_man_tripped\\"')) - assert.ok(metric.includes('cell_id = "EXTRACT(jsonPayload.cellId)"')) -}) - -test('the alert fires on one trip per cell and notifies', () => { - const alert = block('google_monitoring_alert_policy', 'relay_cell_reserve_dead_man_tripped') - // One line per switch-file generation, so a higher bar misses a trip. - assert.match(alert, /threshold_value = 0\n/) - assert.match(alert, /notification_channels = var\.relay_alert_notification_channels/) - assert.ok(alert.includes('metric.type=\\"logging.googleapis.com/user/orca_relay_cell_reserve_dead_man_tripped\\"')) - assert.ok(alert.includes('group_by_fields = ["metric.label.\\"cell_id\\""]')) -}) - -const notifyingOnAnyLine = (name, event, resource, group) => { - const alert = block('google_monitoring_alert_policy', name) - assert.match(alert, /threshold_value = 0\n/) - assert.match(alert, /notification_channels = var\.relay_alert_notification_channels/) - assert.ok(alert.includes(`resource.type=\\"${resource}\\" AND metric.type=\\"logging.googleapis.com/user/${event}\\"`)) - assert.ok(alert.includes(`group_by_fields = ["metric.label.\\"${group}\\""]`)) -} - -test('the flip-back alerts count the lines the cell writes, per cell', () => { - for (const event of ['orca_relay_cell_reregistration_gave_up', 'orca_relay_cell_reregistration_stalled']) { - // logIdentity() carries cellId; both lines name the reason. - assert.match(registry, new RegExp(`event: '${event}',\\s*\\.\\.\\.this\\.logIdentity\\(\\),[\\s\\S]{0,200}reason:`)) - const name = event.replace('orca_', '') - const metric = block('google_logging_metric', name) - assert.ok(metric.includes(`resource.type=\\"gce_instance\\" AND jsonPayload.event=\\"${event}\\"`)) - assert.ok(metric.includes('cell_id = "EXTRACT(jsonPayload.cellId)"')) - assert.ok(metric.includes('reason = "EXTRACT(jsonPayload.reason)"')) - notifyingOnAnyLine(name, event, 'gce_instance', 'cell_id') - } - assert.match(registry, /cellId: this\.config\.cellId/) -}) - -test('the director alert counts its placement-off line on the director service', () => { - const event = 'orca_relay_reserve_placement_off_with_reserve_cells' - assert.match(server, new RegExp(`event: '${event}',\\s*cells,`)) - const metric = block('google_logging_metric', 'relay_reserve_placement_off_with_reserve_cells') - assert.ok( - metric.includes( - `resource.type=\\"cloud_run_revision\\" AND resource.labels.service_name=\\"\${var.relay_cloud_run_service_name}\\" AND jsonPayload.event=\\"${event}\\"` - ) - ) - notifyingOnAnyLine('relay_reserve_placement_off_with_reserve_cells', event, 'cloud_run_revision', 'revision') -}) - -test('the withheld dead-cell verdict alert counts every director withheld line, gated off by default', () => { - const event = 'orca_relay_dead_cell_verdict_withheld' - assert.match(poller, new RegExp(`event: '${event}', cellId, \\.\\.\\.verdict`)) - const metric = block('google_logging_metric', 'relay_dead_cell_verdict_withheld') - assert.ok(metric.includes(`jsonPayload.event=\\"${event}\\""`)) - assert.ok(metric.includes('withheld = "EXTRACT(jsonPayload.withheld)"')) - const alert = block('google_monitoring_alert_policy', 'relay_dead_cell_verdict_withheld') - for (const body of [metric, alert]) assert.match(body, /count = var\.relay_dead_cell_verdict_alert \? 1 : 0/) - assert.match(alert, /threshold_value = 0\n/) - assert.match(alert, /notification_channels = var\.relay_alert_notification_channels/) - assert.ok(alert.includes('logging.googleapis.com/user/${google_logging_metric.relay_dead_cell_verdict_withheld[0].name}')) - assert.ok(alert.includes('group_by_fields = ["metric.label.\\"withheld\\""]')) - assert.match(read('../../infra/terraform/variables.tf'), /variable "relay_dead_cell_verdict_alert" \{[\s\S]*?default = false/) - assert.match(read('../../infra/terraform/environments/staging.tfvars'), /relay_dead_cell_verdict_alert\s+= true/) -}) - -test('the reserve watch alarm pages on any line the watch runner writes, per cell', () => { - // The runner writes a global-resource entry under its own log name. - assert.match(watch, /export const ALARM_EVENT = 'orca_relay_reserve_watch_alarm'/) - assert.match(watch, /export const ALARM_LOG_NAME = 'orca-relay-reserve-watch'/) - assert.match(watch, /JSON\.stringify\(\{ event: ALARM_EVENT, cellId, kind, reason \}\)/) - const metric = block('google_logging_metric', 'relay_reserve_watch_alarm') - assert.ok( - metric.includes( - 'resource.type=\\"global\\" AND logName=\\"projects/${var.project_id}/logs/orca-relay-reserve-watch\\" AND jsonPayload.event=\\"orca_relay_reserve_watch_alarm\\"' - ) - ) - assert.ok(metric.includes('cell_id = "EXTRACT(jsonPayload.cellId)"')) - notifyingOnAnyLine('relay_reserve_watch_alarm', 'orca_relay_reserve_watch_alarm', 'global', 'cell_id') - // The deploy identity the workflow runs as can write that line, and only in production. - const grant = /resource "google_project_iam_member" "github_log_writer" \{([\s\S]*?)\n\}/.exec(githubActions)?.[1] - assert.ok(grant, 'github_log_writer grant not found') - assert.match(grant, /count = local\.relay_create_production_ops_identity \? 1 : 0/) - assert.match(grant, /role = "roles\/logging\.logWriter"/) - assert.match(grant, /member = google_service_account\.github_deploy\[0\]\.member/) -}) diff --git a/cloud/dev/scripts/relay-reserve-watch-rules.mjs b/cloud/dev/scripts/relay-reserve-watch-rules.mjs deleted file mode 100644 index 7b602bdad7f..00000000000 --- a/cloud/dev/scripts/relay-reserve-watch-rules.mjs +++ /dev/null @@ -1,494 +0,0 @@ -// Stop rules for a cell watched in reserve mode (step 5 design §9, "Auto-abort per flip"). Pure: -// it takes one already-read sample and the watch state, and returns the next state and whether -// to flip the cell back. The reader lives in watch-relay-reserve-cell.mjs. -import { backgroundOf, SHADOW_GATE_THRESHOLDS } from './relay-same-cap-shadow-gate-verdict.mjs' - -const MINUTE_MS = 60_000 - -export const RESERVE_WATCH_RULES = { - // The roll ladder's warn line on non-drain 503s: max(1.5x, background + 20), so a zero background - // is not tripped by one 503. Three complete minutes in a row, not two. The read spans more minutes - // than that so a late publishing front still leaves three published minutes to judge. - director503: { - multiple: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMultiple, - marginPerMinute: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMarginPerMinute, - consecutiveMinutes: 3, - readMinutes: 10, - // Cloud Run publishes a minute's request count up to 3 minutes late. - publishDelayMs: 3 * MINUTE_MS, - // The roll ladder's one-minute would-block line, max(10x, 200), here held for 3 minutes. Every - // broad fault since 09-09 ran in the thousands a minute; one stalled instance peaked at 178 - // (10-20/21) and never held 200 for 3 minutes. - broad: { - multiple: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.spikeMultiple, - floorPerMinute: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.spikeFloor - }, - // Between the two lines, two instances each over their even share of the fleet line for the - // same 3 minutes. 10-21 07:55Z: no instance held a fifth, the stalled one fell to 5 in minute 3. - instances: { minOver: 2 } - }, - // The cell closes a demoted seat after 60 s (DEMOTION_CLOSE_MS); 30 s more covers a director's - // map still holding the closed seat. - duplicateSeat: { graceMs: 60_000 + 30_000, lookbackMs: 15 * MINUTE_MS }, - // The floor keeps an almost empty cell's ordinary churn from reading as a 5% drop. - hostDrop: { share: 0.05, windowMs: 5 * MINUTE_MS, minHosts: 10 }, - ledgerDrift: { share: 0.005, sustainMs: 2 * MINUTE_MS }, - // A placing director logs its ledger summary every 60 s; none in this long is no evidence. - ledgerSummaryMaxAgeMs: 150_000, - // A watch dispatched before the reserve write applies waits this long for the cell to report it. - reserveApplyWaitMs: 10 * MINUTE_MS -} - -// The fleet's non-drain 503s no instance's own lines account for. Unclamped per minute: the fleet -// buckets by request minute and an instance's lines by 30 s, so a minute carries that skew and -// only the sum over the judged minutes is judged, against the whole fleet line: no one -// instance's stall explains it then. Its largest 3-minute sum in 10-20/21 was 38, against 90. An -// instance answering 503s without logging its lines lands here and trips on purpose: a director -// that stops logging is not the database stall the split lets through (10-21 07:55Z kept logging). -export const UNATTRIBUTED_503S = 'unattributed' - -// Precedence: the first tripped rule names the verdict, so exact signals come before proxies. The -// sample line's `rules` keys follow this order too. -export const RULE_NAMES = ['deadMan', 'overrun', 'director503', 'duplicateSeat', 'hostDrop', 'ledgerDrift'] - -// Step 6 design §7: judged only while the cell's switch file says phonePath=memory, against the -// cell's own runtime metrics for the 10 minutes before phonePath went memory. A trip turns the -// phone path off (phonePath=db) and the watch goes on with the reserve rules. -export const PHONE_WATCH_RULES = { - // Each rule trips on three complete minutes in a row over its line. - consecutiveMinutes: 3, - readMinutes: 10, - // Judged as a share of the minute's phone attempts (the signal plus completed accepts; for - // confirm failures, the accepts), not a count: a count line false-tripped 33/121 5-hour watches - // on busy c28 (10-18/19), where bursts of refusals arrive with the evening rise in accepts. A - // minute with fewer attempts is not judged, so low-volume c7 (~2 accepts/min) is mostly unjudged. - minAttemptsPerMinute: 4, - // Over half of the minute's attempts, and 0.25 above the share before the flip, capped so two - // failures per accept always trips. Exactly half is not enough: c28 (04:08Z) and c7 (09:13Z) - // both show 3 minutes at 50% of host-and-phone auth failures. - shareFloor: 0.5, - shareOverBackground: 0.25, - shareCeiling: 0.6, - // Page only: confirms still work, only their audit rows or renewals are lost. - pageAfterMs: 5 * MINUTE_MS -} -export const PHONE_RULE_NAMES = [ - 'phoneCredentialRefusals', - 'phoneConfirmFailures', - 'phoneAttachDeadlines', - 'phoneAcceptDatabaseErrors' -] -const PHONE_RULE_SIGNALS = { - phoneCredentialRefusals: 'credentialRefusals', - phoneConfirmFailures: 'confirmFailures', - phoneAttachDeadlines: 'attachDeadlines', - phoneAcceptDatabaseErrors: 'acceptDatabaseErrors' -} - -const sum = (values) => values.reduce((total, value) => total + value, 0) - -// The share line for a signal whose share of attempts before the flip was `backgroundShare`. -export function phoneShareLine(backgroundShare) { - const { shareFloor, shareOverBackground, shareCeiling } = PHONE_WATCH_RULES - return Math.min(shareCeiling, Math.max(shareFloor, backgroundShare + shareOverBackground)) -} - -// A completed accept makes one confirm, so confirm failures are a share of accepts (capped at -// all of them); every other signal is an attempt that did not complete. -function attemptsAndShare(signal, count, completedAccepts) { - if (signal === 'confirmFailures') { - return { attempts: completedAccepts, share: completedAccepts === 0 ? 0 : Math.min(count, completedAccepts) / completedAccepts } - } - const attempts = count + completedAccepts - return { attempts, share: attempts === 0 ? 0 : count / attempts } -} - -// Over the line: enough attempts that minute, and strictly more than the line refused or failed. -export function phoneMinuteOver(signal, count, completedAccepts, line) { - const { attempts, share } = attemptsAndShare(signal, count, completedAccepts) - return attempts >= PHONE_WATCH_RULES.minAttemptsPerMinute && share > line -} - -// The 10 minutes before phonePath went memory, from `readCellPhoneMinutes`: each signal's share of -// attempts. Null until every minute is metered. -export function measuredPhoneBackground(read) { - if (!read.ok || read.unmetered.length > 0 || read.minutes.length === 0) return null - const accepted = sum(read.series.completedAccepts) - return { - share: Object.fromEntries( - Object.values(PHONE_RULE_SIGNALS).map((signal) => { - const count = sum(read.series[signal]) - return [signal, attemptsAndShare(signal, count, accepted).share] - }) - ), - minutes: read.minutes.length - } -} - -export function initialWatchState({ cellId, startedAt, durationMinutes, background503PerMinute, flippedAt }) { - return { - v: 1, - cellId, - startedAt, - deadlineAt: startedAt + durationMinutes * MINUTE_MS, - state: 'watching', - verdict: null, - samples: 0, - unverifiableStreak: 0, - // The background is measured in the minutes before the flip, unless the operator supplied it. - flippedAt: flippedAt ?? null, - background: - background503PerMinute === null ? null : { medianPerMinute: background503PerMinute, source: 'input' }, - hosts: [], - reserveSeenAt: null, - ledgerAboveSince: null, - seatBehindRowFirstSeen: {}, - // Step 6: what a trip writes back, and the phone rules' state. `phoneSince` is when the switch - // file went phonePath=memory (flippedAt if it already said so); `phoneOffAt` when the watch - // turned it off; `phoneTrip` a phone verdict the segment has yet to write. - flip: null, - lastSampleAt: null, - phoneSince: null, - phoneBackground: null, - phoneOffAt: null, - phoneTrip: null, - phoneOffFailures: 0, - phoneUnverifiableStreak: 0, - auditDropsLast: null, - auditDropsSince: null, - renewalFailuresSince: null, - paged: [] - } -} - -// The background from the per-minute non-drain 503s before the flip, the live samples' measure. -// `publishedThrough` is the newest minute any response code has a point for: a 503 minute at or -// before it with no point had none, and the background waits (null) until it covers the last -// minute. Unmetered minutes are left out; under half the minutes metered is no background. -export function measuredBackground({ perMinute, minutes, publishedThrough, unmetered }) { - if (minutes.length === 0 || publishedThrough === null || publishedThrough < minutes.at(-1)) return null - const metered = minutes.filter((minute) => !unmetered.includes(minute)) - if (metered.length * 2 < minutes.length) return null - const background = backgroundOf({ minutes: metered, series: metered.map((minute) => perMinute[minute] ?? 0), unverified: false }) - return { medianPerMinute: background.medianPerMinute, minutes: background.minutes, source: 'measured' } -} - -const pass = (detail = {}) => ({ status: 'pass', ...detail }) -const trip = (reason, detail = {}) => ({ status: 'trip', reason, ...detail }) -const unverifiable = (reason) => ({ status: 'unverifiable', reason }) -// Not readable yet by design, so it does not count toward the unverifiable streak. -const pending = (reason) => ({ status: 'pending', reason }) -const failedPhoneRead = { ok: false, error: 'no cell phone metrics read' } -const minuteFloor = (at) => Math.floor(at / MINUTE_MS) * MINUTE_MS - -function judgeDeadMan(cell) { - if (!cell.ok) return unverifiable(cell.error) - if (cell.admitModeEffective !== 'db' && cell.admitModeEffective !== 'reserve') { - return unverifiable('runtime-status reports no admitModeEffective') - } - if (cell.admitModeEffective === 'db') { - return trip('the cell tripped its dead-man (reserve configured, db effective); Postgres still records reserve') - } - return pass() -} - -// The cell ledger's total (seats, bookings, in-flight and phone units) against its hard cap. The -// ledger refuses past the cap, so this is an invariant check: it trips only if that refusal broke. -function judgeOverrun(cell) { - if (!cell.ok) return unverifiable(cell.error) - if (!Number.isInteger(cell.hardCap) || !Number.isInteger(cell.enforcedUnits)) { - return unverifiable('runtime-status reports no hard cap or ledger units') - } - if (cell.enforcedUnits > cell.hardCap) { - return trip(`ledger units ${cell.enforcedUnits} over hard cap ${cell.hardCap}`, { - enforcedUnits: cell.enforcedUnits, - hardCap: cell.hardCap - }) - } - return pass({ enforcedUnits: cell.enforcedUnits, hardCap: cell.hardCap }) -} - -// Fleet-wide non-drain director 503s: no existing signal attributes a 503 to the cell a host was -// bound for, so a problem local to one cell can stay under the fleet's line. This proxy can miss. -// Over the fleet line, it trips only when at least two instances are each over their share of it: -// one director instance whose database calls stall alone can carry the fleet over (10-21 07:55Z, -// 45/164/40 with 54/155/5 on one instance). Over the broad line no instance split is needed. -function judgeDirector503(read, state, at) { - const { multiple, marginPerMinute, consecutiveMinutes, publishDelayMs, broad, instances } = RESERVE_WATCH_RULES.director503 - const { background } = state - if (!background) { - const publishesAt = state.flippedAt === null ? null : Math.floor(state.flippedAt / MINUTE_MS) * MINUTE_MS + publishDelayMs - if (publishesAt !== null && at < publishesAt) return pending('the 503 background before the flip is not published yet') - return unverifiable('no 503 background yet') - } - if (!read.ok) return unverifiable(read.error) - const perMinute = background.medianPerMinute - const above = Math.max(perMinute * multiple, perMinute + marginPerMinute) - // Minute keys sort as time. Only minutes the publishing front has reached are judged: a 503 - // minute there with no point had none, one past it is not published yet and is left out. - if (read.publishedThrough === null) return unverifiable('the director request series is empty') - const recent = read.minutes.filter((minute) => minute <= read.publishedThrough).slice(-consecutiveMinutes) - if (recent.length < consecutiveMinutes) { - return unverifiable(`director requests are published only through ${read.publishedThrough}`) - } - const unmetered = recent.filter((minute) => read.unmetered.includes(minute)) - if (unmetered.length > 0) return unverifiable(`no director runtime metrics for ${unmetered.join(', ')}`) - const counts = recent.map((minute) => read.perMinute[minute] ?? 0) - const broadLine = Math.max(perMinute * broad.multiple, broad.floorPerMinute) - const detail = { backgroundPerMinute: perMinute, above, broadAbove: broadLine, recent: Object.fromEntries(recent.map((m, i) => [m, counts[i]])) } - const sustained = `for ${consecutiveMinutes} min (${counts.join(', ')})` - if (counts.every((count) => count > broadLine)) { - return trip(`director 503s above ${broadLine}/min ${sustained}`, detail) - } - if (!counts.every((count) => count > above)) return pass(detail) - if (!read.instances?.ok) return unverifiable(read.instances?.error ?? 'no per-instance director 503s') - // The fewest directors logging in a judged minute, so a rollout's overlap cannot shrink the share. - const directors = Math.min(...recent.map((minute) => read.instances.directors?.[minute])) - if (!Number.isInteger(directors)) return unverifiable(`no director instance count for ${recent.join(', ')}`) - const instanceAbove = above / Math.max(directors, instances.minOver) - const { [UNATTRIBUTED_503S]: rest = {}, ...byInstance } = read.instances.perMinute - const unattributed = recent.reduce((sum, minute) => sum + (rest[minute] ?? 0), 0) - const over = Object.entries(byInstance) - .filter(([, byMinute]) => recent.every((minute) => (byMinute[minute] ?? 0) > instanceAbove)) - .map(([instance]) => instance) - const split = { ...detail, instanceAbove, instancesOver: over.length, unattributed } - if (unattributed > above * recent.length) { - return trip(`director 503s above ${above}/min ${sustained}, ${unattributed} of them no director instance logged`, split) - } - if (over.length >= instances.minOver) { - return trip(`director 503s above ${above}/min ${sustained}, ${over.length} instances each above ${instanceAbove}/min`, split) - } - return pass(split) -} - -// A seat the ledger reconcile found behind its row and found again after the demotion should -// have closed it. The line names no host, so (director instance, cell, epoch, rowEpoch) stands in -// for one: a proxy that can only over-count, when two hosts share that key. -function judgeDuplicateSeat(read, firstSeen, at) { - if (!read.ok) return { verdict: unverifiable(read.error), firstSeen } - const { graceMs, lookbackMs } = RESERVE_WATCH_RULES.duplicateSeat - const next = Object.fromEntries( - Object.entries(firstSeen).filter(([, seenAt]) => seenAt >= at - 2 * lookbackMs) - ) - for (const report of [...read.reports].sort((left, right) => left.at - right.at)) { - next[report.key] = Math.min(next[report.key] ?? report.at, report.at) - } - for (const report of read.reports) { - const ageMs = report.at - next[report.key] - if (ageMs > graceMs) { - return { - verdict: trip(`a seat behind its ledger row (${report.key}) is still seated ${Math.round(ageMs / 1000)} s after first report`), - firstSeen: next - } - } - } - return { verdict: pass({ reportsInLookback: read.reports.length }), firstSeen: next } -} - -function judgeHostDrop(hosts) { - const { share, minHosts, windowMs } = RESERVE_WATCH_RULES.hostDrop - const current = hosts.at(-1) - if (!current) return unverifiable('no host count') - if (hosts.some((entry) => entry.draining)) return pass({ controls: current.controls, draining: true }) - const peak = Math.max(...hosts.map((entry) => entry.controls)) - const drop = peak - current.controls - const detail = { controls: current.controls, peakInWindow: peak } - if (peak > 0 && drop >= minHosts && drop / peak > share) { - return trip(`connected hosts fell ${peak} -> ${current.controls} within ${windowMs / MINUTE_MS} min outside a drain`, detail) - } - return pass(detail) -} - -// Rows the placing directors have not written (queued) or gave up on (dropped), per seat. The -// summaries are fleet-wide: exact while this is the only reserve cell, an over-count otherwise. -function judgeLedgerDrift(cell, ledger, aboveSince, at) { - if (!cell.ok) return { verdict: unverifiable(cell.error), aboveSince } - if (!ledger.ok) return { verdict: unverifiable(ledger.error), aboveSince } - const { share, sustainMs } = RESERVE_WATCH_RULES.ledgerDrift - const ratio = ledger.unwrittenRows / Math.max(cell.controls, 1) - const detail = { unwrittenRows: ledger.unwrittenRows, directors: ledger.directors, ratio } - if (ratio <= share) return { verdict: pass(detail), aboveSince: null } - const since = aboveSince ?? at - if (at - since >= sustainMs) { - return { - verdict: trip(`ledger drift ${(ratio * 100).toFixed(2)}% for ${Math.round((at - since) / 1000)} s`, detail), - aboveSince: since - } - } - return { verdict: pass({ ...detail, aboveForMs: at - since }), aboveSince: since } -} - -// A counter still rising (or failing) every sample for 5 minutes pages once; a single drop does not. -function judgeSustained(rising, since, at) { - const next = rising ? (since ?? at) : null - return { since: next, sustained: next !== null && at - next >= PHONE_WATCH_RULES.pageAfterMs } -} - -export function phoneSinceOf(cell, state) { - if (state.phoneSince !== null || !cell.ok || cell.phonePathConfigured !== 'memory') return state.phoneSince - // Memory from the first sample: the operator's flipped-at. Later: the sample before this one. - return state.lastSampleAt === null ? state.flippedAt : state.lastSampleAt -} - -// The phone rules for one sample, or none while the switch file does not say phonePath=memory. -function judgePhoneRules(cell, phone, state, at) { - if (!cell.ok || cell.phonePathConfigured !== 'memory' || state.phoneOffAt !== null) return {} - const background = state.phoneBackground - if (!background) { - const verdict = - state.phoneSince === null - ? unverifiable('the phone rules need --flipped-at: their background is the 10 minutes before it') - : at < minuteFloor(state.phoneSince) + RESERVE_WATCH_RULES.director503.publishDelayMs - ? pending('the phone background before phonePath went memory is not published yet') - : unverifiable('no phone background yet') - return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, verdict])) - } - if (!phone.ok) return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, unverifiable(phone.error)])) - const { consecutiveMinutes } = PHONE_WATCH_RULES - // Only the judged minutes need metrics; a gap earlier in the read changes nothing. - const judged = phone.minutes.slice(-consecutiveMinutes) - const gaps = phone.unmetered.filter((minute) => judged.includes(minute)) - if (gaps.length > 0) { - const gap = unverifiable(`no cell runtime metrics for ${gaps.join(', ')}`) - return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, gap])) - } - const accepts = phone.series.completedAccepts.slice(-consecutiveMinutes) - return Object.fromEntries( - PHONE_RULE_NAMES.map((name) => { - const signal = PHONE_RULE_SIGNALS[name] - const line = phoneShareLine(background.share[signal]) - const recent = phone.series[signal].slice(-consecutiveMinutes) - const detail = { backgroundShare: background.share[signal], line, recent, accepts } - const over = - recent.length === consecutiveMinutes && recent.every((count, index) => phoneMinuteOver(signal, count, accepts[index], line)) - return [ - name, - over ? trip(`${signal} over ${Math.round(line * 100)}% of attempts for ${consecutiveMinutes} min (${recent.join(', ')} against ${accepts.join(', ')} accepts)`, detail) : pass(detail) - ] - }) - ) -} - -// Page-only signals: audit rows dropped since the previous sample, and renewal writes failing in -// every one of the last 5 published minutes. -function judgePhonePages(cell, phone, state, at) { - const dropped = cell.ok && Number.isInteger(cell.resumeAuditDropped) ? cell.resumeAuditDropped : null - const audit = judgeSustained( - dropped !== null && state.auditDropsLast !== null && dropped > state.auditDropsLast, - state.auditDropsSince, - at - ) - const renewals = phone?.ok ? (phone.series.renewalFailures ?? []).slice(-PHONE_WATCH_RULES.pageAfterMs / MINUTE_MS) : [] - const renewal = judgeSustained(renewals.length > 0 && renewals.every((count) => count > 0), state.renewalFailuresSince, at) - const pages = [] - if (audit.sustained && !state.paged.includes('audit-drops')) pages.push('audit-drops') - if (renewal.sustained && !state.paged.includes('renewal-failures')) pages.push('renewal-failures') - return { auditDropsLast: dropped ?? state.auditDropsLast, auditDropsSince: audit.since, renewalFailuresSince: renewal.since, pages } -} - -/** - * One sample. `sample.cell` is the cell's runtime-status, `director503` the complete minutes of - * non-drain director 503s with the request series' publishing front (`publishedThrough`) and the - * minutes no runtime metrics cover (`unmetered`), `ledger` the placing directors' newest ledger summaries, `seatBehindRow` this - * cell's reconcile reports in the lookback. Each is `{ ok: false, error }` when it could not be read. - */ -export function evaluateWatchSample(state, sample, { unverifiableLimit }) { - const at = sample.at - const next = { ...state, samples: state.samples + 1 } - const { cell } = sample - if (cell.ok && cell.admitModeConfigured !== 'reserve') { - const seen = state.reserveSeenAt !== null - if (!seen && at < state.startedAt + RESERVE_WATCH_RULES.reserveApplyWaitMs) { - // Not judged, and the streak holds: the reserve write may not have applied yet. - const rules = { reserve: pending(`switch file says ${cell.admitModeConfigured}; waiting for reserve to apply`) } - return { state: next, rules, line: sampleLine(next, sample, rules), pages: [] } - } - // Someone else flipped it back (or it never became reserve): nothing left to watch. - next.state = seen ? 'stopped' : 'refused' - next.verdict = seen - ? `${state.cellId} is not in reserve mode (switch file says ${cell.admitModeConfigured})` - : `${state.cellId} did not report reserve mode within ${RESERVE_WATCH_RULES.reserveApplyWaitMs / MINUTE_MS} min (switch file says ${cell.admitModeConfigured})` - return { state: next, rules: {}, line: sampleLine(next, sample, {}), pages: [] } - } - if (cell.ok) { - next.reserveSeenAt = state.reserveSeenAt ?? at - next.hosts = [...state.hosts, { at, controls: cell.controls, draining: cell.draining }].filter( - (entry) => entry.at > at - RESERVE_WATCH_RULES.hostDrop.windowMs - ) - } - const duplicate = judgeDuplicateSeat(sample.seatBehindRow, state.seatBehindRowFirstSeen, at) - next.seatBehindRowFirstSeen = duplicate.firstSeen - const ledger = judgeLedgerDrift(cell, sample.ledger, state.ledgerAboveSince, at) - next.ledgerAboveSince = ledger.aboveSince - next.lastSampleAt = at - next.phoneSince = phoneSinceOf(cell, state) - const phoneState = { ...state, phoneSince: next.phoneSince } - const pageCheck = judgePhonePages(cell, sample.phone, state, at) - next.auditDropsLast = pageCheck.auditDropsLast - next.auditDropsSince = pageCheck.auditDropsSince - next.renewalFailuresSince = pageCheck.renewalFailuresSince - next.paged = [...state.paged, ...pageCheck.pages] - const pages = pageCheck.pages - const rules = { - deadMan: judgeDeadMan(cell), - overrun: judgeOverrun(cell), - director503: judgeDirector503(sample.director503, state, at), - duplicateSeat: duplicate.verdict, - hostDrop: cell.ok ? judgeHostDrop(next.hosts) : unverifiable(cell.error), - ledgerDrift: ledger.verdict, - ...judgePhoneRules(cell, sample.phone ?? failedPhoneRead, phoneState, at) - } - const tripped = RULE_NAMES.find((name) => rules[name].status === 'trip') - if (tripped) { - next.state = 'tripped' - next.flip = 'admitMode' - next.verdict = `flip back ${state.cellId}: ${tripped}: ${rules[tripped].reason}` - return { state: next, rules, line: sampleLine(next, sample, rules), pages } - } - // A phone trip turns off only the phone path; the segment writes it and the watch goes on. - const phoneNames = PHONE_RULE_NAMES.filter((name) => name in rules) - const phoneTripped = phoneNames.find((name) => rules[name].status === 'trip') - const phoneMissing = phoneNames.filter((name) => rules[name].status === 'unverifiable') - next.phoneUnverifiableStreak = phoneMissing.length > 0 ? state.phoneUnverifiableStreak + 1 : 0 - if (phoneTripped) { - next.phoneTrip = `turn off the phone path on ${state.cellId}: ${phoneTripped}: ${rules[phoneTripped].reason}` - } else if (next.phoneUnverifiableStreak >= unverifiableLimit) { - next.phoneTrip = - `turn off the phone path on ${state.cellId}: unverifiable: ${next.phoneUnverifiableStreak} samples ` + - `in a row could not read ${phoneMissing.join(', ')}` - } - const missing = RULE_NAMES.filter((name) => rules[name].status === 'unverifiable') - const waiting = [...RULE_NAMES, ...phoneNames].some((name) => rules[name].status === 'pending') - // A pending rule holds the streak where it was: neither evidence of health nor a gap. - next.unverifiableStreak = missing.length > 0 ? state.unverifiableStreak + 1 : waiting ? state.unverifiableStreak : 0 - if (next.unverifiableStreak >= unverifiableLimit) { - next.state = 'tripped' - next.flip = 'admitMode' - next.verdict = - `flip back ${state.cellId}: unverifiable: ${next.unverifiableStreak} samples in a row could not ` + - `read ${missing.join(', ')}` - } else if (at >= state.deadlineAt && next.phoneTrip) { - // Still unwritten at the end: the workflow's flip back writes it, and pages if that fails too. - next.state = 'tripped' - next.flip = 'phonePath' - next.verdict = `${next.phoneTrip}; phonePath=db did not land before the watch ended` - } else if (at >= state.deadlineAt) { - next.state = 'completed' - next.verdict = `${state.cellId} stayed healthy in reserve mode for the whole watch` - } - return { state: next, rules, line: sampleLine(next, sample, rules), pages } -} - -function sampleLine(state, sample, rules) { - return JSON.stringify({ - event: 'orca_relay_reserve_watch_sample', - cellId: state.cellId, - at: new Date(sample.at).toISOString(), - sample: state.samples, - state: state.state, - unverifiableStreak: state.unverifiableStreak, - rules, - ...(state.phoneTrip ? { phoneTrip: state.phoneTrip } : {}), - ...(state.verdict ? { verdict: state.verdict } : {}) - }) -} diff --git a/cloud/dev/scripts/relay-same-cap-job-mode-conditions.test.mjs b/cloud/dev/scripts/relay-same-cap-job-mode-conditions.test.mjs deleted file mode 100644 index 9ee83d9eb32..00000000000 --- a/cloud/dev/scripts/relay-same-cap-job-mode-conditions.test.mjs +++ /dev/null @@ -1,163 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { readRelayWorkflow } from './relay-repository.mjs' - -const parent = readRelayWorkflow('deploy-relay-production-same-cap.yml') -const job = readRelayWorkflow('deploy-relay-production-same-cap-job.yml') - -function jobAcceptedModes() { - const match = /\[\[ "\$\{DEPLOY_MODE\}" =~ \^\(([a-z|-]+)\)\$ \]\]/.exec(job) - assert.ok(match, 'the job no longer validates DEPLOY_MODE') - return new Set(match[1].split('|')) -} - -// Mirrors the gate step's dispatch-mode to job-mode mapping. -function parentJobModes() { - const options = /mode:\n\s+description:[^\n]*\n(?:\s+[a-z]+:[^\n]*\n)*?\s+options: \[([^\]]+)\]/ - .exec(parent) - assert.ok(options, 'the parent has no dispatch mode options') - const collapsed = /if \[\[ "\$\{MODE\}" =~ \^\(([a-z|-]+)\)\$ \]\]; then\n\s+echo 'job-mode=([a-z]+)'/ - .exec(parent) - assert.ok(collapsed, 'the parent no longer collapses apply modes into one job mode') - const collapsedModes = new Set(collapsed[1].split('|')) - return new Set( - options[1].split(',').map((mode) => mode.trim()).map((mode) => - collapsedModes.has(mode) ? collapsed[2] : mode - ) - ) -} - -function stepCondition(name) { - const start = job.indexOf(`- name: ${name}\n`) - assert.notEqual(start, -1, `job has no step named ${name}`) - const step = job.slice(start, job.indexOf('\n - ', start + 1)) - const condition = /^ {8}if: \$\{\{ (.+) \}\}$/m.exec(step) - assert.ok(condition, `${name} has no if condition`) - return condition[1] -} - -// Just enough of the expression grammar for the step conditions this file compares. -function evaluate(expression, context) { - const tokens = expression.match(/'[^']*'|[A-Za-z_.]+|==|!=|&&|\|\||[()]/g) - assert.equal(tokens.join(''), expression.replaceAll(' ', ''), `unparsed: ${expression}`) - let index = 0 - const value = () => { - const token = tokens[index++] - if (token === '(') { - const inner = or() - assert.equal(tokens[index++], ')') - return inner - } - if (token.startsWith("'")) return token.slice(1, -1) - assert.ok(Object.hasOwn(context, token), `unknown operand ${token}`) - return context[token] - } - const comparison = () => { - const left = value() - if (tokens[index] !== '==' && tokens[index] !== '!=') return left - const operator = tokens[index++] - const right = value() - return operator === '==' ? left === right : left !== right - } - const and = () => { - let result = comparison() - while (tokens[index] === '&&') { - index++ - result = comparison() && result - } - return result - } - const or = () => { - let result = and() - while (tokens[index] === '||') { - index++ - result = or() || result - } - return result - } - const result = or() - assert.equal(index, tokens.length, `unparsed tail: ${expression}`) - return result -} - -test('the parent passes only modes the job accepts', () => { - assert.deepEqual(parentJobModes(), jobAcceptedModes()) -}) - -// A comparison against a mode the job never receives is constant, so its step is dead (#24259). -test('every job mode comparison names a mode the job can receive', () => { - const accepted = jobAcceptedModes() - const compared = [...job.matchAll(/inputs\.mode\s*[!=]=\s*'([^']*)'/g)].map(([, mode]) => mode) - assert.ok(compared.length > 0) - for (const mode of compared) assert.ok(accepted.has(mode), `job compares against ${mode}`) -}) - -test('the headroom gate runs whenever the drain runs, and in verify', () => { - const headroom = stepCondition("Require free general-cell slots for the selected cell's hosts") - const drain = stepCondition('Reversibly isolate and drain only the selected cell') - for (const mode of parentJobModes()) { - for (const resume of ['true', 'false']) { - const context = { 'inputs.mode': mode, 'env.ROLLBACK_RESUME': resume } - if (evaluate(drain, context)) { - assert.ok(evaluate(headroom, context), `drain without headroom check: ${mode}/${resume}`) - } - } - } - assert.ok(evaluate(headroom, { 'inputs.mode': 'verify', 'env.ROLLBACK_RESUME': 'false' })) -}) - -// The roll's fleet-health gate lives inside the job now; a separate monitor run must not creep back. -test('neither same-cap workflow depends on monitor evidence', () => { - for (const [name, text] of [['parent', parent], ['job', job]]) { - for (const pattern of [ - /monitor-run-(id|attempt)/, - /MONITOR_RUN_/, - /relay-monitor-evidence/, - /relay-monitor-dry-run-/, - /monitor-consumed/, - /gate-override/, - /GATE_OVERRIDE/, - /SKIP_RELAY_MONITOR_GATE/, - /--state-file/ - ]) { - assert.doesNotMatch(text, pattern, `${name} still references ${pattern}`) - } - } -}) - -test('every apply wave samples fleet health right before it drains', () => { - const sampleName = 'Sample fleet health for a window sized to this drain' - const sample = stepCondition(sampleName) - const drain = stepCondition('Reversibly isolate and drain only the selected cell') - for (const mode of parentJobModes()) { - for (const resume of ['true', 'false']) { - const context = { 'inputs.mode': mode, 'env.ROLLBACK_RESUME': resume } - if (mode === 'apply' && evaluate(drain, context)) { - assert.ok(evaluate(sample, context), `apply drain without a sample: ${resume}`) - } - } - } - assert.equal(evaluate(sample, { 'inputs.mode': 'verify' }), false) - assert.equal(evaluate(sample, { 'inputs.mode': 'rollback' }), false) - const at = (name) => { - const index = job.indexOf(`- name: ${name}\n`) - assert.notEqual(index, -1, name) - return index - } - const order = [ - 'Recheck aggregate SQL, pool, reconnect, migration, and selector safety', - "Require free general-cell slots for the selected cell's hosts", - sampleName, - 'Reversibly isolate and drain only the selected cell' - ].map(at) - assert.deepEqual(order, [...order].sort((left, right) => left - right)) - const step = job.slice(at(sampleName), job.indexOf('\n - ', at(sampleName) + 1)) - // The window is sized from the headroom step's own count, and each wave offsets its selector. - assert.match(step, /pnpm incident:relay-pre-drain-sample/) - assert.match(step, /--target-hosts "\$\{TARGET_HOSTS\}"/) - // The crash rule exempts the cell being rolled, so it must be told which one that is. - assert.match(step, /--target-cell-id "\$\{TARGET_CELL_ID\}"/) - assert.match(step, /--wave-index "\$\{WAVE_INDEX\}"/) - assert.match(step, /--selector-wave-delta "\$\{SELECTOR_WAVE_DELTA\}"/) - assert.match(job, /echo "TARGET_HOSTS=\$\{TARGET_HOSTS\}" >> "\$\{GITHUB_ENV\}"/) -}) diff --git a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs deleted file mode 100644 index 2fea683b82e..00000000000 --- a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs +++ /dev/null @@ -1,692 +0,0 @@ -import assert from 'node:assert/strict' -import { spawnSync } from 'node:child_process' -import { readFileSync } from 'node:fs' -import { describe, it } from 'node:test' -import { parseProductionCapacityCellArguments } from './prepare-relay-production-capacity-canary.mjs' -import { - SAME_CAP_CELLS, - SAME_CAP_MIGRATION_ONLY_CELLS, - entryAdmission, - selectorWaveDelta -} from './relay-production-same-cap-wave.mjs' -import { readRelayWorkflow } from './relay-repository.mjs' -import { validateCapacityPlan } from './validate-relay-capacity-plan.mjs' - -const workflow = readRelayWorkflow('deploy-relay-production-same-cap-job.yml') -const production = readFileSync( - new URL('../../infra/terraform/environments/production.tfvars', import.meta.url), - 'utf8' -) -const REHOME_SOURCE_CELLS = rehomeSourceCells() -const DIRECTOR_IDENTITY = 'relay-director@onorca-cloud.iam.gserviceaccount.com' -const CAPACITY_IDENTITY = 'orca-cloud-gha-cap@onorca-cloud.iam.gserviceaccount.com' -const AUDIENCE = 'https://relay.onorca.dev/v1/admin/host-drain' -const ROLLBACK_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'d'.repeat(64)}` -const TARGET_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'e'.repeat(64)}` - -// The startup template emits rehome trust only for cells in this list, so it is what decides -// whether a cell's plan may carry those lines at all. -function rehomeSourceCells() { - const start = production.indexOf('relay_region_rehome_source_cell_ids = [') - assert.notEqual(start, -1, 'production.tfvars has no rehome source cell list') - const end = production.indexOf(']', start) - assert.notEqual(end, -1, 'the rehome source cell list is unterminated') - return new Set( - [...production.slice(start, end).matchAll(/"([^"]+)"/g)].map(([, cell]) => cell) - ) -} - -// The job cross-checks its pinned pool against the committed map; model the same read. -function tfvarsDatabasePoolMax(cellId) { - return tfvarsCellBlock(cellId).match(/database_pool_max\s*=\s*(\d+)/)?.[1] ?? '10' -} - -function tfvarsHardCap(cellId) { - const cap = /connection_hard_cap\s*=\s*(\d+)/.exec(tfvarsCellBlock(cellId))?.[1] - assert.notEqual(cap, undefined, `${cellId} has no connection_hard_cap`) - return cap -} - -function tfvarsCellBlock(cellId) { - const start = production.indexOf(`"${cellId}" = {`) - assert.notEqual(start, -1, `${cellId} is missing from production.tfvars`) - return production.slice(start, production.indexOf('\n }', start)) -} - -function startupScript({ cap, image, trusted, pool, capacityIdentity = CAPACITY_IDENTITY }) { - return [ - ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`, - ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`, - ...(capacityIdentity === null - ? [] - : [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`]), - ...(pool === undefined - ? [] - : [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]), - ...(trusted ? [ - ` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${DIRECTOR_IDENTITY}'`, - ` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${AUDIENCE}'` - ] : []), - `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${image.split('@')[1]}'`, - `docker pull '${image}'`, - 'docker run --detach \\', - ' --name orca-relay \\', - ` '${image}'` - ].join('\n') -} - -// The exact shape the apply step's plan has: template replaced, MIG rebound to it. -function rollPlan({ - cellId, cap, protocol, pool, - beforeCapacityIdentity = CAPACITY_IDENTITY, - afterCapacityIdentity = CAPACITY_IDENTITY -}) { - return { - configuration: { - root_module: { - resources: [{ - address: 'google_compute_instance_group_manager.relay_gce_cell', - expressions: { - version: [{ - instance_template: { - references: [ - 'google_compute_instance_template.relay_gce_cell', - 'each.key' - ] - }, - name: { constant_value: 'primary' } - }] - } - }] - } - }, - resource_changes: [ - { - address: `google_compute_instance_template.relay_gce_cell[${JSON.stringify(cellId)}]`, - change: { - actions: ['create', 'delete'], - before: { - metadata_startup_script: startupScript({ - cap, - image: ROLLBACK_IMAGE, - trusted: protocol >= 1, - // The live template predates the reviewed pool raise, as every asia cell's does. - pool: pool === undefined ? undefined : '10', - capacityIdentity: beforeCapacityIdentity - }) - }, - after: { - metadata_startup_script: startupScript({ - cap, - image: TARGET_IMAGE, - trusted: protocol >= 1, - pool, - capacityIdentity: afterCapacityIdentity - }), - self_link: null - }, - after_unknown: { self_link: true } - } - }, - { - address: `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(cellId)}]`, - change: { - actions: ['update'], - before: { target_size: 1, version: [{ instance_template: 'old' }] }, - after: { target_size: 1, version: [{ instance_template: null }] }, - after_unknown: { version: [{ instance_template: true }] } - } - } - ] - } -} - -function hostname(cellId) { - return cellId.slice('production-gce-'.length) -} - -// The job resolves cap, region, and pool from the cell id before any admin call; run that block -// alone. An empty pool is the root default, which the startup template emits no line for. -function resolveCellShape(cellId) { - const start = workflow.indexOf(' TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"') - assert.notEqual(start, -1, 'the same-cap cell shape block is missing') - const end = workflow.indexOf('\n esac\n', start) - assert.notEqual(end, -1, 'the same-cap cell shape block has no esac') - const script = workflow.slice(start, end + '\n esac'.length).replace(/^ {10}/gm, '') - return spawnSync('bash', [ - '-euo', - 'pipefail', - '-c', - `${script}\necho "\${EXPECTED_REGION} \${EXPECTED_HARD_CAP} pool=\${EXPECTED_DATABASE_POOL_MAX}"` - ], { env: { ...process.env, TARGET_CELL_ID: cellId }, encoding: 'utf8' }) -} - -function cellShape(cellId) { - const resolved = resolveCellShape(cellId) - assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`) - const [, cap, pool] = resolved.stdout.trim().split(' ') - return { cap: Number(cap), pool: pool.slice('pool='.length) || undefined } -} - -// The class block runs before checkout-independent work and decides the whole wave shape. -function resolveCellClass(cellId, drainPaceWindowMs = '300000') { - return spawnSync('bash', [ - '-euo', - 'pipefail', - '-c', - `${jobBlock( - ' CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \\', - ' SELECTOR_WAVE_DELTA="$(jq -er \'.selectorWaveDelta\' <<< "${CELL_CLASS}")"' - )}\necho "\${ENTRY_ADMISSION} \${SELECTOR_WAVE_DELTA}"` - ], { - cwd: new URL('../..', import.meta.url), - env: { ...process.env, TARGET_CELL_ID: cellId, DRAIN_PACE_WINDOW_MS: drainPaceWindowMs }, - encoding: 'utf8' - }) -} - -function drainingBlock() { - return `${jobBlock( - ' # Rollback is the documented recovery from a failed canary, which', - ' PREDECESSOR_DRAINING_OK=false\n fi' - )}\necho "\${PRECHECK_ADMISSION} \${PRECHECK_DRAINING} \${PREDECESSOR_DRAINING_OK}"` -} - -// The stage decides the predecessor, the plan's reviewed rollback image, and whether the -// MIG is rolled explicitly, so run the real block rather than restating its rule. -function stageBlock() { - return `${jobBlock( - ' # Two different failures leave the cell on the rollback image, and the image', - ' PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"\n fi' - )}\necho "\${ROLLBACK_STAGE} \${ROLLBACK_RESUME} \${PREDECESSOR_IMAGE_DIGEST}` + - ` \${PREDECESSOR_REHOME_PROTOCOL} \${PLAN_ROLLBACK_IMAGE}"` -} - -function generationBlock() { - return `${jobBlock( - ' if test "${DEPLOY_MODE}" = verify; then', - ' fi' - )}\necho "\${EFFECTIVE_SELECTOR_GENERATION}"` -} - -// The job derives both memberships in one block; run that block alone for each class. -function membership(env) { - const script = `${jobBlock( - ' RESTORED_MIGRATION_CELLS="$(jq -rn \\', - ' fi' - )}\njq -cn --arg a "\${ISOLATED_MIGRATION_CELLS}" --arg b "\${ISOLATED_GENERAL_CELLS}" \\ - --arg c "\${RESTORED_MIGRATION_CELLS}" --arg d "\${RESTORED_GENERAL_CELLS}" \\ - '{isolatedMigration:$a,isolatedGeneral:$b,restoredMigration:$c,restoredGeneral:$d}'` - const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', script], { - env: { ...process.env, ...env }, - encoding: 'utf8' - }) - assert.equal(resolved.status, 0, resolved.stderr) - return JSON.parse(resolved.stdout) -} - -function jobBlock(firstLine, lastLine) { - const start = workflow.indexOf(`${firstLine}\n`) - assert.notEqual(start, -1, `the job has no ${firstLine.trim()}`) - const end = workflow.indexOf(`\n${lastLine}\n`, start) - assert.notEqual(end, -1, `that block has no ${lastLine.trim()}`) - return workflow.slice(start, end + lastLine.length + 1).replace(/^ {10}/gm, '') -} - -describe('same-cap roll scripts accept every same-cap cell', () => { - it('parses every wave cell through the same-cap canary allowlist', () => { - for (const cellId of SAME_CAP_CELLS) { - for (const mode of ['isolate', 'drain', 'activate']) { - assert.deepEqual(parseProductionCapacityCellArguments([ - '--director-origin', 'https://relay.onorca.dev', - '--cell-origin', `https://${hostname(cellId)}.relay.onorca.dev`, - '--cell-id', cellId, - '--approved-cells', 'same-cap', - '--mode', mode - ]), { - directorOrigin: 'https://relay.onorca.dev', - cellOrigin: `https://${hostname(cellId)}.relay.onorca.dev`, - cellId, - mode, - paceWindowMs: 0 - }) - } - } - }) - - it('resolves a cap, region, and pool for every wave cell and refuses anything else', () => { - for (const cellId of SAME_CAP_CELLS) { - const resolved = resolveCellShape(cellId) - assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`) - assert.match( - resolved.stdout.trim(), - /^(us-central1 1000 pool=|us-central1 600 pool=|asia-east2 3000 pool=16|us-central1 3000 pool=)$/, - cellId - ) - assert.equal(tfvarsDatabasePoolMax(cellId), cellShape(cellId).pool ?? '10', cellId) - assert.equal(String(cellShape(cellId).cap), tfvarsHardCap(cellId), cellId) - } - assert.equal(resolveCellShape('production-gce-c12').status, 1) - assert.equal(resolveCellShape('production-gce-c36').status, 1) - }) - - - - - - - it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => { - const trusted = SAME_CAP_CELLS.filter((cell) => REHOME_SOURCE_CELLS.has(cell)) - // Only a declared rehome source may roll at a trusted protocol at all; the job refuses - // the rest before it plans, and the next test covers them at protocol 0. - assert.deepEqual( - SAME_CAP_CELLS.filter((cell) => !REHOME_SOURCE_CELLS.has(cell)), - SAME_CAP_MIGRATION_ONLY_CELLS - ) - for (const [cellId, protocol] of trusted.flatMap((cell) => [[cell, 1], [cell, 3]])) { - const { cap, pool } = cellShape(cellId) - const config = { - mode: 'same-cap-cell', - cellId, - hardCap: cap, - unobservedBound: 60, - image: TARGET_IMAGE, - rollbackImage: ROLLBACK_IMAGE, - capacityServiceAccount: CAPACITY_IDENTITY, - rehomeDirectorServiceAccount: DIRECTOR_IDENTITY, - rehomeAudience: AUDIENCE, - regionalRehomeProtocol: String(protocol), - databasePoolMax: pool - } - const plan = rollPlan({ cellId, cap, protocol, pool }) - assert.deepEqual( - validateCapacityPlan(plan, config), - { mode: 'same-cap-cell', changes: 2 }, - cellId - ) - // The other protocol must reject the same plan, or the flag decides nothing. - assert.throws( - () => validateCapacityPlan(plan, { - ...config, - regionalRehomeProtocol: '0' - }), - /reviewed image and capacity/, - cellId - ) - // Dropping the pin must reject a pinned cell, and adding one must reject a default cell. - assert.throws( - () => validateCapacityPlan(plan, { - ...config, - databasePoolMax: pool === undefined ? '16' : undefined - }), - /reviewed image and capacity/, - cellId - ) - } - }) - - it('validates a protocol-0 plan for a cell outside the rehome source list', () => { - const cellId = 'production-gce-c17' - assert.equal(REHOME_SOURCE_CELLS.has(cellId), false) - const config = { - mode: 'same-cap-cell', - cellId, - hardCap: 600, - unobservedBound: 60, - image: TARGET_IMAGE, - rollbackImage: ROLLBACK_IMAGE, - capacityServiceAccount: CAPACITY_IDENTITY, - rehomeDirectorServiceAccount: DIRECTOR_IDENTITY, - rehomeAudience: AUDIENCE, - regionalRehomeProtocol: '0' - } - const plan = rollPlan({ cellId, cap: 600, protocol: 0 }) - assert.deepEqual(validateCapacityPlan(plan, config), { mode: 'same-cap-cell', changes: 2 }) - // Protocol 1 must reject a plan with no rehome lines, or the absent-line rule decides nothing. - assert.throws( - () => validateCapacityPlan(plan, { ...config, regionalRehomeProtocol: '1' }), - /reviewed image and capacity/ - ) - }) - - it('resolves the class and selector delta the wave validator declares', () => { - for (const cellId of SAME_CAP_CELLS) { - const resolved = resolveCellClass(cellId) - assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`) - assert.equal( - resolved.stdout.trim(), - `${entryAdmission(cellId)} ${selectorWaveDelta(cellId)}`, - cellId - ) - } - assert.equal(resolveCellClass('production-gce-c12').status, 1) - // The job's own block refuses a pace its cell may not run, before anything reads production. - assert.equal(resolveCellClass('production-gce-c7', '60000').status, 0) - assert.equal(resolveCellClass('production-gce-c28', '60000').status, 1) - assert.equal(resolveCellClass('production-gce-c7', '45000').status, 1) - }) - - it('offsets a later wave by this cell class\'s own selector delta', () => { - for (const [waveIndex, delta] of [['0', 2], ['3', 2], ['0', 0], ['3', 0]]) { - const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', generationBlock()], { - env: { - ...process.env, - DEPLOY_MODE: 'apply', - EXPECTED_SELECTOR_GENERATION: '40', - WAVE_INDEX: waveIndex, - SELECTOR_WAVE_DELTA: String(delta) - }, - encoding: 'utf8' - }) - assert.equal(resolved.status, 0, resolved.stderr) - assert.equal(resolved.stdout.trim(), String(40 + delta * Number(waveIndex))) - } - }) - - it('hands a migration-only cell back the exact membership it entered with', () => { - const entry = { - EXPECTED_MIGRATION_ONLY_CELLS: 'production-gce-c17,production-gce-c18', - EXPECTED_GENERAL_CELLS: 'production-gce-c7,production-gce-c8' - } - const isolated = membership({ - ...entry, - TARGET_CELL_ID: 'production-gce-c17', - ENTRY_ADMISSION: 'migration-only' - }) - assert.deepEqual(isolated, { - isolatedMigration: 'production-gce-c17,production-gce-c18', - isolatedGeneral: 'production-gce-c7,production-gce-c8', - restoredMigration: 'production-gce-c17,production-gce-c18', - restoredGeneral: 'production-gce-c7,production-gce-c8' - }) - // A general cell still leaves migration-only and returns to general. - assert.deepEqual( - membership({ - ...entry, - TARGET_CELL_ID: 'production-gce-c7', - ENTRY_ADMISSION: 'general' - }), - { - isolatedMigration: 'production-gce-c17,production-gce-c18,production-gce-c7', - isolatedGeneral: 'production-gce-c8', - restoredMigration: 'production-gce-c17,production-gce-c18', - restoredGeneral: 'production-gce-c7,production-gce-c8' - } - ) - }) - - it('never activates a migration-only cell and proves its isolate changed nothing', () => { - const restore = workflow - .split('name: Restore only the verified selected cell to its entry admission')[1] - .split('\n - id:')[0] - assert.match(restore, /if test "\$\{ENTRY_ADMISSION\}" = migration-only; then\n\s+RESTORE_MODE=isolate/) - assert.match(restore, /--admission "\$\{ENTRY_ADMISSION\}"/) - // The pre-mutation check must demand the class the cell is declared to serve in. - assert.match(workflow, /PRECHECK_ADMISSION="\$\{ENTRY_ADMISSION\}"/) - const isolate = workflow - .split('name: Reversibly isolate and drain only the selected cell')[1] - .split('\n - id:')[0] - assert.match(isolate, /migration-only; then\n\s+jq -e '\.changed == false'/) - }) - - it('requires rehome source membership exactly when a roll carries trust lines', () => { - const step = workflow - .split('name: Resolve immutable same-cap cell configuration')[1] - .split('\n - name:')[0] - const guard = step.indexOf('jq -e --arg cell "${TARGET_CELL_ID}" \'index($cell) != null\'') - assert.notEqual(guard, -1) - // The guard reads both protocols, so it has to sit after they are resolved. - assert.ok(step.indexOf('DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"') < guard) - assert.match( - step.slice(0, guard), - /test "\$\{DESIRED_REHOME_PROTOCOL\}" != 0 \|\| test "\$\{CURRENT_REHOME_PROTOCOL\}" != 0\n\s+\}; then\s+$/ - ) - }) - - it('rolls a template stale enough to predate the pinned capacity identity', () => { - // Exactly c17's shape on 2026-09-18: its live template is from 2026-08-07 and has no - // capacity identity line, so the roll adds one. Run 35290908836 failed closed here. - const cellId = 'production-gce-c17' - const config = { - mode: 'same-cap-cell', - cellId, - hardCap: 600, - unobservedBound: 60, - image: TARGET_IMAGE, - rollbackImage: ROLLBACK_IMAGE, - capacityServiceAccount: CAPACITY_IDENTITY, - rehomeDirectorServiceAccount: DIRECTOR_IDENTITY, - rehomeAudience: AUDIENCE, - regionalRehomeProtocol: '0' - } - const stale = rollPlan({ cellId, cap: 600, protocol: 0, beforeCapacityIdentity: null }) - assert.deepEqual(validateCapacityPlan(stale, config), { mode: 'same-cap-cell', changes: 2 }) - // The line may only be gained. A roll may not rewrite it, - assert.throws( - () => validateCapacityPlan(stale, { - ...config, - capacityServiceAccount: 'orca-cloud-gha-other@onorca-cloud.iam.gserviceaccount.com' - }), - /reviewed image and capacity/ - ) - // nor drop it from a template that already carries one. - assert.throws( - () => validateCapacityPlan( - rollPlan({ cellId, cap: 600, protocol: 0, afterCapacityIdentity: null }), - config - ), - /reviewed image and capacity/ - ) - // A same-cap roll cannot run without the identity pinned at all. - assert.throws( - () => validateCapacityPlan(stale, { ...config, capacityServiceAccount: undefined }), - /invalid service account/ - ) - }) - - - it('decides the predecessor draining rule from the real block, for both classes', () => { - // A zero-host cell sheds nothing, and a failed canary's own drain leaves the flag set - // with no restart behind it; run 35292335415 stopped on exactly that residue. - const cases = [ - // mode, entry class, resume, expected [precheck admission, precheck draining, jq ok] - ['apply', 'migration-only', 'false', ['migration-only', 'either', 'true']], - ['apply', 'general', 'false', ['general', 'forbidden', 'false']], - ['verify', 'migration-only', 'false', ['migration-only', 'either', 'true']], - ['verify', 'general', 'false', ['general', 'forbidden', 'false']], - // Every rollback path keeps exactly the behaviour it had. - ['rollback', 'general', 'false', ['general-or-migration-only', 'either', 'true']], - ['rollback', 'general', 'true', ['general-or-migration-only', 'either', 'false']], - ['rollback', 'migration-only', 'false', ['general-or-migration-only', 'either', 'true']], - ['rollback', 'migration-only', 'true', ['general-or-migration-only', 'either', 'false']] - ] - for (const [mode, entry, resume, expected] of cases) { - const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', drainingBlock()], { - env: { - ...process.env, - DEPLOY_MODE: mode, - ENTRY_ADMISSION: entry, - ROLLBACK_RESUME: resume - }, - encoding: 'utf8' - }) - assert.equal(resolved.status, 0, `${mode}/${entry}/${resume}: ${resolved.stderr}`) - assert.deepEqual( - resolved.stdout.trim().split(' '), - expected, - `${mode}/${entry}/${resume}` - ) - } - }) - - it('reads one draining decision in both predecessor checks', () => { - const step = workflow - .split('name: Verify exact current generation, digest, cap, and rollback point')[1] - .split('\n - name:')[0] - // The jq assertion and its diagnostic must not be able to disagree. - assert.equal(step.split('--argjson drainingOk "${PREDECESSOR_DRAINING_OK}"').length, 3) - assert.doesNotMatch(step, /drainingOk "\$\(test/) - // The fresh VM is still required not to be draining, on every path. - const after = workflow - .split('name: Verify new incarnation, exact image, protocol, and durable safety')[1] - .split('\n - name:')[0] - assert.match(after, /--admission migration-only --draining forbidden/) - const restore = workflow - .split('name: Restore only the verified selected cell to its entry admission')[1] - .split('\n - id:')[0] - assert.match(restore, /--draining forbidden --activity allowed/) - }) - - it('classifies every rollback stage from the real block', () => { - const repository = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay' - const target = `sha256:${'7'.repeat(64)}` - const rollback = `sha256:${'0'.repeat(64)}` - const stage = (mode, live, draining) => { - // Exactly how the job assigns them: rollback swaps desired and current. - const desired = mode === 'rollback' ? rollback : target - const current = mode === 'rollback' ? target : rollback - const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', stageBlock()], { - env: { - ...process.env, - DEPLOY_MODE: mode, - CURRENT_RUNTIME: JSON.stringify({ imageDigest: live, draining }), - DESIRED_IMAGE_DIGEST: desired, - CURRENT_IMAGE_DIGEST: current, - DESIRED_IMAGE: `${repository}@${desired}`, - IMAGE_REPOSITORY: repository, - DESIRED_REHOME_PROTOCOL: '1', - CURRENT_REHOME_PROTOCOL: '0' - }, - encoding: 'utf8' - }) - assert.equal(resolved.status, 0, `${mode}/${live}/${draining}: ${resolved.stderr}`) - return resolved.stdout.trim().split(' ') - } - const roll = (current, protocol) => - ['roll', 'false', current, protocol, `${repository}@${current}`] - // Only the last row differs from main: it used to read `resume` and wedge, because the - // resume path refuses a draining cell and never restarts one. - assert.deepEqual(stage('apply', rollback, false), roll(rollback, '0')) - assert.deepEqual(stage('apply', rollback, true), roll(rollback, '0')) - assert.deepEqual(stage('apply', target, false), roll(rollback, '0')) - assert.deepEqual(stage('verify', rollback, false), roll(rollback, '0')) - assert.deepEqual(stage('rollback', target, false), roll(target, '0')) - assert.deepEqual(stage('rollback', target, true), roll(target, '0')) - assert.deepEqual( - stage('rollback', rollback, false), - ['resume', 'true', rollback, '1', `${repository}@${target}`] - ) - assert.deepEqual( - stage('rollback', rollback, true), - ['stranded', 'false', rollback, '1', `${repository}@${rollback}`] - ) - // A runtime that reports no drain flag at all must never read as stranded. - const [missing] = stage('rollback', rollback, null) - assert.equal(missing, 'resume') - }) - - it('recreates the one stranded instance when a stranded plan changes nothing', () => { - const apply = workflow - .split('name: Apply only the selected same-cap template and MIG')[1] - .split('\n - id:')[0] - // The plan is reviewed against the image the cell serves, not an assumed predecessor. - assert.match(apply, /--rollback-image "\$\{PLAN_ROLLBACK_IMAGE\}"/) - assert.doesNotMatch(apply, /--rollback-image "\$\{IMAGE_REPOSITORY\}/) - // A rolling action rewrites the MIG's version name outside Terraform, and the validator then - // refuses every later plan for the cell; recreating the instance leaves the MIG untouched. - assert.doesNotMatch(apply, /rolling-action/) - assert.match( - apply, - /list-instances \\\n\s+"\$\{MIG_NAME\}"[\s\S]*?if length == 1 then \.\[0\]\.instance/ - ) - assert.match( - apply, - /recreate-instances "\$\{MIG_NAME\}" \\\n\s+--instances "\$\{STRANDED_INSTANCE\}"/ - ) - assert.equal(apply.split('recreate-instances').length, 2) - // The recreate has to be waited on, after the apply's own wait. - const recreate = apply.indexOf('recreate-instances') - assert.equal(apply.split('wait-until "${MIG_NAME}" --stable').length, 3) - assert.ok(apply.indexOf('wait-until "${MIG_NAME}" --stable', recreate) > recreate) - }) - - // The stranded branch's instance pick has to refuse anything but exactly one instance. - it('picks the stranded instance only from a one-instance MIG', () => { - const apply = workflow - .split('name: Apply only the selected same-cap template and MIG')[1] - .split('\n - id:')[0] - const filter = /jq -er '(if length == 1[\s\S]*?end)'\)"/.exec(apply)?.[1] - assert.notEqual(filter, undefined, 'the stranded branch no longer asserts one instance') - const pick = (instances) => - spawnSync('jq', ['-er', filter], { input: JSON.stringify(instances), encoding: 'utf8' }) - const link = (name) => - `https://www.googleapis.com/compute/v1/projects/p/zones/z/instances/${name}` - const one = pick([{ instance: link('relay-c29-abcd') }]) - assert.equal(one.error, undefined) - assert.equal(one.status, 0, one.stderr) - assert.equal(one.stdout.trim(), 'relay-c29-abcd') - assert.notEqual(pick([]).status, 0) - assert.notEqual( - pick([{ instance: link('relay-c29-abcd') }, { instance: link('relay-c29-efgh') }]).status, - 0 - ) - }) - - // Run the predicate the job ships rather than restating it, because restating it is how the - // two drift apart. An unconverged resume accepts the template-and-MIG pair and nothing else, - // and it applies nothing: a backend change cannot reach this plan, which no longer targets one. - it('accepts only the template-and-MIG pair on an unconverged resume', () => { - const step = workflow.slice( - workflow.indexOf('- name: Require converged Terraform state and a stable MIG on resume'), - workflow.indexOf('- name: Apply only the selected same-cap template and MIG') - ) - const accept = /jq -e '(\.changes == 2)' <<< "\$\{RESUME_REVIEW\}"/.exec(step) - assert.notEqual(accept, null, 'the resume step no longer gates on a validator verdict') - assert.equal(step.includes('terraform -chdir=infra/terraform apply'), false) - const outcome = (review) => { - const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', [ - `RESUME_REVIEW=${JSON.stringify(JSON.stringify(review))}`, - `jq -e '${accept[1]}' <<< "\${RESUME_REVIEW}" >/dev/null || { echo refuse; exit 0; }`, - 'echo accept' - ].join('\n')], { encoding: 'utf8' }) - assert.equal(resolved.status, 0, resolved.stderr) - return resolved.stdout.trim() - } - assert.equal(outcome({ changes: 2 }), 'accept') - // Anything the validator did not bound to the reviewed rollback-image drift fails the step. - assert.equal(outcome({ changes: 0 }), 'refuse') - assert.equal(outcome({ changes: 0, backendUpdate: ['log_config.0'] }), 'refuse') - assert.equal(outcome({ changes: 1 }), 'refuse') - assert.equal(outcome({ changes: 3 }), 'refuse') - }) - - // The stranded cell's explicit MIG roll is the only thing that clears its drain flag, and - // `changes` is what decides it, so run the shipped predicate rather than restating it. - it('rolls a stranded MIG on the shipped predicate', () => { - const apply = workflow - .split('name: Apply only the selected same-cap template and MIG')[1] - .split('\n - id:')[0] - const condition = - /if test "\$\{ROLLBACK_STAGE\}" = stranded \\\n\s+(&& jq -e '\.changes < 2' <<< "\$\{PLAN_REVIEW\}" >\/dev\/null); then/ - .exec(apply) - assert.notEqual(condition, null, 'the stranded roll no longer gates on the plan review') - const rolls = (stage, review) => { - const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', [ - `ROLLBACK_STAGE=${stage}`, - `PLAN_REVIEW=${JSON.stringify(JSON.stringify(review))}`, - `if test "\${ROLLBACK_STAGE}" = stranded \\\n ${condition[1]}; then`, - 'echo replace; else echo no-replace; fi' - ].join('\n')], { encoding: 'utf8' }) - assert.equal(resolved.status, 0, resolved.stderr) - return resolved.stdout.trim() - } - assert.equal(rolls('stranded', { changes: 0 }), 'replace') - // A MIG-only reconciliation (a version label revert) leaves the template, so no restart. - assert.equal(rolls('stranded', { changes: 1 }), 'replace') - // A real template replacement already restarts the instance; rolling again would be a second. - assert.equal(rolls('stranded', { changes: 2 }), 'no-replace') - assert.equal(rolls('resume', { changes: 0 }), 'no-replace') - assert.equal(rolls('none', { changes: 0 }), 'no-replace') - }) -}) - diff --git a/cloud/dev/scripts/relay-same-cap-shadow-gate-verdict.mjs b/cloud/dev/scripts/relay-same-cap-shadow-gate-verdict.mjs deleted file mode 100644 index ab4cc119493..00000000000 --- a/cloud/dev/scripts/relay-same-cap-shadow-gate-verdict.mjs +++ /dev/null @@ -1,470 +0,0 @@ -// Windowing, thresholds, and the verdict for the same-cap post-wave shadow health gate. Pure: it -// takes already-read log samples and returns a judgement, so every rule here is unit-testable -// without touching production. The reader lives in relay-same-cap-shadow-gate.mjs. - -// Status vocabulary, worst-first. 'unverified' is a read that did not complete or that hit the -// entry limit; it can never settle to 'pass', because a truncated count is not evidence of calm. -// A partial count already past a block line is a block, though: more data only adds to it. -export const CHECK_STATUSES = ['would-block', 'unverified', 'warn', 'pass'] - -export const VERDICTS = { PASS: 'PASS', WARN: 'WARN', WOULD_BLOCK: 'WOULD_BLOCK' } - -// Cloud Logging silently returns only `--limit` entries, so every read is split into sub-windows -// this long and a sub-window that comes back exactly at the limit is reported as truncated. -export const SUB_WINDOW_MINUTES = 10 - -export const ENTRY_LIMIT = 20000 - -// The 503 background is the same day's minutes just before the drain, not the same minutes a day -// or two earlier: a busy morning doubled 10-02's rate against both, and a baseline that held an -// incident blinded the comparison for four cells of that wave. -export const BACKGROUND_MINUTES = 10 - -// The asia-east2 cells share a 16-connection pool at 176 ms RTT, which is where pool pressure -// shows up first for the whole fleet. A cell joins only once it serves: zero samples read as -// unverified, so listing a not-yet-general cell would turn every verdict into WARN. -export const FLEET_POOL_CELL_IDS = [ - 'production-gce-c27', - 'production-gce-c28', - 'production-gce-c29', - 'production-gce-c30', - 'production-gce-c31', - 'production-gce-c34', - 'production-gce-c35' -] - -export const SHADOW_GATE_THRESHOLDS = { - // One sample at 71 waiters is a burst that drains; three in a row is a pool that does not. - pool: { waitersMax: 50, waitersConsecutiveSamples: 3, sqlFailuresDelta: 200 }, - // Pace-ladder rung budget (cloud/docs/relay-workflows.md), on non-drain 503s against the - // pre-drain median. One ordinary minute over is a transient; two in a row past max(1.5x, +20) - // warn and past max(2x, +40) is the abort. One minute past max(10x, 200) is a block on its own, - // the shape of a short sharp drain herd. Replayed (30 windows): no roll without a brownout - // passed 112 in a minute, and every brownout or herd peaked at 5,999 or more. - nonDrain503Budget: { - warnMultiple: 1.5, - warnMarginPerMinute: 20, - blockMultiple: 2, - blockMarginPerMinute: 40, - consecutiveMinutes: 2, - spikeMultiple: 10, - spikeFloor: 200 - }, - // A drain-return deferral tells the host when to come back; past a minute the drain is no - // longer paced by the window but queued behind the director's lane. - drainDeferral: { warnRetryAfterSecondsAbove: 30, blockRetryAfterSecondsAbove: 60 }, - cloudSqlFatal: { warnAbove: 0, blockAbove: 20 }, - // With no drain timestamp (a resumed rollback skips the drain) the window still has to start - // somewhere; this is how far back of the verify end it reaches instead. - fallbackWindowMinutes: 30, - // A read that stalls must not be allowed to spend the job's remaining minutes. - readTimeoutMs: 60_000, - // Reads are serialised, so a failure mode that makes every read cost its full retry budget - // (an expired credential, a Logging 429 storm) scales with the window, not with one read. - // Past this the gate stops reading and reports the rest unverified, which is a verdict; the - // step's own timeout-minutes sits above it and exists only for a hung process. Set well clear - // of a healthy gate's own serial read time, or ordinary days report unverified tails and the - // shadow roll stops measuring the thing it exists to measure. Raise both bounds together. - overallDeadlineMs: 420_000 -} - -const MINUTE_MS = 60_000 - -export function parseTimestamp(value, label) { - const parsed = typeof value === 'string' ? Date.parse(value) : Number.NaN - if (Number.isNaN(parsed)) throw new Error(`${label} is not an RFC 3339 timestamp: ${value}`) - return new Date(parsed) -} - -export function formatTimestamp(date) { - return `${date.toISOString().slice(0, 19)}Z` -} - -/** - * The window a cell's roll is judged over: its drain start to its verify end. A resumed rollback - * never drains, so the apply start, then a fixed lookback, stands in for it. - */ -export function resolveWindow({ - drainStartedAt, - applyStartedAt, - verifyEndedAt, - fallbackMinutes = SHADOW_GATE_THRESHOLDS.fallbackWindowMinutes -}) { - const endedAt = parseTimestamp(verifyEndedAt, 'verify end') - const start = drainStartedAt || applyStartedAt - const startedAt = start - ? parseTimestamp(start, 'window start') - : new Date(endedAt.getTime() - fallbackMinutes * MINUTE_MS) - if (startedAt >= endedAt) throw new Error('shadow gate window starts at or after it ends') - return { startedAt, endedAt, startedFrom: drainStartedAt ? 'drain' : start ? 'apply' : 'fallback' } -} - -export function splitWindow({ startedAt, endedAt }, minutes = SUB_WINDOW_MINUTES) { - const step = minutes * MINUTE_MS - const windows = [] - for (let cursor = startedAt.getTime(); cursor < endedAt.getTime(); cursor += step) { - windows.push({ - startedAt: new Date(cursor), - endedAt: new Date(Math.min(cursor + step, endedAt.getTime())) - }) - } - return windows -} - -/** - * Counts per clock minute across sub-window reads. A sub-window that returned exactly the entry - * limit is truncated, so its minutes are floors, not counts, and the whole read is unverified. - */ -export function countByMinute(reads, limit = ENTRY_LIMIT) { - const perMinute = new Map() - let truncated = false - for (const read of reads) { - if (read.failed || read.timestamps.length >= limit) truncated = true - for (const timestamp of read.timestamps) { - const minute = timestamp.slice(0, 16) - perMinute.set(minute, (perMinute.get(minute) ?? 0) + 1) - } - } - let peak = 0 - let peakMinute = null - let total = 0 - for (const [minute, count] of perMinute) { - total += count - if (count > peak) { - peak = count - peakMinute = minute - } - } - return { perMinute: Object.fromEntries(perMinute), total, peak, peakMinute, truncated } -} - -// Longest run of consecutive samples at or above the threshold. -export function longestRunAtOrAbove(values, threshold) { - let longest = 0 - let run = 0 - for (const value of values) { - run = value > threshold ? run + 1 : 0 - if (run > longest) longest = run - } - return longest -} - -// Director runtime metrics land every 30 s per instance and count the interval before the sample. -export const DIRECTOR_METRICS_INTERVAL_MS = 30_000 - -export function minuteKey(at) { - return new Date(at).toISOString().slice(0, 16) -} - -export function minutesOf({ startedAt, endedAt }) { - const minutes = [] - const first = Math.floor(startedAt.getTime() / MINUTE_MS) * MINUTE_MS - for (let cursor = first; cursor < endedAt.getTime(); cursor += MINUTE_MS) { - minutes.push(minuteKey(cursor)) - } - return minutes -} - -// A host re-dialling inside its own retry interval, or while its last dial is still in flight. The -// drain-return lane already answers these with the per-host interval rather than a place in line; -// on 10-02 they were ~3/4 of background 503s and doubled with placement volume, drain or not. -const OWN_RETRY_REASONS = ['host-rate-limited', 'host-in-flight'] - -function ownRetries(sample) { - return ['stickyRejectionsByReasonDelta', 'placementRejectionsByReasonDelta'].reduce( - (sum, field) => sum + OWN_RETRY_REASONS.reduce( - (count, reason) => count + (sample[field]?.[reason] ?? 0), - 0 - ), - 0 - ) -} - -// A drained host whose redial beats its own release meets its own row. That host was admitted to -// the drain-return lane first (the admission is counted before the assign that is refused), so -// row-busy refusals up to that minute's drain-return admissions are scheduled. Anything beyond is -// row contention the drain does not explain, and stays in the budget. The margin absorbs rounding -// from splitting each 30 s sample across clock minutes. -const ROW_BUSY_MARGIN_PER_MINUTE = 2 - -function rowBusy(sample) { - return sample.assign503sByCauseDelta?.relay_assignment_row_busy ?? 0 -} - -/** - * The director's scheduled 503s per clock minute, from its runtime-metrics samples: drain-return - * deferrals and answers to a host's own early retry, plus the re-placements. Each sample's count is - * split across the minutes its 30 s interval spans, in proportion, so a burst cannot land whole in - * a neighbouring minute and cancel that minute's real 503s. - */ -export function drainReturnByMinute(reads, limit) { - const deferrals = new Map() - const retries = new Map() - const busy = new Map() - const assignments = new Map() - const assign503s = new Map() - let retryAfterSecondsMax = 0 - let truncated = false - const charge = (map, endedAt, count) => { - let cursor = endedAt - DIRECTOR_METRICS_INTERVAL_MS - while (cursor < endedAt) { - const next = Math.min(endedAt, (Math.floor(cursor / MINUTE_MS) + 1) * MINUTE_MS) - const minute = minuteKey(cursor) - map.set(minute, (map.get(minute) ?? 0) + count * (next - cursor) / DIRECTOR_METRICS_INTERVAL_MS) - cursor = next - } - } - for (const read of reads) { - // The director always runs, so a read short of one instance's samples is a short answer (a - // Logging 429 can return one), not a quiet minute. - if (read.failed || read.samples.length >= limit || read.samples.length < (read.minSamples ?? 0)) { - truncated = true - } - for (const sample of read.samples) { - const endedAt = Date.parse(sample.timestamp) - charge(deferrals, endedAt, sample.drainReturnDeferralsDelta ?? 0) - charge(retries, endedAt, ownRetries(sample)) - charge(busy, endedAt, rowBusy(sample)) - charge(assignments, endedAt, sample.drainReturnAssignmentsDelta ?? 0) - charge(assign503s, endedAt, Object.values(sample.assign503sByCauseDelta ?? {}).reduce((sum, count) => sum + count, 0)) - retryAfterSecondsMax = Math.max( - retryAfterSecondsMax, - sample.drainReturnRetryAfterSecondsMax ?? 0 - ) - } - } - const sum = (map) => Math.round([...map.values()].reduce((total, count) => total + count, 0)) - let rowBusyBeyondDrain = 0 - for (const [minute, count] of busy) { - const scheduled = Math.min(count, (assignments.get(minute) ?? 0) + ROW_BUSY_MARGIN_PER_MINUTE) - rowBusyBeyondDrain += count - scheduled - retries.set(minute, (retries.get(minute) ?? 0) + scheduled) - } - return { - deferralsPerMinute: Object.fromEntries(deferrals), - ownRetriesPerMinute: Object.fromEntries(retries), - // Every /v1/assign 503 the director logged, scheduled or not: the same count by instance. - assign503sPerMinute: Object.fromEntries(assign503s), - deferralsTotal: sum(deferrals), - deferralsPeakPerMinute: Math.round(Math.max(0, ...deferrals.values())), - assignmentsTotal: sum(assignments), - assignmentsPeakPerMinute: Math.round(Math.max(0, ...assignments.values())), - rowBusyBeyondDrainTotal: Math.round(rowBusyBeyondDrain), - retryAfterSecondsMax, - truncated - } -} - -/** - * Director 503s per minute over the given minutes, with the scheduled ones taken out. A deferral or - * an early-retry answer tells one host when to come back, so a faster drain multiplies them without - * anything being wrong; what is left is lanes, capacity, and the database refusing work. - */ -export function withoutDrainDeferrals({ perMinute, failed }, drain, minutes) { - const series = minutes.map((minute) => Math.max( - 0, - Math.round( - (perMinute[minute] ?? 0) - - (drain.deferralsPerMinute[minute] ?? 0) - - (drain.ownRetriesPerMinute?.[minute] ?? 0) - ) - )) - const peak = Math.max(0, ...series) - return { - minutes, - series, - total: series.reduce((sum, count) => sum + count, 0), - peak, - peakMinute: peak > 0 ? minutes[series.indexOf(peak)] : null, - allPeak: Math.max(0, ...minutes.map((minute) => perMinute[minute] ?? 0)), - drainDeferralsTotal: Math.round( - minutes.reduce((sum, minute) => sum + (drain.deferralsPerMinute[minute] ?? 0), 0) - ), - ownRetriesTotal: Math.round( - minutes.reduce((sum, minute) => sum + (drain.ownRetriesPerMinute?.[minute] ?? 0), 0) - ), - unverified: Boolean(failed) || drain.truncated - } -} - -// The middle minute of the same-day minutes before the drain: a busy morning raises it with the -// window, and one incident minute inside it does not. -export function backgroundOf(nonDrain) { - const sorted = [...nonDrain.series].sort((left, right) => left - right) - const middle = Math.floor(sorted.length / 2) - const median = sorted.length === 0 - ? 0 - : sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2 - return { - startedAt: nonDrain.minutes[0] ?? null, - minutes: nonDrain.minutes.length, - medianPerMinute: median, - perMinute: nonDrain.series, - unverified: nonDrain.unverified || nonDrain.minutes.length === 0 - } -} - -export function judgeNonDrain503Budget({ observed, background }) { - const { - warnMultiple, - warnMarginPerMinute, - blockMultiple, - blockMarginPerMinute, - consecutiveMinutes, - spikeMultiple, - spikeFloor - } = SHADOW_GATE_THRESHOLDS.nonDrain503Budget - const perMinute = background.medianPerMinute - const warnAbove = Math.max(perMinute * warnMultiple, perMinute + warnMarginPerMinute) - const blockAbove = Math.max(perMinute * blockMultiple, perMinute + blockMarginPerMinute) - const detail = { - backgroundPerMinute: perMinute, - peakPerMinute: observed.peak, - peakMinute: observed.peakMinute, - // The raw count and what was taken out of it, so the subtraction can be checked by hand. - allPeakPerMinute: observed.allPeak, - drainDeferralsTotal: observed.drainDeferralsTotal, - ownRetriesTotal: observed.ownRetriesTotal, - warnAbove, - blockAbove, - spikeAbove: Math.max(perMinute * spikeMultiple, spikeFloor), - consecutiveMinutesOverWarn: longestRunAtOrAbove(observed.series, warnAbove), - consecutiveMinutesOverBlock: longestRunAtOrAbove(observed.series, blockAbove), - // The record a ladder rung keeps: non-drain 503s per minute from the drain start. - perMinute: observed.series - } - if (observed.unverified || background.unverified) return { status: 'unverified', ...detail } - if ( - detail.consecutiveMinutesOverBlock >= consecutiveMinutes || - observed.peak > detail.spikeAbove - ) return { status: 'would-block', ...detail } - if (detail.consecutiveMinutesOverWarn >= consecutiveMinutes) return { status: 'warn', ...detail } - return { status: 'pass', ...detail } -} - -export function judgeDrainDeferrals(drain) { - const { warnRetryAfterSecondsAbove, blockRetryAfterSecondsAbove } = - SHADOW_GATE_THRESHOLDS.drainDeferral - const detail = { - deferralsTotal: drain.deferralsTotal, - deferralsPeakPerMinute: drain.deferralsPeakPerMinute, - // The measured drain rate: re-placements the director's drain-return lane admitted. - replacementsTotal: drain.assignmentsTotal, - replacementsPeakPerMinute: drain.assignmentsPeakPerMinute, - retryAfterSecondsMax: drain.retryAfterSecondsMax, - warnAbove: warnRetryAfterSecondsAbove, - blockAbove: blockRetryAfterSecondsAbove - } - if (drain.truncated) return { status: 'unverified', ...detail } - if (drain.retryAfterSecondsMax > blockRetryAfterSecondsAbove) { - return { status: 'would-block', ...detail } - } - if (drain.retryAfterSecondsMax > warnRetryAfterSecondsAbove) { - return { status: 'warn', ...detail } - } - return { status: 'pass', ...detail } -} - - -/** - * The cell's own container: it has to have announced its listener since the apply began, and it - * must not have crashed anywhere in that span. Counting crashes only after the *last* listener - * would erase a crash-restart loop, whose later announcement looks like a clean boot; the MIG - * recreates the instance, so everything on this instance id since the apply belongs to this roll. - * - * A missing announcement only means a failure where a restart was expected. A resumed rollback - * deliberately restarts nothing, so there is no boot for this oracle to observe and its silence - * says nothing either way. - */ -export function judgeCellServing({ listeningAt, crashesSinceApply, read, expectBoot = true }) { - const detail = { - listeningAt: listeningAt ?? null, - crashesSinceApply: crashesSinceApply ?? 0, - expectBoot - } - if (read?.failed) return { status: 'unverified', ...detail } - if (!listeningAt) return { status: expectBoot ? 'would-block' : 'unverified', ...detail } - if (detail.crashesSinceApply > 0) return { status: 'would-block', ...detail } - return { status: 'pass', ...detail } -} - -/** - * Pool pressure. A single spike is a burst the pool absorbs; the block rule needs the pressure to - * persist across consecutive samples, which is what separates it from the one-sample false - * positives a literal rule produced this week. - */ -export function judgePool({ label, samples, failed = false, truncated = false }) { - const { waitersMax, waitersConsecutiveSamples, sqlFailuresDelta } = SHADOW_GATE_THRESHOLDS.pool - const waiters = samples.map((sample) => sample.databasePoolWaitersMax ?? 0) - const failures = samples.map((sample) => sample.sqlFailuresDelta ?? 0) - const detail = { - label, - samples: samples.length, - waitersMax: Math.max(0, ...waiters), - consecutiveSamplesOverWaitersThreshold: longestRunAtOrAbove(waiters, waitersMax), - sqlFailuresDeltaMax: Math.max(0, ...failures), - reconnectsDeltaMax: Math.max(0, ...samples.map((sample) => sample.reconnectsDelta ?? 0)), - totalConnectionsMax: Math.max(0, ...samples.map((sample) => sample.totalConnections ?? 0)), - databasePoolWaitingMax: Math.max(0, ...samples.map((sample) => sample.databasePoolWaiting ?? 0)), - waitersThreshold: waitersMax, - consecutiveSamplesThreshold: waitersConsecutiveSamples, - sqlFailuresDeltaThreshold: sqlFailuresDelta, - truncated - } - // A sample over the failure line is a fact however many others are missing. - if (detail.sqlFailuresDeltaMax > sqlFailuresDelta) return { status: 'would-block', ...detail } - // A truncated sample run has holes, and the consecutive-sample rule reads a hole as a recovery - // (or joins two runs across one), so it is judged only on a complete run. - if (failed || truncated || samples.length === 0) return { status: 'unverified', ...detail } - if (detail.consecutiveSamplesOverWaitersThreshold >= waitersConsecutiveSamples) { - return { status: 'would-block', ...detail } - } - if (detail.waitersMax > waitersMax) return { status: 'warn', ...detail } - return { status: 'pass', ...detail } -} - -export function judgeCloudSqlFatal({ count, truncated = false, failed = false }) { - const { warnAbove, blockAbove } = SHADOW_GATE_THRESHOLDS.cloudSqlFatal - const detail = { count, warnAbove, blockAbove } - // A truncated count is a lower bound: already over the line is a block, under it proves nothing. - if (count > blockAbove) return { status: 'would-block', ...detail } - if (failed || truncated) return { status: 'unverified', ...detail } - if (count > warnAbove) return { status: 'warn', ...detail } - return { status: 'pass', ...detail } -} - -// The checks a drain pace can move. The rest (Cloud SQL, the Asia pools, the new boot) read the -// fleet or the image, so a clean roll at any pace can still WARN on them. -export const PACE_CHECKS = ['nonDrain503Budget', 'drainDeferrals'] - -export function combineVerdict(checks) { - const statuses = Object.values(checks).map((check) => check.status) - if (statuses.includes('would-block')) return VERDICTS.WOULD_BLOCK - if (statuses.includes('unverified') || statuses.includes('warn')) return VERDICTS.WARN - return VERDICTS.PASS -} - -export function renderStepSummary(report) { - const rows = Object.entries(report.checks).map(([name, check]) => { - const numbers = Object.entries(check) - .filter(([key, value]) => key !== 'status' && value !== null && typeof value !== 'object') - .map(([key, value]) => `${key}=${value}`) - .join(', ') - return `| ${name} | ${check.status} | ${numbers} |` - }) - return [ - `## Shadow health gate (report only): ${report.verdict} (pace checks: ${report.paceVerdict})`, - '', - `Cell \`${report.cellId}\`, window ${report.window.startedAt} to ${report.window.endedAt}`, - `(start taken from: ${report.window.startedFrom}).`, - `Drain pace ${report.drain.paceWindowMs} ms (cell applied: ` + - `${report.drain.appliedPaceWindowMs ?? 'not drained'}), ` + - `${report.drain.targetHosts ?? 'unknown'} hosts, settled ${report.drain.settledAt ?? 'never'}.`, - 'This gate never fails the job. Compare its verdict with the operator call for this cell.', - '', - '| check | status | numbers |', - '| --- | --- | --- |', - ...rows, - '' - ].join('\n') -} diff --git a/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs b/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs deleted file mode 100644 index e9b09b5699e..00000000000 --- a/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs +++ /dev/null @@ -1,494 +0,0 @@ -#!/usr/bin/env node -// Post-wave shadow health gate for a same-cap cell roll. Reads exactly the oracles an operator -// reads by hand today, writes a PASS / WARN / WOULD_BLOCK verdict with its numbers to a JSON -// artifact and the step summary, and always exits 0 on a verdict: this runs in report-only mode so -// its calls can be compared with the operator's over a full roll before it is allowed to block. -// -// Every filter is built from validated, pattern-pinned inputs and handed to gcloud as argv, never -// through a shell. - -import { execFile } from 'node:child_process' -import { appendFile, writeFile } from 'node:fs/promises' -import { pathToFileURL } from 'node:url' -import { promisify } from 'node:util' -import { - BACKGROUND_MINUTES, - DIRECTOR_METRICS_INTERVAL_MS, - ENTRY_LIMIT, - PACE_CHECKS, - FLEET_POOL_CELL_IDS, - SHADOW_GATE_THRESHOLDS, - combineVerdict, - countByMinute, - drainReturnByMinute, - formatTimestamp, - backgroundOf, - judgeCellServing, - judgeCloudSqlFatal, - judgeDrainDeferrals, - judgeNonDrain503Budget, - judgePool, - minuteKey, - minutesOf, - parseTimestamp, - renderStepSummary, - resolveWindow, - splitWindow, - withoutDrainDeferrals -} from './relay-same-cap-shadow-gate-verdict.mjs' -import { SAME_CAP_DRAIN_PACE_WINDOWS_MS } from './relay-production-same-cap-wave.mjs' - -const execFileAsync = promisify(execFile) - -const CELL_ID = /^production-gce-c[1-9][0-9]*$/ -const CELL_HOST = /^c[1-9][0-9]*\.relay\.onorca\.dev$/ -const PROJECT_ID = /^[a-z][a-z0-9-]{4,28}[a-z0-9]$/ -const SERVICE_NAME = /^[a-z][a-z0-9-]{0,62}$/ -const COUNT = /^(0|[1-9][0-9]*)$/ - -export const READ_ATTEMPTS = 3 -const READ_RETRY_DELAY_MS = 5000 -const READ_TIMEOUT_MS = SHADOW_GATE_THRESHOLDS.readTimeoutMs -const OVERALL_DEADLINE_MS = SHADOW_GATE_THRESHOLDS.overallDeadlineMs -// json(timestamp) over a busy minute is a few hundred KB; leave room for the widest sub-window. -const READ_MAX_BUFFER_BYTES = 256 * 1024 * 1024 - -export function parseShadowGateArguments(argv) { - const values = new Map() - for (let index = 0; index < argv.length; index += 2) { - if (!argv[index].startsWith('--')) throw new Error(`expected a flag, got ${argv[index]}`) - values.set(argv[index].slice(2), argv[index + 1]) - } - const required = (name, pattern) => { - const value = values.get(name) ?? '' - if (!pattern.test(value)) throw new Error(`--${name} is not acceptable: ${value}`) - return value - } - const optionalCount = (name) => (values.get(name) ? Number(required(name, COUNT)) : null) - const config = { - cellId: required('cell-id', CELL_ID), - cellHost: required('cell-host', CELL_HOST), - projectId: required('project-id', PROJECT_ID), - directorService: required('director-service', SERVICE_NAME), - drainStartedAt: values.get('drain-started-at') || '', - // The listener lands while the MIG is still converging, so the boot search has to open at the - // apply's start; a bound taken at its completion is already past the announcement it looks for. - applyStartedAt: values.get('apply-started-at') || '', - applyCompletedAt: values.get('apply-completed-at') || '', - verifyEndedAt: values.get('verify-ended-at') || '', - outputFile: values.get('output-file') || '', - summaryFile: values.get('summary-file') || '', - drainPaceWindowMs: Number(required('drain-pace-window-ms', COUNT)), - // Empty on a resumed rollback, which never drains. - drainAppliedPaceWindowMs: optionalCount('drain-applied-pace-window-ms'), - drainSettledAt: values.get('drain-settled-at') || '', - targetHosts: optionalCount('target-hosts') - } - if (!SAME_CAP_DRAIN_PACE_WINDOWS_MS.includes(config.drainPaceWindowMs)) { - throw new Error(`--drain-pace-window-ms is not acceptable: ${config.drainPaceWindowMs}`) - } - if (config.drainSettledAt) parseTimestamp(config.drainSettledAt, '--drain-settled-at') - if (!config.cellHost.startsWith(`${config.cellId.replace('production-gce-', '')}.`)) { - throw new Error(`--cell-host ${config.cellHost} is not the host of ${config.cellId}`) - } - if (!config.outputFile) throw new Error('--output-file is required') - return config -} - -function timestampBounds({ startedAt, endedAt }) { - return `timestamp>="${formatTimestamp(startedAt)}" AND timestamp<"${formatTimestamp(endedAt)}"` -} - -/** - * Runs one bounded read with retries. A read that cannot complete is reported as failed rather than - * thrown: a missing oracle must surface as an unverified check, not as a crashed gate. - */ -async function retryingRead(reader, read) { - let lastError - for (let attempt = 1; attempt <= READ_ATTEMPTS; attempt += 1) { - // Every remaining read short-circuits once the budget is gone, so the gate always reaches a - // verdict instead of being killed part-way through with nothing written. - const remainingMs = reader.deadlineAt - reader.now() - if (remainingMs <= 0) return { failed: true, error: 'shadow gate read deadline exceeded' } - try { - return { ...(await read(Math.min(reader.readTimeoutMs, remainingMs))), failed: false } - } catch (error) { - lastError = error - if (attempt < READ_ATTEMPTS) { - await new Promise((resolve) => setTimeout(resolve, reader.retryDelayMs)) - } - } - } - return { failed: true, error: String(lastError?.message ?? lastError) } -} - -export async function readLogEntries(reader, { filter, projection, limit = ENTRY_LIMIT }) { - const args = [ - 'logging', 'read', filter, - '--project', reader.projectId, - '--format', projection, - '--limit', String(limit), - '--order', 'desc' - ] - const read = await retryingRead(reader, async (timeoutMs) => { - const { stdout } = await reader.runGcloud(args, { timeoutMs }) - return { entries: JSON.parse(stdout || '[]') } - }) - return { entries: [], ...read } -} - -async function readTimestampsOverWindow(reader, { filter, window }) { - const reads = [] - for (const subWindow of splitWindow(window)) { - const read = await readLogEntries(reader, { - filter: `${filter} AND ${timestampBounds(subWindow)}`, - projection: 'json(timestamp)' - }) - reads.push({ - failed: read.failed, - timestamps: read.entries.map((entry) => entry.timestamp) - }) - } - return countByMinute(reads) -} - -// Cloud Run's own request counter, aligned per minute server-side: a brownout's tens of thousands -// of 503s are counted, where a log read of them stops at its entry limit (10-01 c29). -async function readDirectorRequestCountSeries(reader, { config, span, filters = [], groupByFields }) { - const url = new URL(`https://monitoring.googleapis.com/v3/projects/${config.projectId}/timeSeries`) - url.searchParams.set('filter', [ - 'metric.type="run.googleapis.com/request_count"', - 'resource.type="cloud_run_revision"', - `resource.label."service_name"="${config.directorService}"`, - ...filters - ].join(' AND ')) - url.searchParams.set('interval.startTime', span.startedAt.toISOString()) - url.searchParams.set('interval.endTime', span.endedAt.toISOString()) - url.searchParams.set('aggregation.alignmentPeriod', '60s') - url.searchParams.set('aggregation.perSeriesAligner', 'ALIGN_DELTA') - url.searchParams.set('aggregation.crossSeriesReducer', 'REDUCE_SUM') - if (groupByFields) url.searchParams.set('aggregation.groupByFields', groupByFields) - url.searchParams.set('pageSize', '1000') - return await retryingRead(reader, async (timeoutMs) => { - const { stdout } = await reader.runGcloud(['auth', 'print-access-token'], { timeoutMs }) - const response = await reader.fetch(url, { - headers: { authorization: `Bearer ${stdout.trim()}` }, - signal: AbortSignal.timeout(timeoutMs) - }) - if (!response.ok) throw new Error(`Cloud Monitoring returned ${response.status}`) - const body = await response.json() - if (body.nextPageToken) throw new Error('Cloud Monitoring pagination is incomplete') - return { timeSeries: body.timeSeries ?? [] } - }) -} - -function addPointsPerMinute(perMinute, series) { - for (const point of series.points ?? []) { - // A delta point covers the minute that ends at its end time; a minute with no point had none. - const minute = minuteKey(Date.parse(point.interval.endTime) - 60_000) - perMinute[minute] = (perMinute[minute] ?? 0) + Number(point.value.int64Value ?? 0) - } - return perMinute -} - -export async function readDirector503PerMinute(reader, { config, span }) { - const read = await readDirectorRequestCountSeries(reader, { - config, - span, - filters: ['metric.label."response_code"="503"'] - }) - const perMinute = {} - for (const series of read.timeSeries ?? []) addPointsPerMinute(perMinute, series) - return { perMinute, failed: read.failed } -} - -// Every response code in one read, so a minute missing from the 503 series can be told apart from -// one not published yet: the other codes' series carry the same publishing front. -export async function readDirectorRequestsByCodePerMinute(reader, { config, span }) { - const read = await readDirectorRequestCountSeries(reader, { - config, - span, - groupByFields: 'metric.label."response_code"' - }) - const byCode = {} - for (const series of read.timeSeries ?? []) { - const code = series.metric?.labels?.response_code ?? 'unknown' - byCode[code] = addPointsPerMinute(byCode[code] ?? {}, series) - } - return { byCode, failed: read.failed } -} - -// Cells log through the COS container agent, so the text lives in jsonPayload.message; a -// textPayload filter matches nothing here and returns zero without saying so. -const CELL_LOG_SCOPE = 'resource.type="gce_instance" AND logName:"cos_containers"' - -export const DIRECTOR_DRAIN_FIELDS = [ - 'stickyRejectionsByReasonDelta', - 'placementRejectionsByReasonDelta', - 'drainReturnDeferralsDelta', - 'drainReturnAssignmentsDelta', - 'drainReturnRetryAfterSecondsMax', - 'assign503sByCauseDelta' -] - -// Five instances at one sample per 30 s is ~100 per 10-min sub-window; this many is truncation. -export const DIRECTOR_METRICS_LIMIT = 1000 - -export const directorMetricsSample = (entry) => ({ - timestamp: entry.timestamp, - instanceId: entry.labels?.instanceId, - ...entry.jsonPayload -}) - -export async function readDirectorDrainReturn(reader, { config, window }) { - const projection = `json(timestamp,labels.instanceId,${DIRECTOR_DRAIN_FIELDS - .map((field) => `jsonPayload.${field}`) - .join(',')})` - const reads = [] - for (const subWindow of splitWindow(window)) { - const read = await readLogEntries(reader, { - filter: `resource.type="cloud_run_revision"` - + ` AND resource.labels.service_name="${config.directorService}"` - + ` AND jsonPayload.event="orca_relay_runtime_metrics"` - + ` AND ${timestampBounds(subWindow)}`, - projection, - limit: DIRECTOR_METRICS_LIMIT - }) - reads.push({ - failed: read.failed, - samples: read.entries.map(directorMetricsSample), - minSamples: Math.floor( - (subWindow.endedAt.getTime() - subWindow.startedAt.getTime()) / DIRECTOR_METRICS_INTERVAL_MS - ) - }) - } - return reads -} - -// Non-drain 503s in the roll window against the same day's minutes just before the drain, with -// drain-return deferrals taken out of both. -async function readDirector503(reader, { config, window }) { - const minute = 60_000 - const span = { - startedAt: new Date( - Math.floor(window.startedAt.getTime() / minute) * minute - BACKGROUND_MINUTES * minute - ), - endedAt: new Date(Math.ceil(window.endedAt.getTime() / minute) * minute) - } - const counts = await readDirector503PerMinute(reader, { config, span }) - const reads = await readDirectorDrainReturn(reader, { config, window: span }) - const deferrals = drainReturnByMinute(reads, DIRECTOR_METRICS_LIMIT) - const windowMinutes = minutesOf(window) - const backgroundMinutes = minutesOf({ startedAt: span.startedAt, endedAt: window.startedAt }) - .filter((key) => !windowMinutes.includes(key)) - const observed = withoutDrainDeferrals(counts, deferrals, windowMinutes) - const background = backgroundOf(withoutDrainDeferrals(counts, deferrals, backgroundMinutes)) - // Only the roll window's own samples describe this drain; whether the reads were whole is a - // question about all of them. - const drain = { - ...drainReturnByMinute(reads.map((read) => ({ - failed: read.failed, - samples: read.samples.filter( - (sample) => Date.parse(sample.timestamp) > window.startedAt.getTime() - ) - })), DIRECTOR_METRICS_LIMIT), - truncated: deferrals.truncated - } - return { - background, - checks: { - nonDrain503Budget: judgeNonDrain503Budget({ observed, background }), - drainDeferrals: judgeDrainDeferrals(drain) - } - } -} - -/** - * The cell's new container. The listener announcement after the apply identifies both that the - * cell is serving and which instance it is serving on; crashes are then scoped to that instance, - * because instance_id is stable across a container restart and is the only cell label these - * entries carry. - */ -async function readCellServing(reader, { config, window, searchFrom, expectBoot }) { - const listening = await readLogEntries(reader, { - filter: `${CELL_LOG_SCOPE}` - + ` AND jsonPayload.message:"listening on https://${config.cellHost}"` - + ` AND ${timestampBounds({ startedAt: searchFrom, endedAt: window.endedAt })}`, - projection: 'json(timestamp,resource.labels.instance_id)', - limit: 50 - }) - // Newest first: the most recent announcement is the boot this wave produced. - const boot = listening.entries[0] - if (listening.failed || !boot) { - return { - serving: judgeCellServing({ listeningAt: null, read: listening, expectBoot }), - instanceId: null - } - } - const crashes = await readLogEntries(reader, { - filter: `${CELL_LOG_SCOPE}` - + ` AND jsonPayload.message:"throw er"` - + ` AND resource.labels.instance_id="${boot.resource.labels.instance_id}"` - + ` AND ${timestampBounds({ startedAt: searchFrom, endedAt: window.endedAt })}`, - projection: 'json(timestamp)', - limit: 100 - }) - return { - serving: judgeCellServing({ - listeningAt: boot.timestamp, - crashesSinceApply: crashes.entries.length, - read: crashes, - expectBoot - }), - instanceId: boot.resource.labels.instance_id - } -} - -const RUNTIME_METRIC_FIELDS = [ - 'totalConnections', - 'databasePoolWaitersMax', - 'databasePoolWaiting', - 'sqlFailuresDelta', - 'reconnectsDelta' -] - -async function readRuntimeMetrics(reader, { cellId, window }) { - const projection = `json(timestamp,${RUNTIME_METRIC_FIELDS - .map((field) => `jsonPayload.${field}`) - .join(',')})` - const samples = [] - let failed = false - let truncated = false - // Samples land every 30 s, so a 10-minute sub-window holds ~20. A read that comes back at this - // many is not a calm sub-window, it is a truncated one, and its gaps read as recoveries. - const limit = 500 - for (const subWindow of splitWindow(window)) { - const read = await readLogEntries(reader, { - filter: `${CELL_LOG_SCOPE}` - + ` AND jsonPayload.event="orca_relay_runtime_metrics"` - + ` AND jsonPayload.cellId="${cellId}"` - + ` AND ${timestampBounds(subWindow)}`, - projection, - limit - }) - if (read.failed) failed = true - if (read.entries.length >= limit) truncated = true - for (const entry of read.entries) { - samples.push({ timestamp: entry.timestamp, ...entry.jsonPayload }) - } - } - return { samples, failed, truncated } -} - -async function readCloudSqlFatal(reader, { window }) { - const counts = await readTimestampsOverWindow(reader, { - filter: `resource.type="cloudsql_database" AND "FATAL"`, - window - }) - return judgeCloudSqlFatal({ count: counts.total, truncated: counts.truncated }) -} - -export async function evaluateShadowGate(config, { - runGcloud, - retryDelayMs = READ_RETRY_DELAY_MS, - readTimeoutMs = READ_TIMEOUT_MS, - overallDeadlineMs = OVERALL_DEADLINE_MS, - now = Date.now, - fetch = globalThis.fetch -}) { - const reader = { - runGcloud, - retryDelayMs, - readTimeoutMs, - now, - deadlineAt: now() + overallDeadlineMs, - projectId: config.projectId, - fetch - } - const window = resolveWindow(config) - // Everything this roll's instance logged, from the moment the apply could first restart it. - const searchFrom = config.applyStartedAt - ? new Date(Date.parse(config.applyStartedAt)) - : window.startedAt - // Serialised on purpose: a burst of concurrent reads is what earns a Logging 429, and a 429 is - // the one failure that comes back as a short answer rather than an error. - const director = await readDirector503(reader, { config, window }) - // A fallback window start means neither the drain nor the apply ran, which is the resumed - // rollback that restarts nothing; there is then no boot to find. - const cell = await readCellServing(reader, { - config, - window, - searchFrom, - expectBoot: window.startedFrom !== 'fallback' - }) - const cloudSql = await readCloudSqlFatal(reader, { window }) - const cellMetrics = await readRuntimeMetrics(reader, { cellId: config.cellId, window }) - const checks = { - ...director.checks, - cellServing: cell.serving, - cellPool: judgePool({ label: config.cellId, ...cellMetrics }), - cloudSqlFatal: cloudSql - } - for (const fleetCellId of FLEET_POOL_CELL_IDS) { - if (fleetCellId === config.cellId) continue - const metrics = await readRuntimeMetrics(reader, { cellId: fleetCellId, window }) - checks[`fleetPool:${fleetCellId}`] = judgePool({ label: fleetCellId, ...metrics }) - } - return { - reportOnly: true, - cellId: config.cellId, - cellHost: config.cellHost, - cellInstanceId: cell.instanceId, - window: { - startedAt: formatTimestamp(window.startedAt), - endedAt: formatTimestamp(window.endedAt), - startedFrom: window.startedFrom, - // Recorded, not judged: an operator comparing verdicts needs to see how long the apply took - // next to when the cell actually came back. - applyCompletedAt: config.applyCompletedAt || null - }, - // The ladder rung this roll ran at, and what it measured against, for the step's record. - drain: { - paceWindowMs: config.drainPaceWindowMs, - appliedPaceWindowMs: config.drainAppliedPaceWindowMs, - targetHosts: config.targetHosts, - settledAt: config.drainSettledAt || null, - // Isolate to restart-safe: the later of the pace window's end and the cell emptying, plus - // one 5 s sample. - settledAfterSeconds: config.drainStartedAt && config.drainSettledAt - ? Math.round((Date.parse(config.drainSettledAt) - Date.parse(config.drainStartedAt)) / 1000) - : null - }, - // The pre-drain minutes the 503 checks were judged against. - background: director.background, - verdict: combineVerdict(checks), - // What a canary seals and a faster batch requires: only the checks the pace can move. - paceVerdict: combineVerdict(Object.fromEntries(PACE_CHECKS.map((name) => [name, checks[name]]))), - checks - } -} - -async function main() { - const config = parseShadowGateArguments(process.argv.slice(2)) - const report = await evaluateShadowGate(config, { - // `timeout` makes Node kill the child itself; continue-on-error bounds the job's outcome but - // not its clock, and a stalled read would otherwise spend the rollout's remaining minutes. - runGcloud: (args, { timeoutMs }) => execFileAsync('gcloud', args, { - maxBuffer: READ_MAX_BUFFER_BYTES, - timeout: timeoutMs, - killSignal: 'SIGKILL' - }) - }) - await writeFile(config.outputFile, `${JSON.stringify(report, null, 2)}\n`) - if (config.summaryFile) await appendFile(config.summaryFile, renderStepSummary(report)) - console.log(JSON.stringify(report, null, 2)) -} - -// Report only: a verdict, including WOULD_BLOCK, is a successful run. Only a crash exits non-zero, -// and the job still runs this step under continue-on-error. -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/relay-same-cap-shadow-gate.test.mjs b/cloud/dev/scripts/relay-same-cap-shadow-gate.test.mjs deleted file mode 100644 index 681b06c3187..00000000000 --- a/cloud/dev/scripts/relay-same-cap-shadow-gate.test.mjs +++ /dev/null @@ -1,842 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { readRelayWorkflow } from './relay-repository.mjs' -import { - READ_ATTEMPTS, - evaluateShadowGate, - parseShadowGateArguments -} from './relay-same-cap-shadow-gate.mjs' -import { - ENTRY_LIMIT, - FLEET_POOL_CELL_IDS, - PACE_CHECKS, - SHADOW_GATE_THRESHOLDS, - SUB_WINDOW_MINUTES, - backgroundOf, - combineVerdict, - countByMinute, - drainReturnByMinute, - formatTimestamp, - judgeCellServing, - judgeCloudSqlFatal, - judgeDrainDeferrals, - judgeNonDrain503Budget, - judgePool, - longestRunAtOrAbove, - renderStepSummary, - resolveWindow, - splitWindow, - withoutDrainDeferrals -} from './relay-same-cap-shadow-gate-verdict.mjs' - -const ARGV = [ - '--cell-id', 'production-gce-c28', - '--cell-host', 'c28.relay.onorca.dev', - '--project-id', 'onorca-cloud', - '--director-service', 'orca-cloud-relay', - '--drain-started-at', '2026-09-20T20:00:00Z', - '--apply-started-at', '2026-09-20T20:15:00Z', - '--apply-completed-at', '2026-09-20T20:19:30Z', - '--verify-ended-at', '2026-09-20T20:30:00Z', - '--output-file', '/tmp/shadow.json', - '--drain-pace-window-ms', '300000', - '--drain-applied-pace-window-ms', '300000', - '--drain-settled-at', '2026-09-20T20:11:00Z', - '--target-hosts', '857' -] - -function minuteOfTimestamps(minute, count) { - return Array.from( - { length: count }, - (_, index) => `${minute}:${String(index % 60).padStart(2, '0')}Z` - ) -} - -test('binds every gcloud input to a pinned pattern and to one cell', () => { - assert.equal(parseShadowGateArguments(ARGV).cellId, 'production-gce-c28') - // A filter is a string; anything that could steer one has to be refused before it is built. - assert.throws(() => parseShadowGateArguments(ARGV.with(1, 'production-gce-c28" OR "x'))) - assert.throws(() => parseShadowGateArguments(ARGV.with(3, 'evil.example.test'))) - assert.throws(() => parseShadowGateArguments(ARGV.with(5, 'Onorca Cloud'))) - assert.throws(() => parseShadowGateArguments(ARGV.with(7, 'orca cloud relay'))) - // Host and cell id must name the same cell, or the serving check reads a neighbour. - assert.throws(() => parseShadowGateArguments(ARGV.with(3, 'c29.relay.onorca.dev'))) - // A run with nowhere to write its verdict is not a report-only run, it is a silent one. - assert.throws( - () => parseShadowGateArguments(ARGV.filter((_, index) => index !== 16 && index !== 17)), - /--output-file is required/ - ) - // The pace is the rung this roll is evidence for, so it has to be one the wave could run. - for (const pace of ['', '120000', '30000 OR x']) { - assert.throws(() => parseShadowGateArguments(ARGV.with(19, pace)), /drain-pace-window-ms/) - } - assert.throws(() => parseShadowGateArguments(ARGV.with(21, '-1')), /drain-applied/) - assert.throws(() => parseShadowGateArguments(ARGV.with(23, 'soon')), /drain-settled-at/) - // A resumed rollback never drains, so it has no applied pace, settle time, or host count. - const resumed = parseShadowGateArguments(ARGV.with(21, '').with(23, '').with(25, '')) - assert.equal(resumed.drainAppliedPaceWindowMs, null) - assert.equal(resumed.targetHosts, null) -}) - -test('the window runs from drain start to verify end, with named fallbacks', () => { - const full = resolveWindow({ - drainStartedAt: '2026-09-20T20:00:00Z', - applyStartedAt: '2026-09-20T20:15:00Z', - verifyEndedAt: '2026-09-20T20:30:00Z' - }) - assert.equal(formatTimestamp(full.startedAt), '2026-09-20T20:00:00Z') - assert.equal(full.startedFrom, 'drain') - // A resumed rollback never drains, so the apply stands in for the start. - assert.equal(resolveWindow({ - applyStartedAt: '2026-09-20T20:15:00Z', - verifyEndedAt: '2026-09-20T20:30:00Z' - }).startedFrom, 'apply') - assert.equal(formatTimestamp(resolveWindow({ - verifyEndedAt: '2026-09-20T20:30:00Z' - }).startedAt), '2026-09-20T20:00:00Z') - assert.throws(() => resolveWindow({ - drainStartedAt: '2026-09-20T20:30:00Z', - verifyEndedAt: '2026-09-20T20:30:00Z' - })) - assert.throws(() => resolveWindow({ verifyEndedAt: 'not-a-time' })) -}) - -test('reads are split into sub-windows no longer than the truncation bound', () => { - const windows = splitWindow(resolveWindow({ - drainStartedAt: '2026-09-20T20:00:00Z', - verifyEndedAt: '2026-09-20T20:47:00Z' - })) - assert.equal(windows.length, 5) - for (const window of windows) { - const minutes = (window.endedAt - window.startedAt) / 60_000 - assert.ok(minutes > 0 && minutes <= SUB_WINDOW_MINUTES, `${minutes} minutes`) - } - assert.equal(formatTimestamp(windows.at(-1).endedAt), '2026-09-20T20:47:00Z') -}) - -test('a sub-window that came back at the entry limit is truncated, never a count', () => { - const truncated = countByMinute([ - { timestamps: minuteOfTimestamps('2026-09-19T15:34', ENTRY_LIMIT) } - ]) - assert.equal(truncated.truncated, true) - const failed = countByMinute([{ failed: true, timestamps: [] }]) - assert.equal(failed.truncated, true) - const counted = countByMinute([ - { timestamps: minuteOfTimestamps('2026-09-19T15:34', 4722) }, - { timestamps: minuteOfTimestamps('2026-09-19T15:33', 278) } - ]) - assert.deepEqual( - { - peak: counted.peak, - peakMinute: counted.peakMinute, - total: counted.total, - truncated: counted.truncated - }, - { peak: 4722, peakMinute: '2026-09-19T15:34', total: 5000, truncated: false } - ) -}) - -test('the cell has to announce its listener and stay up across the whole apply', () => { - assert.equal(judgeCellServing({ - listeningAt: '2026-09-20T20:18:27Z', - crashesSinceApply: 0 - }).status, 'pass') - assert.equal(judgeCellServing({ listeningAt: null }).status, 'would-block') - // A resumed rollback restarts nothing, so there is no boot to find and silence proves nothing. - assert.equal(judgeCellServing({ listeningAt: null, expectBoot: false }).status, 'unverified') - assert.equal(judgeCellServing({ - listeningAt: '2026-09-20T20:18:27Z', - crashesSinceApply: 1 - }).status, 'would-block') - assert.equal(judgeCellServing({ - listeningAt: null, - read: { failed: true } - }).status, 'unverified') -}) - -test('pool pressure blocks only when it persists across consecutive samples', () => { - assert.equal(longestRunAtOrAbove([10, 60, 10, 60, 60, 60, 10], 50), 3) - const burst = judgePool({ - label: 'production-gce-c27', - // The single-sample waiters=71 that a literal rule called an outage. - samples: [ - { databasePoolWaitersMax: 12 }, - { databasePoolWaitersMax: 71 }, - { databasePoolWaitersMax: 9 } - ] - }) - assert.equal(burst.status, 'warn') - assert.equal(burst.consecutiveSamplesOverWaitersThreshold, 1) - assert.equal(judgePool({ - label: 'production-gce-c28', - samples: [ - { databasePoolWaitersMax: 148 }, - { databasePoolWaitersMax: 125 }, - { databasePoolWaitersMax: 154 } - ] - }).status, 'would-block') - assert.equal(judgePool({ - label: 'production-gce-c28', - samples: [{ databasePoolWaitersMax: 2, sqlFailuresDelta: 489 }] - }).status, 'would-block') - assert.equal(judgePool({ - label: 'production-gce-c29', - samples: [{ databasePoolWaitersMax: 3, sqlFailuresDelta: 0, totalConnections: 500 }] - }).status, 'pass') - // No samples at all is silence, not health. - assert.equal(judgePool({ label: 'production-gce-c29', samples: [] }).status, 'unverified') - assert.equal(judgePool({ - label: 'production-gce-c29', - samples: [{ databasePoolWaitersMax: 1 }], - failed: true - }).status, 'unverified') - // A truncated sample run has holes, and a hole reads to the run rule as a recovery. - assert.equal(judgePool({ - label: 'production-gce-c29', - samples: [{ databasePoolWaitersMax: 1 }], - truncated: true - }).status, 'unverified') - // So is a run read across holes, which may join two separate runs into one. - assert.equal(judgePool({ - label: 'production-gce-c29', - samples: Array.from({ length: 3 }, () => ({ databasePoolWaitersMax: 80 })), - truncated: true - }).status, 'unverified') - // But one sample past the failure line is a fact, whatever the read missed. - assert.equal(judgePool({ - label: 'production-gce-c29', - samples: [{ databasePoolWaitersMax: 1, sqlFailuresDelta: 201 }], - truncated: true - }).status, 'would-block') -}) - -test('Cloud SQL FATALs warn from the first one and block on a run of them', () => { - assert.equal(judgeCloudSqlFatal({ count: 0 }).status, 'pass') - assert.equal(judgeCloudSqlFatal({ count: 1 }).status, 'warn') - assert.equal(judgeCloudSqlFatal({ count: 21 }).status, 'would-block') - assert.equal(judgeCloudSqlFatal({ count: 0, truncated: true }).status, 'unverified') - // A truncated count is a floor: already past the block line, more entries only add to it. - assert.equal(judgeCloudSqlFatal({ count: 20000, truncated: true }).status, 'would-block') - assert.equal(judgeCloudSqlFatal({ count: 5, truncated: true }).status, 'unverified') -}) - -test('the verdict is the worst check, and an unverified read never reads as PASS', () => { - assert.equal(combineVerdict({ a: { status: 'pass' }, b: { status: 'pass' } }), 'PASS') - assert.equal(combineVerdict({ a: { status: 'pass' }, b: { status: 'warn' } }), 'WARN') - assert.equal(combineVerdict({ a: { status: 'pass' }, b: { status: 'unverified' } }), 'WARN') - assert.equal( - combineVerdict({ a: { status: 'would-block' }, b: { status: 'unverified' } }), - 'WOULD_BLOCK' - ) -}) - -// The step that owns each stamp, so a stamp's presence is judged where it has to be written. -const STAMP_STEPS = { - drain: '- name: Reversibly isolate and drain only the selected cell', - apply: '- name: Apply only the selected same-cap template and MIG', - 'verify-target': '- name: Verify new incarnation, exact image, protocol, and durable safety' -} - -// One step's own lines: from its marker to the next sibling step at the same indent. -function stepBody(workflow, marker) { - const start = workflow.indexOf(marker) - assert.notEqual(start, -1, `the job no longer has a step named ${marker}`) - const next = workflow.indexOf('\n - ', start + marker.length) - return workflow.slice(start, next === -1 ? undefined : next) -} - -const C28_INSTANCE = '5031087219978409220' - -// Runtime-metrics samples at the 30 s cadence production emits them at, unless a case needs -// enough of them inside one sub-window to reach the read's limit. -function metricSamples({ cellId, from, count, payload = {}, intervalMs = 30_000 }) { - return Array.from({ length: count }, (_, index) => ({ - matches: ['orca_relay_runtime_metrics', `jsonPayload.cellId="${cellId}"`], - timestamp: new Date(Date.parse(from) + index * intervalMs).toISOString(), - payload: { - totalConnections: 857, - databasePoolWaitersMax: 4, - databasePoolWaiting: 1, - sqlFailuresDelta: 0, - reconnectsDelta: 0, - ...payload - } - })) -} - -// The exact entry shapes production returned for c28 on 2026-09-20: the crash at 20:18:10Z and -// the listener at 20:18:27Z, both on instance 5031087219978409220. -function productionLikeEntries() { - return [ - { - matches: ['listening on https://c28.relay.onorca.dev'], - timestamp: '2026-09-20T20:18:27.470301969Z', - instanceId: C28_INSTANCE - }, - ...metricSamples({ cellId: 'production-gce-c28', from: '2026-09-20T20:20:00Z', count: 20 }), - ...metricSamples({ cellId: 'production-gce-c27', from: '2026-09-20T20:20:00Z', count: 20 }), - ...metricSamples({ cellId: 'production-gce-c29', from: '2026-09-20T20:20:00Z', count: 20 }), - ...metricSamples({ cellId: 'production-gce-c30', from: '2026-09-20T20:20:00Z', count: 20 }), - ...metricSamples({ cellId: 'production-gce-c31', from: '2026-09-20T20:20:00Z', count: 20 }), - ...metricSamples({ cellId: 'production-gce-c34', from: '2026-09-20T20:20:00Z', count: 20 }), - ...metricSamples({ cellId: 'production-gce-c35', from: '2026-09-20T20:20:00Z', count: 20 }), - // One director instance's samples from ten minutes before the drain to the window's end. - ...directorSamples({ from: '2026-09-20T19:50:00Z', count: 80, payload: {} }) - ] -} - -/** - * A gcloud seam that honours the filter it is given: its timestamp bounds, its instance-id scope, - * the `--limit`, and the newest-first order. A fake that ignored the bounds would let a - * wrongly-bounded query pass, which is exactly the bug class these tests exist to catch. - */ -function gcloudSeam(entries = productionLikeEntries()) { - const calls = [] - const monitoringCalls = [] - return { - calls, - monitoringCalls, - retryDelayMs: 0, - // Cloud Monitoring's request counter, aligned per minute: one point per minute that had any, - // stamped at the end of the minute it counts, inside the requested interval only. - fetch: async (url, init) => { - const query = new URL(url).searchParams - monitoringCalls.push({ query, init }) - const startedAt = Date.parse(query.get('interval.startTime')) - const endedAt = Date.parse(query.get('interval.endTime')) - const points = entries - .filter((entry) => entry.minute503 !== undefined) - .map((entry) => ({ endedAt: Date.parse(`${entry.minute503}:00Z`) + 60_000, count: entry.count })) - .filter((point) => point.endedAt > startedAt && point.endedAt <= endedAt) - .map((point) => ({ - interval: { endTime: new Date(point.endedAt).toISOString() }, - value: { int64Value: String(point.count) } - })) - return { ok: true, json: async () => ({ timeSeries: points.length ? [{ points }] : [] }) } - }, - runGcloud: async (args, options) => { - if (args[0] === 'auth') return { stdout: 'token\n' } - const filter = args[2] - const limit = Number(args[args.indexOf('--limit') + 1]) - calls.push({ filter, limit, options }) - const startedAt = Date.parse(/timestamp>="([^"]+)"/.exec(filter)[1]) - const endedAt = Date.parse(/timestamp<"([^"]+)"/.exec(filter)[1]) - const instanceId = /resource\.labels\.instance_id="([^"]+)"/.exec(filter)?.[1] - const matched = entries.filter((entry) => { - if (entry.minute503 !== undefined) return false - const at = Date.parse(entry.timestamp) - if (at < startedAt || at >= endedAt) return false - if (instanceId && entry.instanceId !== instanceId) return false - return entry.matches.every((needle) => filter.includes(needle)) - }) - matched.sort((left, right) => Date.parse(right.timestamp) - Date.parse(left.timestamp)) - return { - stdout: JSON.stringify(matched.slice(0, limit).map((entry) => ({ - timestamp: entry.timestamp, - ...(entry.instanceId ? { resource: { labels: { instance_id: entry.instanceId } } } : {}), - ...(entry.payload ? { jsonPayload: entry.payload } : {}) - }))) - } - } - } -} - -test('reads every promoted asia-east2 cell as fleet pool, C35 included', () => { - assert.deepEqual(FLEET_POOL_CELL_IDS, [ - 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30', - 'production-gce-c31', 'production-gce-c34', 'production-gce-c35' - ]) -}) - -test('a healthy roll reads as PASS and names the instance it proved serving', async () => { - const seam = gcloudSeam() - const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), seam) - assert.equal(report.verdict, 'PASS') - assert.equal(report.reportOnly, true) - assert.equal(report.cellInstanceId, C28_INSTANCE) - assert.equal(report.window.startedFrom, 'drain') - assert.equal(report.window.applyCompletedAt, '2026-09-20T20:19:30Z') - assert.deepEqual(Object.keys(report.checks).sort(), [ - 'cellPool', - 'cellServing', - 'cloudSqlFatal', - 'drainDeferrals', - 'fleetPool:production-gce-c27', - 'fleetPool:production-gce-c29', - 'fleetPool:production-gce-c30', - 'fleetPool:production-gce-c31', - 'fleetPool:production-gce-c34', - 'fleetPool:production-gce-c35', - 'nonDrain503Budget' - ]) - assert.deepEqual(report.drain, { - paceWindowMs: 300_000, - appliedPaceWindowMs: 300_000, - targetHosts: 857, - settledAt: '2026-09-20T20:11:00Z', - settledAfterSeconds: 660 - }) - // Every read carries explicit bounds: --freshness does not bind on these logs. - for (const { filter } of seam.calls) { - assert.match(filter, /timestamp>="[^"]+" AND timestamp<"[^"]+"/) - } - // Cell text lives in jsonPayload.message; a textPayload filter matches nothing and says so. - assert.equal(seam.calls.some(({ filter }) => filter.includes('textPayload')), false) - assert.match(renderStepSummary(report), /Shadow health gate \(report only\): PASS/) - assert.match(renderStepSummary(report), /Drain pace 300000 ms \(cell applied: 300000\), 857 hosts/) -}) - -// The listener lands while the MIG is still converging, so a boot search opening at the apply's -// completion finds nothing and calls a healthy roll a failure. -test('the boot search opens at the apply start, not at its completion', async () => { - const seam = gcloudSeam() - const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), seam) - assert.equal(report.checks.cellServing.status, 'pass') - assert.equal(report.checks.cellServing.listeningAt, '2026-09-20T20:18:27.470301969Z') - const listenerRead = seam.calls.find(({ filter }) => filter.includes('listening on https://')) - assert.match(listenerRead.filter, /timestamp>="2026-09-20T20:15:00Z"/) - // The listener at 20:18:27 sits after the apply start and before its completion at 20:19:30, - // so a completion-bounded search would have missed it entirely. - assert.ok(Date.parse('2026-09-20T20:18:27.470301969Z') < Date.parse('2026-09-20T20:19:30Z')) -}) - -// A crash-restart loop ends with a listener announcement that looks like a clean boot. Counting -// crashes only after the last announcement erases the loop that produced it. -test('a crash before the final listener still counts against the roll', async () => { - const seam = gcloudSeam([ - ...productionLikeEntries(), - { - matches: ['throw er'], - timestamp: '2026-09-20T20:18:10.651702662Z', - instanceId: C28_INSTANCE - } - ]) - const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), seam) - assert.equal(report.checks.cellServing.crashesSinceApply, 1) - assert.equal(report.checks.cellServing.status, 'would-block') - assert.equal(report.verdict, 'WOULD_BLOCK') - const crashRead = seam.calls.find(({ filter }) => filter.includes('throw er')) - // Bounded at the apply start, and still scoped to the instance the listener identified. - assert.match(crashRead.filter, /timestamp>="2026-09-20T20:15:00Z"/) - assert.match(crashRead.filter, new RegExp(`resource\\.labels\\.instance_id="${C28_INSTANCE}"`)) -}) - -test('a crash on a neighbouring instance is not charged to this cell', async () => { - const seam = gcloudSeam([ - ...productionLikeEntries(), - { matches: ['throw er'], timestamp: '2026-09-20T20:18:10Z', instanceId: '9999999999999999999' } - ]) - const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), seam) - assert.equal(report.checks.cellServing.crashesSinceApply, 0) - assert.equal(report.checks.cellServing.status, 'pass') -}) - -// A sample run returned at the read's limit has holes, and the consecutive-sample rule reads a -// hole as a recovery, so it must not be judged as though it were complete. -test('a runtime-metrics read at its limit is unverified, not a calm cell', async () => { - const seam = gcloudSeam([ - ...productionLikeEntries(), - // 600 samples packed into the first sub-window, past the 500-entry read limit. - ...metricSamples({ - cellId: 'production-gce-c28', - from: '2026-09-20T20:00:00Z', - count: 600, - intervalMs: 500, - payload: { databasePoolWaitersMax: 1 } - }) - ]) - const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), seam) - assert.equal(report.checks.cellPool.status, 'unverified') - assert.equal(report.checks.cellPool.truncated, true) - // The neighbours were read normally, so only the truncated cell is unverified. - assert.equal(report.checks['fleetPool:production-gce-c27'].status, 'pass') - assert.equal(report.verdict, 'WARN') -}) - -test('a resume, which restarts nothing, does not read a missing boot as a failure', async () => { - const resumed = ARGV.with(9, '').with(11, '').with(13, '') - const seam = gcloudSeam(productionLikeEntries().filter( - (entry) => !entry.matches[0].startsWith('listening') - )) - const report = await evaluateShadowGate(parseShadowGateArguments(resumed), seam) - assert.equal(report.window.startedFrom, 'fallback') - assert.equal(report.checks.cellServing.status, 'unverified') - assert.equal(report.verdict, 'WARN') -}) - -test('a gcloud read that never completes is unverified, not a crashed gate', async () => { - const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), { - retryDelayMs: 0, - runGcloud: async () => { throw new Error('PERMISSION_DENIED') } - }) - assert.equal(report.verdict, 'WARN') - assert.equal(report.checks.nonDrain503Budget.status, 'unverified') - assert.equal(report.checks.cellServing.status, 'unverified') -}) - -// continue-on-error bounds the job's outcome but not its clock; an unbounded read could spend the -// rollout's remaining minutes before the job's own timeout noticed. -test('every read is given a bounded timeout, and a timed-out read is just a failed read', async () => { - const seam = gcloudSeam() - await evaluateShadowGate(parseShadowGateArguments(ARGV), seam) - assert.ok(seam.calls.length > 0) - for (const { options } of seam.calls) { - assert.equal(options.timeoutMs, SHADOW_GATE_THRESHOLDS.readTimeoutMs) - assert.ok(options.timeoutMs > 0 && options.timeoutMs <= 120_000) - } - const timedOut = await evaluateShadowGate(parseShadowGateArguments(ARGV), { - retryDelayMs: 0, - runGcloud: async () => { throw Object.assign(new Error('ETIMEDOUT'), { killed: true }) } - }) - assert.equal(timedOut.checks.nonDrain503Budget.status, 'unverified') - assert.equal(timedOut.verdict, 'WARN') -}) - -// The reads are serialised, so the cost of a failure that makes every one of them spend its full -// retry budget scales with the window. The deadline is what turns that into a verdict rather than -// a cancelled job, which would take every later cell in the wave with it. -test('the gate stops reading at its own deadline and still reports a verdict', async () => { - const seam = gcloudSeam() - // A clock where every read costs its whole retry budget, which is the case the deadline exists - // for: an expired credential or a Logging 429 storm answers nothing, slowly, every time. - let elapsedMs = 0 - const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), { - ...seam, - now: () => { - elapsedMs += SHADOW_GATE_THRESHOLDS.readTimeoutMs * READ_ATTEMPTS - return elapsedMs - } - }) - // Everything past the deadline is skipped rather than attempted, so the gate cannot outlive it. - assert.ok(seam.calls.length > 0, 'the gate must still attempt reads inside its budget') - assert.ok( - seam.calls.length * SHADOW_GATE_THRESHOLDS.readTimeoutMs * READ_ATTEMPTS <= - SHADOW_GATE_THRESHOLDS.overallDeadlineMs, - 'the gate read past its own deadline' - ) - // A verdict, not a crash: a skipped read is an unverified check, which can never read as PASS. - assert.equal(report.reportOnly, true) - assert.equal(report.verdict, 'WARN') - assert.equal(report.checks.cellServing.status, 'unverified') - // No read is ever given more time than the budget still has left. - for (const { options } of seam.calls) { - assert.ok(options.timeoutMs > 0) - assert.ok(options.timeoutMs <= SHADOW_GATE_THRESHOLDS.readTimeoutMs) - } -}) - -test('the job runs the gate report-only, after verification, and uploads its artifact', () => { - const workflow = readRelayWorkflow('deploy-relay-production-same-cap-job.yml') - const gate = workflow.slice(workflow.indexOf('- name: Shadow health gate (report only)')) - assert.notEqual(gate, '') - // Two independent guarantees that no verdict can fail a cell: the step's own exit code and this. - assert.match(gate.slice(0, gate.indexOf('run:')), /continue-on-error: true/) - assert.match(gate, /relay-same-cap-shadow-gate\.mjs/) - // The gate and its upload must be bounded in time as well as in outcome: a step that runs past - // the job's timeout-minutes gets the job cancelled, and cancellation stops the whole wave. - const gateHeader = gate.slice(0, gate.indexOf('run:')) - assert.match(gateHeader, /timeout-minutes: (\d+)/) - const stepTimeoutMinutes = Number(/timeout-minutes: (\d+)/.exec(gateHeader)[1]) - assert.equal(stepTimeoutMinutes, 8) - // The script has to settle on its own before the runner kills it, or the artifact is never - // written and the step reports nothing at all. - assert.ok( - SHADOW_GATE_THRESHOLDS.overallDeadlineMs < stepTimeoutMinutes * 60_000, - 'the gate deadline must leave the step time to write its verdict' - ) - const upload = workflow.slice(workflow.indexOf('- name: Publish the shadow health gate verdict')) - assert.match(upload.slice(0, upload.indexOf('uses:')), /timeout-minutes: 2/) - assert.match( - workflow, - /name: relay-same-cap-shadow-gate-\$\{\{ inputs\.target-cell-id \}\}-\$\{\{ github\.run_id \}\}\.json/ - ) - // The gate is judged over the wave it just ran, so the job has to stamp its own steps, and the - // stamps reach the script through the environment rather than being expanded into its shell. - for (const [step, output] of [ - ['drain', 'drain-started-at'], - ['drain', 'drain-settled-at'], - ['apply', 'apply-started-at'], - ['apply', 'apply-completed-at'], - ['verify-target', 'verify-ended-at'] - ]) { - // Scoped to the step that owns the stamp: a stamp written anywhere else in the job would - // still satisfy a whole-file match while recording the wrong instant. - assert.match( - stepBody(workflow, STAMP_STEPS[step]), - new RegExp(`${output}=\\$\\(date -u \\+%FT%TZ\\)`), - `${output} must be stamped inside the ${step} step` - ) - assert.match(gate, new RegExp(`\\$\\{\\{ steps\\.${step}\\.outputs\\.${output} \\}\\}`)) - assert.match(gate, new RegExp(`--${output} "\\$\\{[A-Z_]+\\}"`)) - } - // Settled means restart-safe was proven, so the stamp follows that wait, not the drain call. - const drainStep = stepBody(workflow, STAMP_STEPS.drain) - assert.notEqual(drainStep.indexOf('--activity restart-safe'), -1) - assert.ok( - drainStep.indexOf('drain-settled-at=') > drainStep.indexOf('--activity restart-safe'), - 'drain-settled-at must be stamped after the restart-safe wait' - ) - assert.match(drainStep, /drain-applied-pace-window-ms=\$\(jq -er '\.paceWindowMs' <<< "\$\{DRAIN_RESULT\}"\)/) - for (const flag of ['drain-pace-window-ms', 'drain-applied-pace-window-ms', 'target-hosts']) { - assert.match(gate, new RegExp(`--${flag} "\\$\\{[A-Z_]+(:-)?\\}"`), flag) - } - // The apply-start stamp has to precede the operation that can restart the instance, or the - // listener it bounds the search by has already happened. Presence is asserted before order, - // because indexOf answers -1 for an absent stamp and -1 precedes everything. - const applyStep = stepBody(workflow, STAMP_STEPS.apply) - const stampedAt = applyStep.indexOf('apply-started-at=') - const appliedAt = applyStep.indexOf('terraform -chdir=infra/terraform apply') - assert.notEqual(stampedAt, -1, 'the apply step does not stamp apply-started-at at all') - assert.notEqual(appliedAt, -1, 'the apply step no longer runs terraform apply') - assert.ok(stampedAt < appliedAt, 'apply-started-at must be stamped before terraform apply') - // Verification has to have happened first, or the gate judges a cell nothing checked, and the - // restore too, so reading logs never holds the cell out of admission for longer than today. - for (const earlier of [ - '- name: Verify new incarnation, exact image, protocol, and durable safety', - '- name: Restore only the verified selected cell to its entry admission' - ]) { - assert.ok( - workflow.indexOf(earlier) < workflow.indexOf('- name: Shadow health gate (report only)'), - earlier - ) - } -}) - -// Director runtime-metrics samples: one per 30 s per instance, each counting the 30 s before it. -function directorSamples({ from, count, payload }) { - return Array.from({ length: count }, (_, index) => ({ - matches: ['orca_relay_runtime_metrics', 'resource.type="cloud_run_revision"'], - timestamp: new Date(Date.parse(from) + index * 30_000).toISOString(), - payload - })) -} - -function director503s(minute, count) { - return [{ minute503: minute, count }] -} - -test('row-busy 503s are scheduled only up to the drain-return admissions they ride on', () => { - const drain = drainReturnByMinute([{ - failed: false, - samples: [{ - timestamp: '2026-10-05T20:01:00Z', - drainReturnAssignmentsDelta: 10, - assign503sByCauseDelta: { relay_assignment_row_busy: 8, 'placement-lane': 5 } - }] - }], 1000) - assert.deepEqual(drain.ownRetriesPerMinute, { '2026-10-05T20:00': 8 }) - assert.equal(drain.rowBusyBeyondDrainTotal, 0) - const split = withoutDrainDeferrals( - { perMinute: { '2026-10-05T20:00': 13 } }, - drain, - ['2026-10-05T20:00'] - ) - // The placement-lane refusals stay: only the row-busy ones were scheduled. - assert.deepEqual(split.series, [5]) -}) - -test('row-busy 503s beyond the drain stay in the non-drain budget and fail it', () => { - const minutes = Array.from({ length: 10 }, (_, index) => `2026-10-05T20:0${index}`) - // No drain in the background, and 3 drain-return admissions a minute in the window against 60 - // row-busy refusals: row contention the drain does not explain. - const samples = minutes.map((minute, index) => ({ - timestamp: new Date(Date.parse(`${minute}:30Z`) + 30_000).toISOString(), - drainReturnAssignmentsDelta: index < 5 ? 0 : 3, - assign503sByCauseDelta: { relay_assignment_row_busy: index < 5 ? 0 : 60 } - })) - const drain = drainReturnByMinute([{ failed: false, samples }], 1000) - assert.equal(drain.rowBusyBeyondDrainTotal, 5 * (60 - 3 - 2)) - const perMinute = Object.fromEntries(minutes.map((minute, index) => [minute, index < 5 ? 2 : 60])) - const background = backgroundOf(withoutDrainDeferrals({ perMinute }, drain, minutes.slice(0, 5))) - const observed = withoutDrainDeferrals({ perMinute }, drain, minutes.slice(5)) - assert.deepEqual(observed.series, [55, 55, 55, 55, 55]) - assert.equal(judgeNonDrain503Budget({ observed, background }).status, 'would-block') -}) - -test('scheduled 503s come out of the count, split across the minutes they cover', () => { - const drain = drainReturnByMinute([{ - failed: false, - samples: [ - // Counts 20:00:30-20:01:00, so all of it belongs to 20:00. - { timestamp: '2026-10-05T20:01:00Z', drainReturnDeferralsDelta: 40, drainReturnAssignmentsDelta: 70 }, - // Counts 20:00:50-20:01:20: a third in 20:00, two thirds in 20:01. - { - timestamp: '2026-10-05T20:01:20Z', - drainReturnDeferralsDelta: 30, - drainReturnAssignmentsDelta: 90, - drainReturnRetryAfterSecondsMax: 12, - placementRejectionsByReasonDelta: { 'host-rate-limited': 6, 'wait-timeout': 50 }, - stickyRejectionsByReasonDelta: { 'host-in-flight': 3, 'queue-full': 9 } - } - ] - }], 1000) - assert.deepEqual(drain.deferralsPerMinute, { '2026-10-05T20:00': 50, '2026-10-05T20:01': 20 }) - // A host's own early retry is scheduled; a lane that timed out or was full is not. - assert.deepEqual(drain.ownRetriesPerMinute, { '2026-10-05T20:00': 3, '2026-10-05T20:01': 6 }) - assert.equal(drain.assignmentsPeakPerMinute, 100) - assert.equal(drain.retryAfterSecondsMax, 12) - const minutes = ['2026-10-05T20:00', '2026-10-05T20:01', '2026-10-05T20:02'] - const split = withoutDrainDeferrals({ - perMinute: { '2026-10-05T20:00': 90, '2026-10-05T20:01': 20, '2026-10-05T20:02': 25 } - }, drain, minutes) - // A minute cannot go negative when more was scheduled than was counted. - assert.deepEqual(split.series, [37, 0, 25]) - assert.equal(split.peak, 37) - assert.equal(split.peakMinute, '2026-10-05T20:00') - assert.equal(split.allPeak, 90) - assert.equal(split.drainDeferralsTotal, 70) - assert.equal(split.ownRetriesTotal, 9) - assert.equal(split.unverified, false) - // A failed count, or a metrics read that failed, hit its limit, or came back short of one - // instance's samples, leaves the answer unverified: an empty answer is not a calm director. - assert.equal(drainReturnByMinute([{ failed: true, samples: [] }], 1000).truncated, true) - assert.equal(drainReturnByMinute([{ samples: [], minSamples: 20 }], 1000).truncated, true) - assert.equal(drainReturnByMinute([{ - samples: Array.from({ length: 19 }, (_, index) => ({ - timestamp: new Date(Date.parse('2026-10-05T20:00:30Z') + index * 30_000).toISOString() - })), - minSamples: 20 - }], 1000).truncated, true) - assert.equal(withoutDrainDeferrals({ perMinute: {}, failed: true }, drain, minutes).unverified, true) - assert.equal(withoutDrainDeferrals( - { perMinute: {} }, - { deferralsPerMinute: {}, truncated: true }, - minutes - ).unverified, true) -}) - -test('the background is the median pre-drain minute, so one incident minute cannot move it', () => { - const background = backgroundOf({ - minutes: Array.from({ length: 10 }, (_, index) => `2026-10-05T19:5${index}`), - series: [2, 0, 1, 3, 900, 1, 0, 2, 4, 1], - peak: 900, - unverified: false - }) - assert.equal(background.medianPerMinute, 1.5) - assert.equal(backgroundOf({ minutes: [], series: [], peak: 0, unverified: false }).unverified, true) -}) - -test('the rung budget needs two straight minutes over its line, never one', () => { - const judge = (series, medianPerMinute = 1.5, unverified = false) => judgeNonDrain503Budget({ - observed: { series, peak: Math.max(...series), unverified }, - background: { medianPerMinute, unverified: false } - }) - const calm = judge([0, 3, 87, 1, 0, 39, 0, 22, 12]) - assert.equal(calm.peakPerMinute, 87) - // 10-02 c16 and c21: single minutes at 87 and 39 are transients. - assert.equal(calm.status, 'pass') - assert.equal(calm.warnAbove, 21.5) - assert.equal(calm.blockAbove, 41.5) - assert.equal(judge([0, 25, 23, 0]).status, 'warn') - // 10-01 c29: thousands a minute for nine minutes. - assert.equal(judge([288, 2619, 4226, 5357, 6256, 6662]).status, 'would-block') - assert.equal(judge([0, 42, 42, 0]).status, 'would-block') - // One minute past max(10x, 200) blocks alone: a short herd is over before a second minute. - const spike = judge([0, 201, 0]) - assert.equal(spike.spikeAbove, 200) - assert.equal(spike.status, 'would-block') - assert.equal(judge([0, 112, 0]).status, 'pass') - // 10-01 00:30 c28: a brownout already under way lifts the median, and the sustained rule holds. - assert.equal(judge([8136, 7000, 3000], 824).status, 'would-block') - // A busy background lifts both lines by its multiple, not just the margin. - const busy = judge([100, 100], 60) - assert.equal(busy.warnAbove, 90) - assert.equal(busy.blockAbove, 120) - assert.equal(busy.status, 'warn') - assert.equal(judge([0], 1.5, true).status, 'unverified') -}) - -test('a drain is judged on how long it tells hosts to wait, not on how many it defers', () => { - const drain = (retryAfterSecondsMax, truncated = false) => ({ - deferralsTotal: 400, - deferralsPeakPerMinute: 300, - assignmentsTotal: 530, - assignmentsPeakPerMinute: 260, - retryAfterSecondsMax, - truncated - }) - assert.equal(judgeDrainDeferrals(drain(30)).status, 'pass') - assert.equal(judgeDrainDeferrals(drain(31)).status, 'warn') - assert.equal(judgeDrainDeferrals(drain(61)).status, 'would-block') - assert.equal(judgeDrainDeferrals(drain(2, true)).status, 'unverified') - assert.equal(judgeDrainDeferrals(drain(2)).replacementsPeakPerMinute, 260) -}) - -// A 30 s rung on a 530-host cell: hundreds of scheduled deferrals in two minutes, over a pre-drain -// background whose own 503s were ordinary. Counting the deferrals as failures would block every -// fast drain. -test('a fast drain\'s deferrals do not read as a brownout', async () => { - const entries = [ - ...productionLikeEntries(), - ...director503s('2026-09-20T19:52', 20), - ...director503s('2026-09-20T19:55', 25), - ...director503s('2026-09-20T20:00', 400), - ...director503s('2026-09-20T20:01', 330), - ...director503s('2026-09-20T20:05', 30), - ...directorSamples({ - from: '2026-09-20T20:00:30Z', - count: 4, - payload: { - drainReturnDeferralsDelta: 190, - drainReturnAssignmentsDelta: 130, - drainReturnRetryAfterSecondsMax: 14 - } - }) - ] - const seam = gcloudSeam(entries) - const report = await evaluateShadowGate( - parseShadowGateArguments(ARGV.with(19, '30000').with(21, '30000')), - seam - ) - // One count of every minute from ten before the drain to the end of the window, whole minutes. - assert.equal(seam.monitoringCalls.length, 1) - assert.equal(seam.monitoringCalls[0].query.get('interval.startTime'), '2026-09-20T19:50:00.000Z') - assert.equal(seam.monitoringCalls[0].query.get('interval.endTime'), '2026-09-20T20:30:00.000Z') - assert.match(seam.monitoringCalls[0].query.get('filter'), /run\.googleapis\.com\/request_count/) - assert.match(seam.monitoringCalls[0].query.get('filter'), /"response_code"="503"/) - assert.equal(seam.monitoringCalls[0].init.headers.authorization, 'Bearer token') - assert.equal(report.background.minutes, 10) - assert.equal(Math.max(...report.background.perMinute), 25) - assert.equal(report.checks.nonDrain503Budget.allPeakPerMinute, 400) - assert.equal(report.checks.nonDrain503Budget.drainDeferralsTotal, 760) - assert.equal(report.checks.nonDrain503Budget.peakPerMinute, 30) - assert.equal(report.checks.nonDrain503Budget.status, 'pass') - assert.equal(report.checks.drainDeferrals.status, 'pass') - assert.equal(report.checks.drainDeferrals.replacementsTotal, 520) - assert.equal(report.drain.paceWindowMs, 30_000) - assert.equal(report.paceVerdict, 'PASS') - assert.equal(report.verdict, 'PASS') - // The same 503s with no deferrals behind them are exactly what the gate exists to catch. - const brownout = await evaluateShadowGate( - parseShadowGateArguments(ARGV), - gcloudSeam(entries.filter((entry) => !entry.payload?.drainReturnDeferralsDelta)) - ) - assert.equal(brownout.checks.nonDrain503Budget.status, 'would-block') - assert.equal(brownout.paceVerdict, 'WOULD_BLOCK') -}) - -// The pace verdict is what a canary seals, so fleet noise the pace cannot cause must not reach it. -test('the pace verdict reads only the checks a drain pace can move', async () => { - const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), gcloudSeam([ - ...productionLikeEntries(), - { matches: ['FATAL'], timestamp: '2026-09-20T20:10:00Z' } - ])) - assert.equal(report.checks.cloudSqlFatal.status, 'warn') - assert.equal(report.verdict, 'WARN') - assert.equal(report.paceVerdict, 'PASS') - assert.deepEqual(PACE_CHECKS, ['nonDrain503Budget', 'drainDeferrals']) -}) - -// Reproduced in final review: every read succeeds and returns nothing. That must not read as a calm -// drain, or it seals a canary PASS that authorizes a fast batch. -test('a gate whose reads all come back empty is unverified, never a pace PASS', async () => { - const seam = { - retryDelayMs: 0, - fetch: async () => ({ ok: true, json: async () => ({}) }), - runGcloud: async (args) => ({ stdout: args[0] === 'auth' ? 'token\n' : '[]' }) - } - const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), seam) - assert.equal(report.checks.nonDrain503Budget.status, 'unverified') - assert.equal(report.checks.drainDeferrals.status, 'unverified') - assert.equal(report.paceVerdict, 'WARN') -}) diff --git a/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs b/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs deleted file mode 100644 index 5dee88b11e0..00000000000 --- a/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs +++ /dev/null @@ -1,172 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import { test } from 'node:test' -import { relayWorkflowFile, relayWorkflowUrl } from './relay-repository.mjs' - -const workflow = readFileSync( - relayWorkflowUrl('prove-relay-staging-capacity.yml'), - 'utf8' -) -const recoveryWorkflow = readFileSync( - relayWorkflowUrl('recover-relay-staging-c4-image.yml'), - 'utf8' -) -const requeueWorkflow = readFileSync( - relayWorkflowUrl('requeue-relay-staging-c4-recovery.yml'), - 'utf8' -) -const githubActions = readFileSync( - new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url), - 'utf8' -) -const cells = readFileSync( - new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url), - 'utf8' -) -const relay = readFileSync(new URL('../../infra/terraform/relay.tf', import.meta.url), 'utf8') -const stagingTfvars = readFileSync( - new URL('../../infra/terraform/environments/staging.tfvars', import.meta.url), - 'utf8' -) -const productionTfvars = readFileSync( - new URL('../../infra/terraform/environments/production.tfvars', import.meta.url), - 'utf8' -) - -const launchDigest = '5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563' - -// Scoped to the Asia cells by name: the production capacity cells now serve this digest too, -// so a file-wide count no longer isolates Asia. -const asiaCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'] -// C30 launches after the launch cells rolled, so it pins the director's digest instead. -const c30Digest = '4158d8a2e18e9caec439d257f0c1e45d92ffea8c0262f057b2f08c76a134bcf0' -// C31 launches on the digest the director served when it was declared. -const c31Digest = 'f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269' -// Staging C4 is declared on image J (main 30f5bef9, step-6 phonePath staging test); the staging -// director deploy binds its digest to this line. -const stagingC4Digest = '21f1fead4d3b9900d3b1179fd72569dcba548dc0ee29c76c8389fec94f74a7b4' - -function cellBlock(tfvars, cellId) { - const start = tfvars.indexOf(`"${cellId}"`) - assert.notEqual(start, -1, `${cellId} is missing`) - return tfvars.slice(start, tfvars.indexOf('\n }', start)) -} - -const productionCell = (cellId) => cellBlock(productionTfvars, cellId) - -// Scoped to C4 by name: staging C3 serves this digest too since its 2026-09-03 re-pin. -test('pins staging C4 to the step-5 image, the launch Asia cells to one image, and C30/C31 to director images', () => { - assert.match(cellBlock(stagingTfvars, 'staging-gce-c4'), new RegExp(`relay@sha256:${stagingC4Digest}"`)) - for (const cellId of asiaCells) { - assert.match(productionCell(cellId), new RegExp(`relay@sha256:${launchDigest}"`), cellId) - } - assert.match(recoveryWorkflow, new RegExp(`TARGET_IMAGE_DIGEST: sha256:${launchDigest}`)) - assert.match(productionCell('production-gce-c30'), new RegExp(`relay@sha256:${c30Digest}"`)) - assert.match(productionCell('production-gce-c31'), new RegExp(`relay@sha256:${c31Digest}"`)) -}) - -test('refreshes only empty staging C4 through the trusted capacity identity', () => { - const refresh = workflow.slice(workflow.indexOf(' refresh-asia-c4-image:')) - assert.match(refresh, /terraform_version: 1\.17\.0/) - assert.doesNotMatch(refresh, /terraform_version: 1\.5\.7/) - assert.match(refresh, /github\.ref == 'refs\/heads\/main'/) - assert.match(refresh, /REFRESH_STAGING_ASIA_C4_IMAGE/) - assert.match(refresh, /APPROVED_PREDECESSOR_IMAGE_DIGEST/) - assert.match(refresh, /--activity quiescent/) - assert.match(refresh, /--admission migration-only/) - assert.match(refresh, /fence_digests="\$\{PREDECESSOR_IMAGE_DIGEST\}"/) - assert.match(refresh, /--expected-image-digests "\$\{TARGET_IMAGE_DIGEST\}"/) - assert.match(refresh, /--mode same-cap-image/) - assert.match(refresh, /test "\$\(jq -r '\.changes'/) - assert.match(refresh, /REFRESH_PHASE=\$\{refresh_phase\}/) - assert.match(refresh, /MUTATION_STARTED=false/) - assert.match(refresh, /MIG_STABLE_AT_MS=/) - assert.match(refresh, /PLAN_CHANGES=/) - assert.match(refresh, /obsolete-template-delete/) - assert.match( - refresh, - /\*:manager-convergence\|\*:replacement-with-obsolete-template\) refresh_phase=converging/ - ) - assert.match(refresh, /--mode isolate/) - assert.match(refresh, /--mode verify/) - assert.match(refresh, /\.status\.runtime\.ready/) - assert.match(refresh, /\.status\.runtime\.startedAt/) - assert.match(refresh, /\.status\.runtime\.lastHeartbeatAt/) - assert.match(refresh, /--runtime unavailable/) - const plan = refresh.indexOf('Save, validate, and classify the exact C4 plan') - const currentState = refresh.indexOf('Verify the exact selector and current C4 state') - const isolate = refresh.indexOf('--mode isolate') - const apply = refresh.indexOf('terraform -chdir=infra/terraform apply -auto-approve') - assert.ok(plan < currentState && currentState < isolate && isolate < apply) - assert.match(refresh, /Require an empty targeted Terraform readback/) -}) - -test('recovers a failed or cancelled C4 refresh from an independent workflow', () => { - assert.match(recoveryWorkflow, /terraform_version: 1\.17\.0/) - assert.doesNotMatch(recoveryWorkflow, /terraform_version: 1\.5\.7/) - assert.match(recoveryWorkflow, /workflow_run:/) - assert.match(recoveryWorkflow, /workflows: \[Prove Relay Staging Capacity\]/) - assert.match(recoveryWorkflow, /RECOVER_STAGING_ASIA_C4_IMAGE/) - assert.match(recoveryWorkflow, /outputs:\n\s+recover: \$\{\{ steps\.trigger\.outputs\.recover \}\}/) - assert.match(recoveryWorkflow, /if test "\$\{count\}" = 0; then\n\s+echo "recover=false"/) - assert.match(recoveryWorkflow, /needs: gate\n\s+if: \$\{\{ needs\.gate\.outputs\.recover == 'true' \}\}/) - assert.match(recoveryWorkflow, /concurrency:\n\s+group: relay-staging-mutation/) - assert.match(recoveryWorkflow, /group: relay-staging-mutation/) - assert.match(recoveryWorkflow, /\.name == "refresh-asia-c4-image"/) - assert.match(recoveryWorkflow, /PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13/) - assert.match(recoveryWorkflow, /TARGET_IMAGE_DIGEST: sha256:5aedbca5/) - assert.match(recoveryWorkflow, /id: preflight-auth/) - assert.match(recoveryWorkflow, /id: verify-auth/) - assert.match(recoveryWorkflow, /\.status\.runtime\.ready/) - assert.match(recoveryWorkflow, /--runtime unavailable/) - assert.match(recoveryWorkflow, /current_digest.*\^sha256:\[a-f0-9\]\{64\}\$/) - assert.match(recoveryWorkflow, /expected_digests="\$\{expected_digests\},\$\{current_digest\}"/) - assert.match(recoveryWorkflow, /--timeout-ms 240000/) - assert.doesNotMatch(recoveryWorkflow, /--timeout-ms 900000/) - assert.match(recoveryWorkflow, /-var-file=environments\/staging.tfvars -lock-timeout=5m/) - assert.doesNotMatch(recoveryWorkflow, /manage_artifact_dns/) - assert.match(recoveryWorkflow, /--mode same-cap-image/) - assert.match(recoveryWorkflow, /test "\$\(jq -r '\.changes'/) - assert.equal(recoveryWorkflow.match(/\*:replacement-with-obsolete-template/g)?.length, 2) - assert.match( - recoveryWorkflow, - /\^\(replacement\|replacement-with-obsolete-template\|manager-convergence\)\$/ - ) - const preflightAuth = recoveryWorkflow.indexOf('id: preflight-auth') - const preflight = recoveryWorkflow.indexOf('Inspect the exact C4 recovery state') - const recoveryPlan = recoveryWorkflow.indexOf('Classify both exact recovery end states') - const apply = recoveryWorkflow.indexOf('Apply and stabilize the saved predecessor plan') - const restore = recoveryWorkflow.indexOf('Restart only when the plan did not replace C4') - const verifyAuth = recoveryWorkflow.indexOf('id: verify-auth') - const verify = recoveryWorkflow.indexOf('Verify the recovered image and unchanged isolation') - assert.ok(preflightAuth < preflight && preflight < recoveryPlan && recoveryPlan < apply) - assert.ok(apply < restore) - assert.ok(restore < verifyAuth && verifyAuth < verify) - assert.match(githubActions, /"recover-relay-staging-c4-image\.yml"/) -}) - -test('requeues a protected C4 recovery cancelled while pending', () => { - assert.match(requeueWorkflow, /workflows: \[Recover Relay Staging C4 Image\]/) - assert.match(requeueWorkflow, /conclusion == 'cancelled'/) - assert.match(requeueWorkflow, /permissions:\n\s+actions: write\n\s+contents: read/) - assert.match(requeueWorkflow, /group: relay-staging-c4-recovery-requeue/) - assert.match(requeueWorkflow, /\.name == "recover" and\n\s+\.conclusion == "cancelled"/) - assert.match(requeueWorkflow, /\.started_at == null/) - assert.match(requeueWorkflow, /\.name == "gate" and \.conclusion == "success"/) - assert.match(requeueWorkflow, /\.name == "recover" and \.status != "completed"/) - assert.match(requeueWorkflow, /actions\/runs\/\$\{run_id\}\/jobs\?filter=latest/) - assert.match(requeueWorkflow, /if test "\$\{active\}" != 0; then exit 0; fi/) - assert.ok(requeueWorkflow.includes(`gh workflow run ${relayWorkflowFile('recover-relay-staging-c4-image.yml')}`)) - assert.match(requeueWorkflow, /-f confirmation=RECOVER_STAGING_ASIA_C4_IMAGE/) - assert.doesNotMatch(requeueWorkflow, /id-token: write/) - assert.doesNotMatch(requeueWorkflow, /relay-staging-mutation/) -}) - -test('keeps cell-only plans independent from service-account description drift', () => { - assert.match(cells, /runtime_service_account\s+= local\.relay_runtime_service_account_email/) - assert.match( - cells, - /rehome_director_service_account\s+= local\.relay_director_runtime_service_account_email/ - ) - assert.match(relay, /var\.environment == "staging" \? "Orca Relay"/) -}) diff --git a/cloud/dev/scripts/relay-staging-capacity-identity.test.mjs b/cloud/dev/scripts/relay-staging-capacity-identity.test.mjs deleted file mode 100644 index 31a38d08124..00000000000 --- a/cloud/dev/scripts/relay-staging-capacity-identity.test.mjs +++ /dev/null @@ -1,269 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { relayWorkflowUrl } from './relay-repository.mjs' - -const terraform = readFileSync( - new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url), - 'utf8' -) -const outputs = readFileSync(new URL('../../infra/terraform/outputs.tf', import.meta.url), 'utf8') -const workflow = readFileSync( - relayWorkflowUrl('prove-relay-staging-capacity.yml'), - 'utf8' -) -const deployWorkflow = readFileSync( - relayWorkflowUrl('deploy-relay-staging.yml'), - 'utf8' -) -const publishWorkflow = readFileSync( - relayWorkflowUrl('publish-relay-production.yml'), - 'utf8' -) -const bootstrapWorkflow = readFileSync( - relayWorkflowUrl('bootstrap-relay-staging-capacity.yml'), - 'utf8' -) - -function resource(type, name) { - const start = terraform.indexOf(`resource "${type}" "${name}"`) - assert.notEqual(start, -1, `${type}.${name} is missing`) - const next = terraform.indexOf('\nresource "', start + 1) - return terraform.slice(start, next === -1 ? undefined : next) -} - -function terraformStringList(block, attribute) { - const match = block.match(new RegExp(`${attribute}\\s*=\\s*\\[([\\s\\S]*?)\\]`)) - assert.ok(match, `${attribute} is missing`) - return [...match[1].matchAll(/"([^"]+)"/g)].map((entry) => entry[1]) -} - -test('capacity workflow uses only its exact staging identity', () => { - assert.match(workflow, /STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/) - assert.match(workflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/) - assert.doesNotMatch(workflow, /vars\.STAGING_GCP_WORKLOAD_IDENTITY_PROVIDER/) - assert.doesNotMatch(workflow, /vars\.STAGING_GCP_DEPLOY_SERVICE_ACCOUNT/) - - const provider = resource( - 'google_iam_workload_identity_pool_provider', - 'github_staging_relay_capacity' - ) - // The three repository claims are pinned once in relay-shared.tf; every provider concatenates - // that list rather than restating the repository on its own. - assert.match(provider, /concat\(local\.relay_github_leading_repository_claims, \[/) - for (const boundary of [ - "assertion.ref == 'refs/heads/main'", - "assertion.environment == 'staging'" - ]) { - assert.match(provider, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'))) - } - assert.match(provider, /local\.relay_github_workflow_conditions\["github_staging_relay_capacity"\]/) - assert.deepEqual( - terraformStringList(terraform, 'github_staging_relay_capacity_workflow_files'), - [ - 'bootstrap-relay-staging-capacity.yml', - 'prove-relay-staging-capacity.yml', - 'recover-relay-staging-c4-image.yml' - ] - ) - assert.match( - terraform, - /for workflow_file in local\.github_staging_relay_capacity_workflow_files : "assertion\.workflow_ref == '\$\{prefix\}\$\{workflow_file\}@refs\/heads\/main'"/ - ) - assert.doesNotMatch(terraform, /github_staging_relay_capacity_workflow_file\s*=/) -}) - -test('job gates do not read environment variables before the environment is attached', () => { - for (const source of [workflow, deployWorkflow, bootstrapWorkflow]) { - const jobGate = source.match(/^\s{4}if:.*$/m)?.[0] ?? '' - assert.doesNotMatch(jobGate, /STAGING_GCP_RELAY_CAPACITY_/) - } -}) - -test('capacity identity has bounded mutation and state permissions', () => { - const role = resource( - 'google_project_iam_custom_role', - 'github_staging_relay_capacity_mutation' - ) - assert.deepEqual(terraformStringList(role, 'permissions'), [ - 'compute.disks.create', - 'compute.healthChecks.use', - 'compute.images.useReadOnly', - 'compute.instanceGroupManagers.get', - 'compute.instanceGroupManagers.update', - 'compute.instances.create', - 'compute.instances.setLabels', - 'compute.instances.setMetadata', - 'compute.instances.setTags', - 'compute.instanceTemplates.create', - 'compute.instanceTemplates.delete', - 'compute.instanceTemplates.get', - 'compute.instanceTemplates.useReadOnly', - 'compute.networks.use', - 'compute.subnetworks.use', - 'compute.zoneOperations.get' - ]) - assert.doesNotMatch( - role, - /compute\.(?:disks\.delete|instances\.(?:delete|start|stop|update))|cloudsql|secretmanager/ - ) - - for (const source of [workflow, bootstrapWorkflow]) { - assert.match(source, /instance-groups managed recreate-instances/) - assert.match(source, /--instances/) - assert.doesNotMatch(source, /rolling-action restart/) - } - - const state = resource( - 'google_storage_bucket_iam_member', - 'github_staging_relay_capacity_state' - ) - assert.match(state, /roles\/storage\.objectAdmin/) - assert.match(state, /objects\/terraform\/state\/default\.tfstate/) - assert.match(state, /objects\/terraform\/state\/default\.tflock/) - assert.doesNotMatch(state, /resource\.name\.startsWith/) - - const runtime = resource( - 'google_service_account_iam_member', - 'github_staging_relay_capacity_runtime_user' - ) - assert.match(runtime, /google_service_account\.relay_runtime\.name/) - assert.match(runtime, /roles\/iam\.serviceAccountUser/) - - const cloudRun = resource( - 'google_cloud_run_v2_service_iam_member', - 'github_staging_relay_capacity_developer' - ) - assert.match(cloudRun, /name\s*=\s*var\.relay_cloud_run_service_name/) - assert.doesNotMatch(cloudRun, /google_cloud_run_v2_service\.relay/) - - const relay = readFileSync(new URL('../../infra/terraform/relay.tf', import.meta.url), 'utf8') - const startup = readFileSync( - new URL('../../infra/terraform/relay-gce-startup.sh.tftpl', import.meta.url), - 'utf8' - ) - assert.match(relay, /ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT/) - assert.match(startup, /ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT/) -}) - -test('capacity identity exposes only its provider and service account', () => { - assert.match(outputs, /output "github_staging_relay_capacity_workload_identity_provider"/) - assert.match(outputs, /output "github_staging_relay_capacity_service_account"/) -}) - -test('director capacity configuration stays on the audited blue-green path', () => { - assert.match(deployWorkflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/) - assert.match(deployWorkflow, /--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/) - assert.match(deployWorkflow, /expected-image-digest/) - assert.match(deployWorkflow, /var\.relay_gce_cells\["staging-gce-c4"\]\.image/) - assert.match(deployWorkflow, /init -reconfigure \\\n\s+-backend-config=backend\/staging\.hcl/) - assert.ok( - deployWorkflow.indexOf('id: google-auth') < - deployWorkflow.indexOf('Bind the request to the checked-in staging C4 image') - ) - assert.match(deployWorkflow, /artifacts docker images describe "\$\{IMAGE\}"/) - assert.doesNotMatch(deployWorkflow, /docker (?:build|push)/) - assert.match(workflow, /--director-cells-json "\$\{DESIRED_CELLS_JSON\}"/) - assert.doesNotMatch(workflow, /target=google_cloud_run_v2_service\.relay/) - assert.doesNotMatch(workflow, /--mode director/) -}) - -test('mirrors the exact production manifest through the production deploy identity', () => { - assert.match(publishWorkflow, /options: \[publish, mirror-staging\]/) - assert.match(publishWorkflow, /MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING/) - assert.match(publishWorkflow, /docker pull "\$\{source_image\}"/) - assert.match(publishWorkflow, /docker tag "\$\{source_image\}" "\$\{target_tag\}"/) - assert.match(publishWorkflow, /test "\$\{source_digest\}" = "\$\{MIRROR_DIGEST\}"/) - assert.match(publishWorkflow, /test "\$\{target_digest\}" = "\$\{MIRROR_DIGEST\}"/) - const mirrorWriter = resource( - 'google_artifact_registry_repository_iam_member', - 'github_production_relay_staging_mirror_writer' - ) - assert.match(mirrorWriter, /var\.environment == "staging"/) - assert.match(mirrorWriter, /roles\/artifactregistry\.writer/) - assert.match( - mirrorWriter, - /serviceAccount:orca-cloud-gha-deploy@onorca-cloud\.iam\.gserviceaccount\.com/ - ) -}) - -test('cells bootstrap one at a time with bounded deploy and capacity identities', () => { - assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER/) - assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT/) - assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/) - assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/) - assert.match(bootstrapWorkflow, /--mode bootstrap-cell/) - assert.match(bootstrapWorkflow, /--cell-id "\$\{fallback_cell_id\}"/) - assert.match(bootstrapWorkflow, /google_compute_instance_template\.relay_gce_cell/) - assert.match(bootstrapWorkflow, /google_compute_instance_group_manager\.relay_gce_cell/) - assert.match(bootstrapWorkflow, /--mode restore-fallback/) - assert.doesNotMatch(bootstrapWorkflow, /target=google_cloud_run_v2_service\.relay/) - assert.ok( - bootstrapWorkflow.indexOf('id: deploy-auth') < bootstrapWorkflow.indexOf('id: capacity-auth') - ) - assert.match( - bootstrapWorkflow, - /restore_fallback\(\) \{[\s\S]*?verify_fallback[\s\S]*?--mode restore-fallback/ - ) - const rollCell = bootstrapWorkflow.slice(bootstrapWorkflow.indexOf('roll_cell()')) - assert.ok( - rollCell.indexOf('verify_fallback\n trap restore_fallback EXIT') < - rollCell.indexOf('--mode isolate') - ) - assert.match( - bootstrapWorkflow, - /staging-gce-c2 general[\s\S]*?trap restore_legacy_c3_fallback EXIT[\s\S]*?--mode isolate/ - ) - const normalize = bootstrapWorkflow.slice( - bootstrapWorkflow.indexOf('normalize_legacy_c3() {'), - bootstrapWorkflow.indexOf('\n roll_cell()', bootstrapWorkflow.indexOf('normalize_legacy_c3() {')) - ) - const trapInstalled = normalize.indexOf('trap restore_legacy_c3_fallback EXIT') - const isolated = normalize.indexOf('--mode isolate', trapInstalled) - const recreated = normalize.indexOf('recreate-instances', isolated) - const restored = normalize.indexOf('--mode restore', recreated) - const c3Verified = normalize.indexOf('staging-gce-c3 general', restored) - const restoreDisabled = normalize.indexOf('legacy_c3_isolated=false', c3Verified) - const trapCleared = normalize.indexOf('trap - EXIT', restoreDisabled) - assert.ok( - trapInstalled < isolated && - isolated < recreated && - recreated < restored && - restored < c3Verified && - c3Verified < restoreDisabled && - restoreDisabled < trapCleared - ) - assert.equal(normalize.indexOf('trap - EXIT', trapInstalled), trapCleared) - assert.match(bootstrapWorkflow, /--heartbeat either/) - assert.doesNotMatch(bootstrapWorkflow, /heartbeat=stale/) - assert.match( - rollCell, - /"\$\{desired_cap\}" "\$\{desired_bound\}" absent-or-stale[\s\S]*?deploy-relay-blue-green\.mjs[\s\S]*?"\$\{desired_cap\}" "\$\{desired_bound\}"/ - ) -}) - -test('workflows read desired topology from configuration and gate exact predecessors', () => { - assert.match(workflow, /<<< 'local\.relay_director_cells_json' \| jq -r '\.'/) - assert.match(bootstrapWorkflow, /<<< 'local\.relay_director_cells_json' \| jq -r '\.'/) - assert.match(workflow, /1000\/0\)[\s\S]*?PREDECESSOR_C3_CAP=600/) - assert.match(workflow, /1000\/60\)[\s\S]*?PREDECESSOR_C3_BOUND=0/) - assert.match(workflow, /600\/60\)[\s\S]*?PREDECESSOR_C3_CAP=1000/) - assert.match(workflow, /Unsupported staging capacity transition/) -}) - -test('capacity apply resumes after director or cell success and preserves the no-op restart proof', () => { - for (const phase of ['predecessor', 'director-ready', 'cell-ready', 'cell-active']) { - assert.match(workflow, new RegExp(`TRANSITION_PHASE=${phase}`)) - } - assert.match(workflow, /--argjson expected "\$\{PREDECESSOR_CELLS_JSON\}"/) - assert.match(workflow, /--argjson expected "\$\{DESIRED_CELLS_JSON\}"/) - assert.match( - workflow, - /test "\$\{TRANSITION_PHASE\}" = cell-ready; then[\s\S]*?test "\$\{CELL_PLAN_CHANGES\}" = 0/ - ) - assert.match( - workflow, - /test "\$\{TRANSITION_PHASE\}" = cell-active; then[\s\S]*?recreate_fixed_one_instance/ - ) - assert.match(workflow, /--admission migration-only/) -}) diff --git a/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs b/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs deleted file mode 100644 index 66168db1309..00000000000 --- a/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs +++ /dev/null @@ -1,238 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import test from 'node:test' -import { readWorkflow, workflowFiles } from './cloud-sql-rollout-lock-census.mjs' -import { prefixedRelayWorkflowPath, relayWorkflowFile } from './relay-repository.mjs' - -const identity = readFileSync( - new URL('../../infra/terraform/relay-staging-deploy-iam.tf', import.meta.url), - 'utf8' -) -const shared = readFileSync( - new URL('../../infra/terraform/relay-shared.tf', import.meta.url), - 'utf8' -) -const variables = readFileSync( - new URL('../../infra/terraform/variables.tf', import.meta.url), - 'utf8' -) -const outputs = readFileSync(new URL('../../infra/terraform/outputs.tf', import.meta.url), 'utf8') -const stagingTfvars = readFileSync( - new URL('../../infra/terraform/environments/staging.tfvars', import.meta.url), - 'utf8' -) - -// The exact six staging Relay workflows the relay-owned deploy identity serves. -const DEPLOY_WORKFLOWS = [ - 'bootstrap-relay-staging-capacity.yml', - 'deploy-relay-staging-gce-candidate.yml', - 'deploy-relay-staging.yml', - 'operate-relay-asia-admission.yml', - 'operate-relay-staging-cell-flags.yml', - 'power-relay-staging.yml' -] - -const GENERIC_STAGING_PAIR = - /vars\.STAGING_GCP_(?:WORKLOAD_IDENTITY_PROVIDER|DEPLOY_SERVICE_ACCOUNT)\b/ - -const workflowNames = workflowFiles -// DEPLOY_WORKFLOWS holds the names Terraform pins; the files on disk carry the copy's prefix. -const workflow = (name) => readWorkflow(relayWorkflowFile(name)) - -function block(type, name) { - const start = identity.indexOf(`resource "${type}" "${name}"`) - assert.notEqual(start, -1, `${type}.${name} is missing`) - const next = identity.indexOf('\nresource "', start + 1) - return identity.slice(start, next === -1 ? undefined : next) -} - -function declaredFamilies() { - return [...identity.matchAll(/^resource "([a-z0-9_]+)" "([a-z0-9_]+)"/gm)] - .map((match) => `${match[1]}.${match[2]}`) - .sort() -} - -function providerWorkflowFiles() { - const match = identity.match( - /github_staging_relay_deploy_workflow_files\s*=\s*\[([\s\S]*?)\n {2}\]/ - ) - assert.ok(match, 'github_staging_relay_deploy_workflow_files is missing') - return [...match[1].matchAll(/"([^"]+)"/g)].map((entry) => entry[1]) -} - -function variableDefault(name) { - const match = variables.match( - new RegExp(`variable "${name}" \\{[\\s\\S]*?default\\s*=\\s*"([^"]*)"`) - ) - assert.ok(match, `variable ${name} has no default`) - return match[1] -} - -test('no Relay workflow authenticates as the shared staging deploy identity', () => { - for (const name of workflowNames()) { - if (!name.includes('relay')) continue - assert.doesNotMatch(readWorkflow(name), GENERIC_STAGING_PAIR, name) - } -}) - -test('the six staging Relay workflows name the relay deploy pair', () => { - for (const name of DEPLOY_WORKFLOWS) { - const source = workflow(name) - assert.match(source, /vars\.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER\b/, name) - assert.match(source, /vars\.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT\b/, name) - } -}) - -// Why: the Asia workflow serves both environments from one job. Repointing its staging arm must -// not move production off the relay-owned shared account. -test('the Asia admission production arm keeps the production deploy pair', () => { - const source = workflow('operate-relay-asia-admission.yml') - assert.match(source, /vars\.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER\b/) - assert.match(source, /vars\.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT\b/) -}) - -test('the provider allowlists exactly those six workflow refs', () => { - const files = providerWorkflowFiles() - assert.deepEqual([...files].sort(), [...DEPLOY_WORKFLOWS].sort()) - for (const file of files) { - assert.match(file, /^[a-z0-9-]+\.yml$/) - } - // Each accepted repository turns that file list into its own exact refs. - assert.match( - identity, - /for workflow_file in local\.github_staging_relay_deploy_workflow_files : "assertion\.workflow_ref == '\$\{prefix\}\$\{workflow_file\}@refs\/heads\/main'"/ - ) - - const provider = block( - 'google_iam_workload_identity_pool_provider', - 'github_staging_relay_deploy' - ) - assert.match(provider, /workload_identity_pool_provider_id\s*=\s*"github-relay-deploy"/) - assert.match(provider, /concat\(local\.relay_github_leading_repository_claims/) - assert.match(provider, /assertion\.ref == 'refs\/heads\/main'/) - assert.match(provider, /assertion\.environment == 'staging'/) - assert.match(provider, /local\.relay_github_workflow_conditions\["github_staging_relay_deploy"\]/) - // A prefix match would turn the allowlist into a namespace grant with no Terraform diff. - assert.doesNotMatch(provider, /startsWith|endsWith/) -}) - -// Why: the documented attribute_condition limit is 4096 characters and the expression grows with -// every workflow added. Render it the way Terraform does and keep the headroom visible. -test('the rendered attribute condition stays inside the provider limit', () => { - const repository = `${variableDefault('github_owner')}/${variableDefault('github_repo')}` - const claims = [ - `assertion.repository == '${repository}'`, - `assertion.repository_id == '${variableDefault('github_repo_id')}'`, - `assertion.repository_owner_id == '${variableDefault('github_owner_id')}'` - ] - // The prefix is the Terraform variable, not this checkout's own workflow filenames: the - // condition names the files as the trusted repository carries them. - const prefix = variableDefault('github_workflow_file_prefix') - const workflowRefs = providerWorkflowFiles().map( - (file) => `${repository}/${prefixedRelayWorkflowPath(prefix, file)}@refs/heads/main` - ) - const rendered = [ - ...claims, - "assertion.ref == 'refs/heads/main'", - "assertion.environment == 'staging'", - `(${workflowRefs.map((ref) => `assertion.workflow_ref == '${ref}'`).join(' || ')})` - ].join(' && ') - assert.ok(rendered.length < 4096, `rendered condition is ${rendered.length} characters`) - assert.equal(rendered.length, 913) -}) - -// Why: the census is the point. A binding added here without a workflow step behind it, or one -// silently dropped, changes what the staging Relay credential can reach. -test('the staging deploy identity declares exactly its enumerated grants', () => { - assert.deepEqual(declaredFamilies(), [ - 'google_artifact_registry_repository_iam_member.github_staging_relay_deploy_artifact_reader', - 'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_auth_developer', - 'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer', - 'google_iam_workload_identity_pool_provider.github_staging_relay_deploy', - 'google_project_iam_member.github_staging_relay_deploy_compute_viewer', - 'google_project_iam_member.github_staging_relay_deploy_logging_viewer', - 'google_service_account.github_staging_relay_deploy', - 'google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user', - 'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user', - 'google_storage_bucket_iam_member.github_staging_relay_deploy_state', - 'google_storage_bucket_iam_member.github_staging_relay_deploy_state_list' - ]) - // Each grant carries a comment naming the workflow step that needs it; the account, its - // provider, and the pool binding are the identity itself and are covered by the file header. - const identityFamilies = new Set([ - 'google_service_account.github_staging_relay_deploy', - 'google_iam_workload_identity_pool_provider.github_staging_relay_deploy', - 'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user' - ]) - for (const family of declaredFamilies()) { - if (identityFamilies.has(family)) continue - const [type, name] = family.split('.') - const preceding = identity.slice(0, identity.indexOf(`resource "${type}" "${name}"`)).trimEnd() - assert.match(preceding.slice(preceding.lastIndexOf('\n') + 1), /^#/, `${family} has no justifying comment`) - } - assert.match(identity, /var\.environment == "staging"/) - - const state = block('google_storage_bucket_iam_member', 'github_staging_relay_deploy_state') - assert.match(state, /roles\/storage\.objectViewer/) - assert.match(state, /objects\/terraform\/state\/default\.tfstate/) - assert.match(state, /objects\/terraform\/state\/default\.tflock/) - assert.doesNotMatch(state, /resource\.name\.startsWith/) - assert.doesNotMatch(state, /objectAdmin/) - - const director = block( - 'google_cloud_run_v2_service_iam_member', - 'github_staging_relay_deploy_director_developer' - ) - assert.match(director, /name\s*=\s*var\.relay_cloud_run_service_name/) - - // Project-wide run.developer or artifactregistry.writer would let the staging Relay credential - // deploy the API service or push images; the shared account holds both today. - const projectRoles = declaredFamilies() - .filter((family) => family.startsWith('google_project_iam_member.')) - .map((family) => block('google_project_iam_member', family.split('.')[1]).match(/role\s*=\s*"([^"]+)"/)[1]) - assert.deepEqual(projectRoles, ['roles/compute.viewer', 'roles/logging.viewer']) - assert.doesNotMatch(identity, /roles\/artifactregistry\.writer/) - assert.doesNotMatch(identity, /"roles\/(?:owner|editor|viewer)"/) -}) - -// Why: the auth-plane grants are guarded on a variable, so an unset tfvars entry would drop them -// silently and Power Relay Staging would fail only on the sleep path. -test('staging pins the shared auth service the power workflow scales', () => { - assert.match(stagingTfvars, /relay_staging_power_auth_service_name\s*=\s*"orca-cloud-auth-staging"/) - assert.match(variables, /variable "relay_staging_power_auth_service_name"/) - for (const name of [ - 'github_staging_relay_deploy_auth_developer', - 'github_staging_relay_deploy_auth_runtime_user' - ]) { - const type = name.endsWith('runtime_user') - ? 'google_service_account_iam_member' - : 'google_cloud_run_v2_service_iam_member' - assert.match(block(type, name), /var\.relay_staging_power_auth_service_name != ""/) - } -}) - -// Why: flipping this local is what moves the staging cells' startup metadata and the director's -// ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT onto the new account. Production must keep the shared one. -test('the deploy account email is environment-conditional', () => { - assert.match( - shared, - /relay_github_deploy_service_account_email = \(\s*var\.environment == "production"\s*\? "\$\{var\.name_prefix\}-gha-deploy@\$\{var\.project_id\}\.iam\.gserviceaccount\.com"\s*: "\$\{var\.name_prefix\}-gha-relay@\$\{var\.project_id\}\.iam\.gserviceaccount\.com"\s*\)/ - ) - assert.match(identity, /account_id\s*=\s*"\$\{var\.name_prefix\}-gha-relay"/) -}) - -test('the identity is exposed through its own outputs', () => { - assert.match(outputs, /output "github_staging_relay_deploy_workload_identity_provider"/) - assert.match(outputs, /output "github_staging_relay_deploy_service_account"/) -}) - -// Why: relay-deploy-workflow-argv.test.mjs proves the argv parses; this pins the identity it names. -test('the staging director deploy runs as relay-dir with its rehome identity', () => { - const source = workflow('deploy-relay-staging.yml') - const start = source.indexOf('node dev/scripts/deploy-relay-blue-green.mjs') - const block = source.slice(start, source.indexOf('\n\n', start)) - assert.match(block, /--runtime-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/) - assert.match(block, /--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/) - assert.match(source, /DIRECTOR_RUNTIME_SERVICE_ACCOUNT: orca-cloud-staging-relay-dir@/) - assert.match(block, /--deploy-service-account "\$\{DEPLOY_SERVICE_ACCOUNT\}"/) -}) diff --git a/cloud/dev/scripts/render-workload-identity-conditions.mjs b/cloud/dev/scripts/render-workload-identity-conditions.mjs deleted file mode 100644 index 815ba1b2c17..00000000000 --- a/cloud/dev/scripts/render-workload-identity-conditions.mjs +++ /dev/null @@ -1,536 +0,0 @@ -// Renders every Workload Identity provider `attribute_condition` exactly as -// Terraform would, so contract tests can pin the resulting strings without a -// plan. Understands only the HCL subset those expressions use. -// -// Each root is loaded on its own: the relay and apps roots both declare a provider named -// `github` while the staging copy waits on its state surgery, and only separate scopes can -// show that the two render the same string. -// -// Only the relay root ships in this repository. The apps root is still declared so this stays a -// straight copy of the private original, and is skipped when its directory is absent. -import { existsSync } from 'node:fs' -import { readFile } from 'node:fs/promises' - -const TERRAFORM_ROOTS = { - relay: { - directory: 'infra/terraform', - sources: [ - 'infra/terraform/relay-shared.tf', - 'infra/terraform/relay-github-workflow-trust.tf', - 'infra/terraform/relay-github-actions.tf', - 'infra/terraform/push-deploy-identity.tf', - 'infra/terraform/relay-staging-deploy-iam.tf', - 'infra/terraform/relay-asia-topology-iam.tf', - 'infra/terraform/relay-asia-proof-iam.tf' - ] - }, - apps: { - directory: 'infra/terraform-apps', - sources: ['infra/terraform-apps/github-actions.tf'] - } -} - -export function hasTerraformRoot(root) { - const directory = TERRAFORM_ROOTS[root]?.directory - return directory !== undefined && existsSync(repoFile(directory)) -} - -export const TERRAFORM_ROOT_NAMES = Object.keys(TERRAFORM_ROOTS).filter(hasTerraformRoot) - -const PROVIDER_RESOURCE = 'google_iam_workload_identity_pool_provider' - -function repoFile(path) { - return new URL(`../../${path}`, import.meta.url) -} - -function skipTrivia(src, index) { - let i = index - for (;;) { - while (i < src.length && /\s/.test(src[i])) i += 1 - if (src[i] === '#') { - while (i < src.length && src[i] !== '\n') i += 1 - continue - } - return i - } -} - -// Returns the index just past the closing quote of the string starting at `i`. -function endOfString(src, i) { - let cursor = i + 1 - while (src[cursor] !== '"') { - if (src[cursor] === '\\') { - cursor += 2 - continue - } - if (src[cursor] === '$' && src[cursor + 1] === '{') { - cursor = endOfInterpolation(src, cursor + 2).next - continue - } - cursor += 1 - } - return cursor + 1 -} - -function endOfInterpolation(src, i) { - let depth = 1 - let cursor = i - while (depth > 0) { - const char = src[cursor] - if (char === undefined) throw new Error('unterminated interpolation') - if (char === '"') { - cursor = endOfString(src, cursor) - continue - } - if (char === '{') depth += 1 - else if (char === '}') { - depth -= 1 - if (depth === 0) break - } - cursor += 1 - } - return { text: src.slice(i, cursor), next: cursor + 1 } -} - -// Stands in for a loop variable when the collection is empty: the body still has to be parsed -// once to find where it ends, and any attribute of the probe is another probe. -const PROBE = new Proxy( - {}, - { - get: (target, key) => (key === Symbol.toPrimitive ? () => '' : PROBE) - } -) - -function readMember(value, key) { - if (value === PROBE) return PROBE - if (value === null || value === undefined) throw new Error(`cannot read ${String(key)} of ${value}`) - if (Array.isArray(value)) { - if (typeof key !== 'number') throw new Error(`list index must be a number, got ${String(key)}`) - if (!Number.isInteger(key) || key < 0 || key >= value.length) { - throw new Error(`list index ${key} is out of range`) - } - return value[key] - } - if (typeof value !== 'object') throw new Error(`cannot index ${typeof value}`) - if (!Object.hasOwn(value, key)) throw new Error(`unknown attribute ${String(key)}`) - return value[key] -} - -// [key, value] pairs the way HCL iterates: list index and element, or object key and value. -function collectionEntries(collection) { - if (Array.isArray(collection)) return collection.map((item, index) => [index, item]) - if (collection && typeof collection === 'object') return Object.entries(collection) - throw new Error(`cannot iterate ${typeof collection}`) -} - -class ExpressionParser { - constructor(source, scope) { - this.source = source - this.scope = scope - this.index = 0 - } - - parse() { - const value = this.parseTernary() - this.index = skipTrivia(this.source, this.index) - if (this.index !== this.source.length) { - throw new Error(`trailing expression text: ${this.source.slice(this.index)}`) - } - return value - } - - peek(token) { - this.index = skipTrivia(this.source, this.index) - return this.source.startsWith(token, this.index) - } - - eat(token) { - if (!this.peek(token)) return false - this.index += token.length - return true - } - - expect(token) { - if (!this.eat(token)) { - throw new Error(`expected ${token} at ${this.source.slice(this.index, this.index + 40)}`) - } - } - - parseTernary() { - const condition = this.parseOr() - if (!this.eat('?')) return condition - const consequent = this.parseTernary() - this.expect(':') - const alternate = this.parseTernary() - return condition ? consequent : alternate - } - - parseOr() { - let left = this.parseAnd() - while (this.eat('||')) left = Boolean(this.parseAnd()) || Boolean(left) - return left - } - - parseAnd() { - let left = this.parseEquality() - while (this.eat('&&')) left = Boolean(this.parseEquality()) && Boolean(left) - return left - } - - parseEquality() { - let left = this.parseUnary() - for (;;) { - if (this.eat('==')) left = left === this.parseUnary() - else if (this.eat('!=')) left = left !== this.parseUnary() - else return left - } - } - - parseUnary() { - return this.parsePostfix(this.parsePrimary()) - } - - parsePrimary() { - if (this.eat('(')) { - const value = this.parseTernary() - this.expect(')') - return value - } - if (this.peek('"')) return this.parseString() - if (this.peek('[')) return this.parseList() - if (this.peek('{')) return this.parseObject() - const number = /^[0-9]+/.exec(this.source.slice(this.index)) - if (number) { - this.index += number[0].length - return Number(number[0]) - } - return this.parseIdentifier() - } - - parsePostfix(value) { - let current = value - for (;;) { - if (this.eat('.')) { - current = readMember(current, this.readWord()) - continue - } - if (this.peek('[')) { - this.index += 1 - const key = this.parseTernary() - this.expect(']') - current = readMember(current, key) - continue - } - return current - } - } - - // `for a in x : body` / `for a, b in x : body`, shared by list and object comprehensions. - parseComprehension(readBody) { - const names = [this.readWord()] - if (this.eat(',')) names.push(this.readWord()) - this.expect('in') - const collection = this.parseUnary() - this.expect(':') - const bodyStart = skipTrivia(this.source, this.index) - const entries = collectionEntries(collection) - const bodyParser = ([key, item]) => { - const bindings = { ...this.scope.bindings } - if (names.length === 1) bindings[names[0]] = Array.isArray(collection) ? item : key - else { - bindings[names[0]] = key - bindings[names[1]] = item - } - const parser = new ExpressionParser(this.source, { ...this.scope, bindings }) - parser.index = bodyStart - return parser - } - // Parse once with a probe binding to find where the body ends, because an empty - // collection would never parse it. - const probe = bodyParser(entries[0] ?? [PROBE, PROBE]) - readBody(probe) - this.index = probe.index - return entries.map((entry) => readBody(bodyParser(entry))) - } - - parseObject() { - this.expect('{') - if (this.eat('for')) { - const pairs = this.parseComprehension((parser) => { - const key = parser.parseTernary() - parser.expect('=>') - return [key, parser.parseTernary()] - }) - this.expect('}') - return Object.fromEntries(pairs) - } - const object = {} - if (this.eat('}')) return object - for (;;) { - const key = this.peek('"') ? this.parseString() : this.readWord() - this.expect('=') - object[key] = this.parseTernary() - this.eat(',') - if (this.eat('}')) return object - } - } - - parseString() { - this.index = skipTrivia(this.source, this.index) - const src = this.source - let cursor = this.index + 1 - let rendered = '' - while (src[cursor] !== '"') { - if (src[cursor] === '\\') { - rendered += src[cursor + 1] - cursor += 2 - continue - } - if (src[cursor] === '$' && src[cursor + 1] === '{') { - const { text, next } = endOfInterpolation(src, cursor + 2) - rendered += String(evaluate(text, this.scope)) - cursor = next - continue - } - rendered += src[cursor] - cursor += 1 - } - this.index = cursor + 1 - return rendered - } - - parseList() { - this.expect('[') - if (this.eat('for')) { - const items = this.parseComprehension((parser) => parser.parseTernary()) - this.expect(']') - return items - } - const items = [] - if (this.eat(']')) return items - for (;;) { - items.push(this.parseTernary()) - if (this.eat(',')) { - if (this.eat(']')) return items - continue - } - this.expect(']') - return items - } - } - - readWord() { - this.index = skipTrivia(this.source, this.index) - const match = /^[A-Za-z_][A-Za-z0-9_]*/.exec(this.source.slice(this.index)) - if (!match) throw new Error(`expected identifier at ${this.source.slice(this.index, this.index + 40)}`) - this.index += match[0].length - return match[0] - } - - parseIdentifier() { - const word = this.readWord() - if (word === 'join') { - this.expect('(') - const separator = this.parseTernary() - this.expect(',') - const parts = this.parseTernary() - this.eat(',') - this.expect(')') - return parts.join(separator) - } - if (word === 'concat') { - this.expect('(') - const lists = [] - for (;;) { - lists.push(this.parseTernary()) - if (this.eat(',')) { - if (this.eat(')')) break - continue - } - this.expect(')') - break - } - return lists.flat() - } - if (word === 'length') { - this.expect('(') - const value = this.parseTernary() - this.eat(',') - this.expect(')') - return collectionEntries(value).length - } - if (word === 'local') { - this.expect('.') - return resolveLocal(this.readWord(), this.scope) - } - if (word === 'var') { - this.expect('.') - const name = this.readWord() - if (!(name in this.scope.variables)) throw new Error(`unknown variable ${name}`) - return this.scope.variables[name] - } - if (word in this.scope.bindings) return this.scope.bindings[word] - if (word === 'true') return true - if (word === 'false') return false - throw new Error(`unsupported identifier ${word}`) - } -} - -function evaluate(source, scope) { - return new ExpressionParser(source, scope).parse() -} - -function resolveLocal(name, scope) { - if (scope.resolved.has(name)) return scope.resolved.get(name) - if (!scope.locals.has(name)) throw new Error(`unknown local ${name}`) - if (scope.resolving.has(name)) throw new Error(`local cycle at ${name}`) - scope.resolving.add(name) - const value = evaluate(scope.locals.get(name), { ...scope, bindings: {} }) - scope.resolving.delete(name) - scope.resolved.set(name, value) - return value -} - -function collectLocals(source, locals) { - const blockPattern = /^locals \{$/gm - let match - while ((match = blockPattern.exec(source)) !== null) { - const end = source.indexOf('\n}\n', match.index) - const body = source.slice(match.index + match[0].length, end) - let name = null - let buffer = [] - const flush = () => { - if (name) locals.set(name, buffer.join('\n')) - } - for (const line of body.split('\n')) { - const assignment = /^ {2}([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line) - if (assignment) { - flush() - name = assignment[1] - buffer = [assignment[2]] - continue - } - if (name && line.trim() !== '' && !line.trim().startsWith('#')) buffer.push(line) - } - flush() - } -} - -function collectProviderFields(source, fields) { - const pattern = new RegExp(`resource "${PROVIDER_RESOURCE}" "([A-Za-z_0-9]+)" \\{`, 'g') - let match - while ((match = pattern.exec(source)) !== null) { - const end = source.indexOf('\n}\n', match.index) - const body = source.slice(match.index, end) - const conditionStart = body.indexOf(' attribute_condition = ') - const conditionEnd = body.indexOf('\n\n oidc {', conditionStart) - const countStart = body.indexOf(' count = ') - fields.set(match[1], { - count: body.slice(countStart + ' count = '.length, body.indexOf('\n', countStart)), - condition: body.slice(conditionStart + ' attribute_condition = '.length, conditionEnd) - }) - } -} - -// Values that are not a plain quoted string (a list of objects, say) are read with the -// expression parser; anything it cannot evaluate is left undefined, exactly as before. -function parseValueAt(source, index) { - const parser = new ExpressionParser(source, { - locals: new Map(), - variables: {}, - bindings: {}, - resolved: new Map(), - resolving: new Set() - }) - parser.index = index - return parser.parseTernary() -} - -function collectVariableDefaults(source, variables) { - const pattern = /variable "([A-Za-z_0-9]+)" \{([\s\S]*?)\n\}/g - let match - while ((match = pattern.exec(source)) !== null) { - const body = match[2] - const fallback = /\n\s*default\s*=\s*"([^"]*)"/.exec(body) - if (fallback) { - variables[match[1]] = fallback[1] - continue - } - const assignment = /\n\s*default\s*=\s*/.exec(body) - if (!assignment) continue - const start = match.index + match[0].indexOf(body) + assignment.index + assignment[0].length - try { - variables[match[1]] = parseValueAt(source, start) - } catch { - // A default this evaluator does not understand is not one any condition reads. - } - } -} - -function collectTfvars(source, variables) { - let offset = 0 - for (const line of source.split('\n')) { - const quoted = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*"([^"]*)"\s*$/.exec(line) - if (quoted) { - variables[quoted[1]] = quoted[2] - offset += line.length + 1 - continue - } - const structured = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*/.exec(line) - if (structured) { - try { - variables[structured[1]] = parseValueAt(source, offset + structured[0].length) - } catch { - // Same as above: unreadable here means unread by every condition. - } - } - offset += line.length + 1 - } -} - -async function loadScope(root, environment) { - const { directory, sources } = TERRAFORM_ROOTS[root] ?? {} - if (!sources) throw new Error(`unknown terraform root ${root}`) - const locals = new Map() - const providers = new Map() - for (const path of sources) { - const source = await readFile(repoFile(path), 'utf8') - collectLocals(source, locals) - collectProviderFields(source, providers) - } - const variables = {} - collectVariableDefaults(await readFile(repoFile(`${directory}/variables.tf`), 'utf8'), variables) - collectTfvars( - await readFile(repoFile(`${directory}/environments/${environment}.tfvars`), 'utf8'), - variables - ) - return { - providers, - scope: { locals, variables, bindings: {}, resolved: new Map(), resolving: new Set() } - } -} - -// Rendered `attribute_condition` per provider that the given root creates in the environment. -export async function renderRootAttributeConditions(root, environment) { - const { providers, scope } = await loadScope(root, environment) - const rendered = {} - for (const [name, fields] of providers) { - if (evaluate(fields.count, scope) === 0) continue - rendered[name] = evaluate(fields.condition, scope) - } - return rendered -} - -// Every root's rendered conditions, keyed by root and then by provider. -export async function renderAttributeConditions(environment) { - const rendered = {} - for (const root of TERRAFORM_ROOT_NAMES) { - rendered[root] = await renderRootAttributeConditions(root, environment) - } - return rendered -} - -export async function readTerraformLocal(name, environment, root = 'relay') { - const { scope } = await loadScope(root, environment) - return resolveLocal(name, scope) -} diff --git a/cloud/dev/scripts/run-relay-load-model.mjs b/cloud/dev/scripts/run-relay-load-model.mjs deleted file mode 100644 index 6915b8eb8c2..00000000000 --- a/cloud/dev/scripts/run-relay-load-model.mjs +++ /dev/null @@ -1,8 +0,0 @@ -import { assertSpreadModel, modeledRelayLoad } from './relay-load-model.mjs' - -const counts = process.argv.slice(2).length > 0 ? process.argv.slice(2).map(Number) : [4_000, 10_000] -for (const count of counts) { - const model = modeledRelayLoad(count) - assertSpreadModel(model) - console.log(JSON.stringify(model)) -} diff --git a/cloud/dev/scripts/run-relay-recovery-wave-gate.mjs b/cloud/dev/scripts/run-relay-recovery-wave-gate.mjs deleted file mode 100644 index fcc58d3474c..00000000000 --- a/cloud/dev/scripts/run-relay-recovery-wave-gate.mjs +++ /dev/null @@ -1,19 +0,0 @@ -import { readFileSync, statSync } from 'node:fs' -import { evaluateRecoveryWaveReport } from './relay-recovery-wave-gate.mjs' - -const args = process.argv.slice(2) -if (args.length !== 2 || args[0] !== '--report') { - process.stderr.write('usage: pnpm load:relay:recovery-gate -- --report \n') - process.exitCode = 1 -} else { - try { - if (statSync(args[1]).size > 1024 * 1024) throw new Error('report exceeds 1 MiB') - const report = JSON.parse(readFileSync(args[1], 'utf8')) - const result = evaluateRecoveryWaveReport(report) - process.stdout.write(`${JSON.stringify(result)}\n`) - if (result.status !== 'PASS') process.exitCode = 1 - } catch (error) { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - } -} diff --git a/cloud/dev/scripts/sanitize-relay-asia-admission-result.mjs b/cloud/dev/scripts/sanitize-relay-asia-admission-result.mjs deleted file mode 100644 index 1ac07841514..00000000000 --- a/cloud/dev/scripts/sanitize-relay-asia-admission-result.mjs +++ /dev/null @@ -1,98 +0,0 @@ -import { readFileSync } from 'node:fs' -import { fileURLToPath } from 'node:url' - -const MODES = new Set([ - 'inspect', 'initialize', 'verify', 'registered', 'register', - 'promote', 'recover-promotion', 'rollback' -]) -const STATES = new Set(['absent', 'existing-only', 'migration-only', 'general']) - -function validCellId(value) { - return typeof value === 'string' && value.length >= 1 && value.length <= 128 -} - -function cellIds(value, label) { - if ( - !Array.isArray(value) || value.length > 256 || - value.some((cellId) => !validCellId(cellId)) - ) { - throw new Error(`${label} is invalid`) - } - return [...value] -} - -function membership(value) { - if (value === undefined || value === null) return null - if (typeof value !== 'object' || Array.isArray(value)) { - throw new Error('membership is invalid') - } - return { - existingOnly: cellIds(value.existingOnly, 'existing-only membership'), - migrationOnly: cellIds(value.migrationOnly, 'migration-only membership'), - general: cellIds(value.general, 'general membership') - } -} - -function states(value) { - if (typeof value !== 'object' || Array.isArray(value)) throw new Error('states are invalid') - const entries = Object.entries(value) - if ( - entries.length === 0 || entries.length > 256 || - entries.some(([cellId, state]) => !validCellId(cellId) || !STATES.has(state)) - ) { - throw new Error('states are invalid') - } - return Object.fromEntries(entries) -} - -function optionalBoolean(value, label) { - if (value === undefined || value === null) return null - if (typeof value !== 'boolean') throw new Error(`${label} is invalid`) - return value -} - -export function sanitizeRelayAsiaAdmissionResult(input) { - if (!input || typeof input !== 'object' || Array.isArray(input)) { - throw new Error('admission result must be an object') - } - if (!MODES.has(input.mode)) throw new Error('mode is invalid') - if (!Number.isSafeInteger(input.generation) || input.generation < 0) { - throw new Error('generation is invalid') - } - if ( - input.membershipSha256 !== undefined && input.membershipSha256 !== null && - !/^[a-f0-9]{64}$/.test(input.membershipSha256) - ) throw new Error('membership SHA-256 is invalid') - const sanitizedMembership = membership(input.membership) - if ( - input.mode === 'inspect' && - (sanitizedMembership === null || !/^[a-f0-9]{64}$/.test(input.membershipSha256 ?? '')) - ) throw new Error('inspect membership evidence is incomplete') - const recovered = optionalBoolean(input.recovered, 'recovered') - const promoted = optionalBoolean(input.promoted, 'promoted') - if (input.mode === 'recover-promotion' && promoted === null) { - throw new Error('promotion recovery evidence is incomplete') - } - return { - v: 1, - mode: input.mode, - generation: input.generation, - states: states(input.states), - membership: sanitizedMembership, - membershipSha256: input.membershipSha256 ?? null, - recovered, - promoted - } -} - -function main() { - try { - const input = JSON.parse(readFileSync(0, 'utf8')) - process.stdout.write(`${JSON.stringify(sanitizeRelayAsiaAdmissionResult(input))}\n`) - } catch { - console.error('invalid Relay Asia admission result') - process.exitCode = 1 - } -} - -if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) main() diff --git a/cloud/dev/scripts/sanitize-relay-asia-admission-result.test.mjs b/cloud/dev/scripts/sanitize-relay-asia-admission-result.test.mjs deleted file mode 100644 index 2f1cf64a8cb..00000000000 --- a/cloud/dev/scripts/sanitize-relay-asia-admission-result.test.mjs +++ /dev/null @@ -1,120 +0,0 @@ -import assert from 'node:assert/strict' -import { spawnSync } from 'node:child_process' -import { fileURLToPath } from 'node:url' -import { test } from 'node:test' -import { sanitizeRelayAsiaAdmissionResult } from './sanitize-relay-asia-admission-result.mjs' - -const script = fileURLToPath(new URL('./sanitize-relay-asia-admission-result.mjs', import.meta.url)) -const expectedKeys = [ - 'v', 'mode', 'generation', 'states', 'membership', 'membershipSha256', 'recovered', 'promoted' -] - -test('preserves explicit false and emits only the admission evidence allowlist', () => { - const result = sanitizeRelayAsiaAdmissionResult({ - mode: 'verify', generation: 7, states: { 'staging-gce-c4': 'migration-only' }, - membership: { - existingOnly: [], migrationOnly: ['staging-gce-c4'], general: [], token: 'nested-secret' - }, - membershipSha256: 'a'.repeat(64), - recovered: false, promoted: false, - token: 'secret', credential: 'secret', userId: 'user', relayHostId: 'host', arbitrary: true - }) - - assert.deepEqual(Object.keys(result), expectedKeys) - assert.equal(result.recovered, false) - assert.equal(result.promoted, false) - assert.equal(JSON.stringify(result).includes('secret'), false) - assert.deepEqual(Object.keys(result.membership), ['existingOnly', 'migrationOnly', 'general']) - assert.equal('token' in result, false) - assert.equal('credential' in result, false) - assert.equal('userId' in result, false) - assert.equal('relayHostId' in result, false) - assert.equal('arbitrary' in result, false) -}) - -test('uses null for missing optional admission evidence', () => { - assert.deepEqual(sanitizeRelayAsiaAdmissionResult({ - mode: 'verify', generation: 0, states: { 'staging-gce-c4': 'absent' } - }), { - v: 1, - mode: 'verify', - generation: 0, - states: { 'staging-gce-c4': 'absent' }, - membership: null, - membershipSha256: null, - recovered: null, - promoted: null - }) -}) - -test('preserves valid historical cell IDs accepted by the director schema', () => { - const legacyCellId = 'legacy staging cell' - const result = sanitizeRelayAsiaAdmissionResult({ - mode: 'inspect', - generation: 0, - states: { [legacyCellId]: 'general' }, - membership: { existingOnly: [], migrationOnly: [], general: [legacyCellId] }, - membershipSha256: 'a'.repeat(64) - }) - - assert.deepEqual(result.states, { [legacyCellId]: 'general' }) - assert.deepEqual(result.membership.general, [legacyCellId]) -}) - -test('sanitizes stdin through the command-line entry point', () => { - const run = spawnSync(process.execPath, [script], { - input: JSON.stringify({ - mode: 'verify', generation: 0, states: { 'staging-gce-c4': 'absent' }, - recovered: false, token: 'secret' - }), - encoding: 'utf8' - }) - - assert.equal(run.status, 0, run.stderr) - const result = JSON.parse(run.stdout) - assert.deepEqual(Object.keys(result), expectedKeys) - assert.equal(result.recovered, false) - assert.equal(JSON.stringify(result).includes('secret'), false) -}) - -test('requires the evidence needed by every sequenced operation mode', () => { - const states = { 'staging-gce-c4': 'migration-only' } - const membership = { - existingOnly: [], migrationOnly: ['staging-gce-c4'], general: [] - } - const validByMode = { - inspect: { membership, membershipSha256: 'a'.repeat(64) }, - initialize: {}, - verify: {}, - registered: {}, - register: {}, - promote: {}, - 'recover-promotion': { promoted: false }, - rollback: {} - } - - for (const [mode, evidence] of Object.entries(validByMode)) { - assert.doesNotThrow(() => sanitizeRelayAsiaAdmissionResult({ - mode, generation: 1, states, ...evidence - })) - assert.throws(() => sanitizeRelayAsiaAdmissionResult({ mode, generation: 1, ...evidence })) - } - assert.throws(() => sanitizeRelayAsiaAdmissionResult({ - mode: 'inspect', generation: 1, states, membership - })) - assert.throws(() => sanitizeRelayAsiaAdmissionResult({ - mode: 'inspect', generation: 1, states, membershipSha256: 'a'.repeat(64) - })) - assert.throws(() => sanitizeRelayAsiaAdmissionResult({ - mode: 'recover-promotion', generation: 1, states - })) -}) - -test('rejects invalid stdin without echoing it', () => { - const run = spawnSync(process.execPath, [script], { input: '{secret', encoding: 'utf8' }) - - assert.equal(run.status, 1) - assert.equal(run.stdout, '') - assert.equal(run.stderr, 'invalid Relay Asia admission result\n') - assert.equal(run.stderr.includes('{secret'), false) -}) diff --git a/cloud/dev/scripts/smoke-relay.mjs b/cloud/dev/scripts/smoke-relay.mjs deleted file mode 100644 index 71689680a94..00000000000 --- a/cloud/dev/scripts/smoke-relay.mjs +++ /dev/null @@ -1,211 +0,0 @@ -import { - createHash, - createHmac, - createPrivateKey, - createPublicKey, - randomUUID -} from 'node:crypto' -import { readFileSync } from 'node:fs' -import { createRequire } from 'node:module' - -const requireFromRelay = createRequire(new URL('../../apps/relay/package.json', import.meta.url)) - -const relayUrl = process.argv[2]?.replace(/\/$/, '') -if (!relayUrl) throw new Error('usage: smoke-relay.mjs ') - -const health = await fetch(`${relayUrl}/health`) -if (!health.ok || (await health.json()).ok !== true) { - throw new Error(`relay health failed: ${health.status}`) -} - -const accessToken = process.env.ORCA_RELAY_SMOKE_ACCESS_TOKEN -const authUrl = process.env.ORCA_RELAY_SMOKE_AUTH_URL?.replace(/\/$/, '') -const signingKeyFile = process.env.ORCA_RELAY_SMOKE_SIGNING_KEY_FILE -if (!authUrl || (!accessToken && !signingKeyFile)) { - console.log('relay health smoke passed; provide auth URL plus an access token or operator signing-key file for a splice round-trip') - process.exit(0) -} - -const nacl = requireFromRelay('tweetnacl') -const WebSocket = requireFromRelay('ws') -const { buildHostProofMacInput, HOST_CHALLENGE_PLAINTEXT_DOMAIN } = await import( - requireFromRelay.resolve('@orca-cloud/relay-contract') -) - -function nextMessage(socket) { - return new Promise((resolve, reject) => { - const onMessage = (data) => { - cleanup() - try { - resolve(JSON.parse(data.toString())) - } catch (error) { - reject(error) - } - } - const onError = (error) => { - cleanup() - reject(error) - } - const onClose = (code, reason) => { - cleanup() - reject(new Error(`socket closed before message: ${code} ${reason.toString()}`)) - } - const cleanup = () => { - socket.off('message', onMessage) - socket.off('error', onError) - socket.off('close', onClose) - } - socket.once('message', onMessage) - socket.once('error', onError) - socket.once('close', onClose) - }) -} - -function opened(socket) { - return new Promise((resolve, reject) => { - socket.once('open', resolve) - socket.once('error', reject) - }) -} - -const hostKeys = nacl.box.keyPair() -const relayHostId = createHash('sha256') - .update(hostKeys.publicKey) - .digest('base64url') - .slice(0, 16) -let relayToken -if (accessToken) { - const tokenResponse = await fetch(`${authUrl}/v1/desktop/auth/relay-token`, { - method: 'POST', - headers: { - authorization: `Bearer ${accessToken}`, - 'content-type': 'application/json' - }, - body: JSON.stringify({ - relayHostId, - hostPublicKeyB64: Buffer.from(hostKeys.publicKey).toString('base64') - }) - }) - if (!tokenResponse.ok) throw new Error(`relay-token exchange failed: ${tokenResponse.status}`) - ;({ relayToken } = await tokenResponse.json()) -} else { - // This operator-only path proves the deployed data plane before desktop UI - // exists; possession of the auth signing key remains the security boundary. - const { SignJWT } = await import(requireFromRelay.resolve('jose')) - const privateKey = createPrivateKey(readFileSync(signingKeyFile, 'utf8')) - const keyId = createHash('sha256') - .update(createPublicKey(privateKey).export({ type: 'spki', format: 'der' })) - .digest('base64url') - .slice(0, 16) - relayToken = await new SignJWT({ - prof: 'staging-smoke-profile', - org: 'staging-smoke-org', - purpose: 'host-control', - relayHostId - }) - .setProtectedHeader({ alg: 'ES256', kid: keyId }) - .setIssuer(authUrl) - .setAudience('orca-relay') - .setSubject('staging-smoke-user') - .setIssuedAt() - .setExpirationTime('5m') - .sign(privateKey) -} -if (!relayToken) throw new Error('relay token was not produced') - -const wsOrigin = relayUrl.replace(/^http/, 'ws') -const control = new WebSocket(`${wsOrigin}/v1/host/control`, { - headers: { authorization: `Bearer ${relayToken}` }, - perMessageDeflate: false -}) -await opened(control) -control.send( - JSON.stringify({ - type: 'host-hello', - v: 1, - relayHostId, - assignmentEpoch: 1, - hostPublicKeyB64: Buffer.from(hostKeys.publicKey).toString('base64'), - appVersion: 'relay-smoke' - }) -) -const challenge = await nextMessage(control) -const plaintext = nacl.box.open( - Buffer.from(challenge.ciphertextB64, 'base64'), - Buffer.from(challenge.nonceB64, 'base64'), - Buffer.from(challenge.relayEphemeralPublicKeyB64, 'base64'), - hostKeys.secretKey -) -if (!plaintext) throw new Error('host proof challenge did not decrypt') -const domain = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`) -const transcriptLength = new DataView( - plaintext.buffer, - plaintext.byteOffset + domain.length, - 4 -).getUint32(0, false) -const transcriptStart = domain.length + 4 -const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength) -const secret = plaintext.slice(transcriptStart + transcriptLength) -const proofB64 = createHmac('sha256', secret) - .update(buildHostProofMacInput(transcript)) - .digest('base64') -control.send(JSON.stringify({ - type: 'host-challenge-ack', - challengeId: challenge.challengeId, - proofB64 -})) -const hostAck = await nextMessage(control) - -const relayDeviceId = `smoke-${randomUUID()}` -const invitePromise = nextMessage(control) -control.send(JSON.stringify({ type: 'invite-create', reqId: randomUUID(), relayDeviceId })) -const invite = await invitePromise - -const phone = new WebSocket(`${wsOrigin}/v1/connect/${relayHostId}`, { perMessageDeflate: false }) -await opened(phone) -const connectionPromise = nextMessage(control) -phone.send(JSON.stringify({ - type: 'relay-auth', - v: 1, - mode: 'connect', - credential: invite.inviteToken -})) -const connection = await connectionPromise -const data = new WebSocket(`${wsOrigin}/v1/host/data/${connection.connId}`, { - perMessageDeflate: false -}) -await opened(data) -const phoneHelloPromise = nextMessage(phone) -data.send(JSON.stringify({ - type: 'host-data-auth', - v: 1, - connTicket: connection.connTicket, - generation: hostAck.generation -})) -const phoneHello = await phoneHelloPromise -if (phoneHello.ok !== true) throw new Error('relay rejected smoke splice') - -const phoneEcho = new Promise((resolve, reject) => { - phone.once('message', (bytes, binary) => resolve({ bytes: Buffer.from(bytes), binary })) - phone.once('error', reject) -}) -data.send(Buffer.from([0x4f, 0x52, 0x43, 0x41])) -const echoed = await phoneEcho -if (!echoed.binary || !echoed.bytes.equals(Buffer.from([0x4f, 0x52, 0x43, 0x41]))) { - throw new Error('relay splice changed binary payload or opcode') -} - -const dataEcho = new Promise((resolve, reject) => { - data.once('message', (bytes, binary) => resolve({ bytes: Buffer.from(bytes), binary })) - data.once('error', reject) -}) -phone.send('orca-relay-smoke') -const returned = await dataEcho -if (returned.binary || returned.bytes.toString() !== 'orca-relay-smoke') { - throw new Error('relay splice changed text payload or opcode') -} - -phone.close() -data.close() -control.close() -console.log('relay authenticated splice smoke passed') diff --git a/cloud/dev/scripts/staging-relay-apply-guard.mjs b/cloud/dev/scripts/staging-relay-apply-guard.mjs deleted file mode 100644 index 7595a585769..00000000000 --- a/cloud/dev/scripts/staging-relay-apply-guard.mjs +++ /dev/null @@ -1,51 +0,0 @@ -import { execFileSync } from 'node:child_process' - -const PROJECT = 'onorca-cloud-staging' -const SQL_INSTANCE = 'orca-cloud-staging-auth-db' -const MIG_PREFIX = 'orca-cloud-staging-relay-gce-' - -function defaultGcloud(args) { - return execFileSync('gcloud', args, { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'pipe'] - }).trim() -} - -export function stagingRelayPowerState(gcloud = defaultGcloud) { - const sqlActivationPolicy = gcloud([ - 'sql', - 'instances', - 'describe', - SQL_INSTANCE, - '--project', - PROJECT, - '--format=value(settings.activationPolicy)' - ]) - const groups = JSON.parse( - gcloud([ - 'compute', - 'instance-groups', - 'managed', - 'list', - '--project', - PROJECT, - `--filter=name~'^${MIG_PREFIX}'`, - '--format=json(name,targetSize)' - ]) - ) - return { sqlActivationPolicy, groups } -} - -export function assertStagingRelayAwake(gcloud = defaultGcloud) { - const state = stagingRelayPowerState(gcloud) - const sleepingGroups = state.groups.filter((group) => Number(group.targetSize) !== 1) - if ( - state.sqlActivationPolicy !== 'ALWAYS' || - state.groups.length < 2 || - sleepingGroups.length > 0 - ) { - throw new Error( - 'Staging Relay is asleep or partially awake. Run the Power Relay Staging wake workflow before any staging Terraform apply.' - ) - } -} diff --git a/cloud/dev/scripts/staging-relay-apply-guard.test.mjs b/cloud/dev/scripts/staging-relay-apply-guard.test.mjs deleted file mode 100644 index 8035ffaf3a4..00000000000 --- a/cloud/dev/scripts/staging-relay-apply-guard.test.mjs +++ /dev/null @@ -1,30 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { - assertStagingRelayAwake, - stagingRelayPowerState -} from './staging-relay-apply-guard.mjs' - -function gcloud(policy, targetSizes) { - return (args) => - args[0] === 'sql' - ? policy - : JSON.stringify( - targetSizes.map((targetSize, index) => ({ - name: `orca-cloud-staging-relay-gce-c${index + 1}`, - targetSize - })) - ) -} - -test('reads and accepts a fully awake staging topology', () => { - const command = gcloud('ALWAYS', [1, 1, 1]) - assert.equal(stagingRelayPowerState(command).groups.length, 3) - assert.doesNotThrow(() => assertStagingRelayAwake(command)) -}) - -test('refuses Terraform apply while SQL or any staging cell is asleep', () => { - assert.throws(() => assertStagingRelayAwake(gcloud('NEVER', [0, 0, 0])), /wake workflow/) - assert.throws(() => assertStagingRelayAwake(gcloud('ALWAYS', [1, 0, 0])), /partially awake/) - assert.throws(() => assertStagingRelayAwake(gcloud('ALWAYS', [])), /partially awake/) -}) diff --git a/cloud/dev/scripts/terraform-root-partition.mjs b/cloud/dev/scripts/terraform-root-partition.mjs deleted file mode 100644 index 07ada7f624c..00000000000 --- a/cloud/dev/scripts/terraform-root-partition.mjs +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env node -import { readdirSync, readFileSync, writeFileSync } from 'node:fs' -import { fileURLToPath } from 'node:url' - -// Why: the relay Terraform root is being carved into foundation / relay / apps. Until every -// family is assigned to exactly one root per environment, a state move can silently orphan or -// double-manage a resource. This file is the single authority; the test pins it to the .tf files. - -export const ROOTS = ['foundation', 'apps', 'relay'] -export const ENVIRONMENTS = ['production', 'staging'] - -// Each root is its own Terraform directory; a family is declared in exactly the roots that own -// it somewhere, so the directory is the authority the fixture is checked against. Only the relay -// directory ships here, so only relay declarations can be read back; the partition still names -// all three roots because it is the authority for the whole carve. -export const ROOT_DIRECTORIES = { - relay: 'infra/terraform/' -} - -export const DECLARING_ROOTS = Object.keys(ROOT_DIRECTORIES) - -export const fixturePath = fileURLToPath( - new URL('../fixtures/terraform-root-partition/families.json', import.meta.url) -) - -function rootDirectory(root) { - const relative = ROOT_DIRECTORIES[root] - if (!relative) throw new Error(`unknown root ${root}`) - return fileURLToPath(new URL(`../../${relative}`, import.meta.url)) -} - -export function declaredFamilies(root = 'relay') { - const directory = rootDirectory(root) - const families = new Map() - for (const entry of readdirSync(directory).filter((name) => name.endsWith('.tf')).sort()) { - const text = readFileSync(`${directory}${entry}`, 'utf8') - for (const match of text.matchAll(/^resource "([a-z0-9_]+)" "([a-z0-9_]+)"/gm)) { - const address = `${match[1]}.${match[2]}` - if (families.has(address)) throw new Error(`duplicate declaration ${address}`) - families.set(address, entry) - } - } - return families -} - -// A family may be declared in two roots at once (the environment-conditional ten), so ownership -// is per (root, environment) while declaration is per root. -export function declaredRootFamilies(root) { - return new Set(declaredFamilies(root).keys()) -} - -export function ownedRootFamilies(partition, root) { - const families = new Set(partition[root]) - for (const [family, owners] of Object.entries(partition.env_conditional)) { - if (Object.values(owners).includes(root)) families.add(family) - } - return families -} - -export function readPartition(path = fixturePath) { - return JSON.parse(readFileSync(path, 'utf8')) -} - -// Family address for a state entry: strips [index] / ["key"] and ignores data sources. -export function familyOf(stateAddress) { - return stateAddress.replace(/\[.*$/, '') -} - -export function rootFor(partition, family, environment) { - if (!ENVIRONMENTS.includes(environment)) throw new Error(`unknown environment ${environment}`) - const conditional = partition.env_conditional[family] - if (conditional) return conditional[environment] - for (const root of ROOTS) if (partition[root].includes(family)) return root - return undefined -} - -export function expectedRootFamilies(partition, root, environment) { - const families = new Set(partition[root]) - for (const [family, owners] of Object.entries(partition.env_conditional)) { - if (owners[environment] === root) families.add(family) - } - return families -} - -// Compares `terraform state list` output for one root against the partition. -export function auditStateList(partition, root, environment, stateList) { - const expected = expectedRootFamilies(partition, root, environment) - const orphans = new Set(partition.state_orphans[environment] ?? []) - const unexpected = [] - const seen = new Set() - for (const line of stateList.split('\n').map((entry) => entry.trim()).filter(Boolean)) { - if (orphans.has(line)) continue - if (line.startsWith('data.')) continue - const family = familyOf(line) - seen.add(family) - if (!expected.has(family)) unexpected.push(line) - } - return { unexpected, seen: [...seen].sort() } -} - -if (process.argv[1] === fileURLToPath(import.meta.url)) { - const [command, root, environment, stateFile] = process.argv.slice(2) - const partition = readPartition() - if (command === 'audit') { - const { unexpected } = auditStateList(partition, root, environment, readFileSync(stateFile, 'utf8')) - if (unexpected.length > 0) { - process.stderr.write(`entries in ${root}/${environment} state outside its partition:\n`) - for (const line of unexpected) process.stderr.write(` ${line}\n`) - process.exitCode = 1 - } else { - process.stdout.write(`${root}/${environment}: state matches partition\n`) - } - } else if (command === 'list') { - for (const family of [...expectedRootFamilies(partition, root, environment)].sort()) { - process.stdout.write(`${family}\n`) - } - } else if (command === 'write-families') { - // Refreshes only the declared-family lists; ownership edits stay manual. - for (const name of DECLARING_ROOTS) { - for (const family of declaredRootFamilies(name)) { - if (rootFor(partition, family, 'production') === undefined) { - throw new Error(`unassigned family ${family}; add it to the fixture first`) - } - } - } - writeFileSync(fixturePath, `${JSON.stringify(partition, null, 2)}\n`) - } else { - process.stderr.write('usage: terraform-root-partition.mjs audit|list [state-list-file]\n') - process.exitCode = 2 - } -} diff --git a/cloud/dev/scripts/terraform-root-partition.test.mjs b/cloud/dev/scripts/terraform-root-partition.test.mjs deleted file mode 100644 index b92cb92366d..00000000000 --- a/cloud/dev/scripts/terraform-root-partition.test.mjs +++ /dev/null @@ -1,117 +0,0 @@ -import assert from 'node:assert/strict' -import { existsSync, readdirSync, readFileSync } from 'node:fs' -import { test } from 'node:test' -import { - DECLARING_ROOTS, - ENVIRONMENTS, - ROOTS, - auditStateList, - declaredFamilies, - declaredRootFamilies, - expectedRootFamilies, - ownedRootFamilies, - readPartition, - rootFor -} from './terraform-root-partition.mjs' - -const partition = readPartition() -const declared = new Map(DECLARING_ROOTS.flatMap((root) => [...declaredFamilies(root)].map(([family, file]) => [family, `${root}/${file}`]))) - -test('every declared resource family is assigned to exactly one root per environment', () => { - for (const family of declared.keys()) { - for (const environment of ENVIRONMENTS) { - const owners = ROOTS.filter((root) => expectedRootFamilies(partition, root, environment).has(family)) - assert.deepEqual(owners, [rootFor(partition, family, environment)], `${family} in ${environment}`) - } - } -}) - -// Only the relay directory ships here, so only the families the partition assigns to a declaring -// root can be checked back against a .tf file. The full listing is still checked for duplicates. -test('the partition names no family that is not declared', () => { - const listed = [ - ...ROOTS.flatMap((root) => partition[root]), - ...Object.keys(partition.env_conditional) - ] - for (const root of DECLARING_ROOTS) { - for (const family of ownedRootFamilies(partition, root)) { - assert.ok(declared.has(family), `${family} is not declared`) - } - } - assert.equal(new Set(listed).size, listed.length, 'a family is listed twice') -}) - -test('environment-conditional families are owned by different roots per environment', () => { - for (const [family, owners] of Object.entries(partition.env_conditional)) { - assert.deepEqual(Object.keys(owners).sort(), [...ENVIRONMENTS].sort(), family) - assert.notEqual(owners.production, owners.staging, `${family} is not really conditional`) - } -}) - -// Why: this is the census that survives the carve. Filename prefixes stopped meaning anything -// once each root became its own directory, so ownership is checked against the directory that -// declares the family. A root declares exactly what it owns in at least one environment; the -// environment-conditional ten are therefore declared twice, once per complementary count. -test('each root declares exactly the families it owns in some environment', () => { - for (const root of DECLARING_ROOTS) { - assert.deepEqual( - [...declaredRootFamilies(root)].sort(), - [...ownedRootFamilies(partition, root)].sort(), - root - ) - } -}) - -// Why: the removed blocks were a guard for the config-first window between the carve and the two -// state surgeries. Both are done; a removed block that resurfaces would silently turn a stray apply -// from "destroy" into "forget" and hide a real ownership mistake. -test('the carved families are gone from the relay root and nothing is guarded by a removed block', () => { - const relay = declaredRootFamilies('relay') - for (const family of [...partition.foundation, ...partition.apps]) { - assert.ok(!relay.has(family), `${family} is still declared in the relay root`) - } - assert.ok(!existsSync(new URL('../../infra/terraform/relay-root-carve-removed.tf', import.meta.url))) - for (const file of readdirSync(new URL('../../infra/terraform/', import.meta.url))) { - if (!file.endsWith('.tf')) continue - const source = readFileSync(new URL(`../../infra/terraform/${file}`, import.meta.url), 'utf8') - assert.doesNotMatch(source, /^removed \{/m, `${file} declares a removed block`) - } -}) - -// Why: every binding on the shared deploy account follows that account (relay in production, -// apps in staging). Letting one drift back into foundation recreates the dependency cycle the -// split exists to remove: foundation would need the account while relay needs the pool. -test('foundation owns only what both other roots must be able to bootstrap against', () => { - assert.deepEqual(partition.foundation, [ - 'google_artifact_registry_repository.api', - 'google_iam_workload_identity_pool.github', - 'google_project_service.required', - 'google_project_service.sqladmin', - 'google_service_account.runtime', - 'google_sql_database_instance.auth', - 'google_storage_bucket_iam_member.cloud_sql_rollout_lease', - 'google_storage_bucket_iam_member.cloud_sql_rollout_lease_bucket_reader' - ]) -}) - -// Why: the two staging orphans were cleared by the runbook; the allowance is empty so a stray -// entry is reported instead of silently tolerated again. -test('audit reports entries outside the partition and no longer tolerates the staging orphans', () => { - const stateList = [ - 'google_project_service.required["run.googleapis.com"]', - 'google_secret_manager_secret_iam_member.runtime_artifact_write_secret_accessor[0]', - 'data.google_compute_image.relay_gce_cos[0]', - 'google_cloud_run_v2_service.relay' - ].join('\n') - assert.deepEqual(partition.state_orphans, { staging: [], production: [] }) - const foundation = auditStateList(partition, 'foundation', 'staging', stateList) - assert.deepEqual(foundation.unexpected, [ - 'google_secret_manager_secret_iam_member.runtime_artifact_write_secret_accessor[0]', - 'google_cloud_run_v2_service.relay' - ]) - const relay = auditStateList(partition, 'relay', 'staging', stateList) - assert.deepEqual(relay.unexpected, [ - 'google_project_service.required["run.googleapis.com"]', - 'google_secret_manager_secret_iam_member.runtime_artifact_write_secret_accessor[0]' - ]) -}) diff --git a/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs b/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs deleted file mode 100644 index 3b73383730f..00000000000 --- a/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs +++ /dev/null @@ -1,345 +0,0 @@ -import { readFileSync } from 'node:fs' -import { fileURLToPath } from 'node:url' - -// The root region's cells carry no region label or line, and a pool line only off its default. -const PRIMARY_REGION = 'us-central1' -const PRIMARY_DATABASE_POOL_MAX = '10' -// Mirrors local.relay_gce_topology in infra/terraform/relay-gce-cells.tf, which Terraform -// cannot export to JS; the census test below the validator equates the two by reading the -// .tf source, so this pair and the topology `check` assert cannot drift apart. -export const RELAY_CELL_BACKEND_TIMEOUT_SECONDS = 86_400 -export const RELAY_CELL_CONNECTION_DRAIN_SECONDS = 60 -// Not a topology local: the default of var.relay_gce_cell_log_sample_rate, which no -// environment overrides. The same census test equates it with variables.tf. -export const RELAY_CELL_LOG_SAMPLE_RATE = 1 -const CELL_SHAPES = { - production: { - domain: 'relay.onorca.dev', - project: 'onorca-cloud', - databasePoolMax: { 'asia-east2': '16', 'us-central1': '10' }, - cells: { - 'production-gce-c27': 'asia-east2-a', - 'production-gce-c28': 'asia-east2-b', - 'production-gce-c29': 'asia-east2-c', - 'production-gce-c30': 'asia-east2-a', - 'production-gce-c31': 'asia-east2-b', - 'production-gce-c32': 'us-central1-a', - 'production-gce-c33': 'us-central1-b', - 'production-gce-c34': 'asia-east2-c', - 'production-gce-c35': 'asia-east2-a' - }, - waves: [ - ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - ['production-gce-c30'], - ['production-gce-c31'], - // Declared together, so they plan together: a lone C32 plan would hit C33's missing template. - ['production-gce-c32', 'production-gce-c33'], - ['production-gce-c34'], - ['production-gce-c35'] - ] - }, - staging: { - domain: 'relay-staging.onorca.dev', - project: 'onorca-cloud-staging', - databasePoolMax: { 'asia-east2': '10' }, - cells: { 'staging-gce-c4': 'asia-east2-a' }, - waves: [['staging-gce-c4']] - } -} - -// Every reviewed wave is single-region, so its first cell's zone names the region. -function waveRegion(environment, cells) { - const zone = CELL_SHAPES[environment]?.cells[cells[0]] - if (!zone) throw new Error('--cell-ids must be the exact reviewed Asia topology set') - return zone.slice(0, zone.lastIndexOf('-')) -} - -export function parseRelayAsiaTopologyPlanArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of ['plan-json', 'environment', 'cell-ids', 'region', 'image']) { - if (!values[key]) throw new Error(`missing --${key}`) - } - if (!(values.environment in CELL_SHAPES)) throw new Error('--environment is invalid') - const cells = values['cell-ids'].split(',').map((value) => value.trim()).filter(Boolean) - const sorted = JSON.stringify([...cells].sort()) - if ( - new Set(cells).size !== cells.length || - !CELL_SHAPES[values.environment].waves.some((wave) => JSON.stringify([...wave].sort()) === sorted) - ) { - throw new Error('--cell-ids must be the exact reviewed Asia topology set') - } - const region = waveRegion(values.environment, cells) - if (values.region !== region) throw new Error(`--region must be ${region}`) - const expectedImagePrefix = `us-central1-docker.pkg.dev/${CELL_SHAPES[values.environment].project}/orca-cloud/relay@sha256:` - if (!values.image.startsWith(expectedImagePrefix) || !/sha256:[a-f0-9]{64}$/.test(values.image)) { - throw new Error('--image must be the environment Relay image pinned by digest') - } - return { planJson: values['plan-json'], environment: values.environment, cells, image: values.image } -} - -function address(resource, key) { - return `${resource}[${JSON.stringify(key)}]` -} - -function actions(change) { - return change.change?.actions ?? [] -} - -function sameActions(change, expected) { - return JSON.stringify(actions(change)) === JSON.stringify(expected) -} - -function startupValue(script, name) { - return new RegExp(`printf '${name}=%s\\\\n' '([^']+)'`).exec(script)?.[1] -} - -function relayGceName(environment) { - return environment === 'production' ? 'orca-cloud-relay-gce' : 'orca-cloud-staging-relay-gce' -} - -function unknownOrMatches(value, predicate) { - return value === undefined || value === null || predicate(String(value)) -} - -function requireCellTemplate(change, config, cellId, region) { - const after = change.change.after - const script = after?.metadata_startup_script ?? '' - const regionLine = region === PRIMARY_REGION ? undefined : region - const pool = CELL_SHAPES[config.environment].databasePoolMax[region] - if ( - after?.machine_type !== 'e2-standard-4' || - after?.labels?.['orca-relay-cell'] !== cellId || - after?.labels?.['orca-relay-region'] !== regionLine || - !unknownOrMatches( - after?.network_interface?.[0]?.subnetwork, - (value) => value.includes(`/regions/${region}/subnetworks/`) - ) || - (after?.network_interface?.[0]?.access_config?.length ?? 0) !== 0 || - startupValue(script, 'ORCA_RELAY_REGION') !== regionLine || - startupValue(script, 'ORCA_RELAY_CELL_CAPACITY') !== '6000' || - startupValue(script, 'ORCA_RELAY_DATABASE_POOL_MAX') !== - (regionLine === undefined && pool === PRIMARY_DATABASE_POOL_MAX ? undefined : pool) || - startupValue(script, 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP') !== '3000' || - startupValue(script, 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND') !== '60' || - startupValue(script, 'ORCA_RELAY_IMAGE_DIGEST') !== config.image.split('@')[1] || - !script.includes(`docker pull '${config.image}'`) || - !script.trimEnd().includes(`'${config.image}'`) - ) throw new Error(`${change.address} does not have the reviewed Asia cell shape`) -} - -function requireCellManager(change, config, cellId) { - const after = change.change.after - const hostname = cellId.split('-').at(-1) - const version = after?.version?.[0] - if ( - after?.zone !== CELL_SHAPES[config.environment].cells[cellId] || - after?.target_size !== 1 || - after?.version?.length !== 1 || - version?.name !== 'primary' || - !unknownOrMatches(version?.instance_template, (value) => - value.includes( - `/global/instanceTemplates/${relayGceName(config.environment)}-${hostname}-` - )) || - after?.update_policy?.[0]?.replacement_method !== 'RECREATE' || - after?.update_policy?.[0]?.max_surge_fixed !== 0 || - after?.update_policy?.[0]?.max_unavailable_fixed !== 1 - ) throw new Error(`${change.address} does not have the reviewed fixed-one Asia MIG shape`) -} - -function requireCellBackend(change, config, cellId) { - const after = change.change.after - const backend = after?.backend?.[0] - const zone = CELL_SHAPES[config.environment].cells[cellId] - const hostname = cellId.split('-').at(-1) - const name = `${relayGceName(config.environment)}-${hostname}` - if ( - after?.timeout_sec !== RELAY_CELL_BACKEND_TIMEOUT_SECONDS || - after?.connection_draining_timeout_sec !== RELAY_CELL_CONNECTION_DRAIN_SECONDS || - after?.load_balancing_scheme !== 'EXTERNAL_MANAGED' || - after?.protocol !== 'HTTP' || - after?.port_name !== 'relay' || - after?.session_affinity !== 'NONE' || - after?.health_checks?.length !== 1 || - !unknownOrMatches(after.health_checks[0], (value) => - value.endsWith(`/global/healthChecks/${relayGceName(config.environment)}-ready`)) || - after?.backend?.length !== 1 || - backend?.balancing_mode !== 'UTILIZATION' || - backend?.max_utilization !== 0.8 || - backend?.capacity_scaler !== 1 || - !unknownOrMatches(backend?.group, (value) => - value.endsWith(`/zones/${zone}/instanceGroups/${name}`)) - ) throw new Error(`${change.address} does not have the reviewed Asia backend shape`) -} - -function requireNetworkResource(change, config, region) { - const after = change.change.after - const networkSuffix = `/global/networks/${relayGceName(config.environment)}` - if (after?.region !== region) throw new Error(`${change.address} is outside ${region}`) - if ( - change.address.startsWith('google_compute_subnetwork.') && - (after.ip_cidr_range !== '10.42.1.0/24' || - after.private_ip_google_access !== true || - after.stack_type !== 'IPV4_ONLY' || - !unknownOrMatches(after.network, (value) => value.endsWith(networkSuffix))) - ) throw new Error(`${change.address} does not have the reviewed Asia subnet shape`) - if ( - change.address.startsWith('google_compute_router.') && - !unknownOrMatches(after.network, (value) => value.endsWith(networkSuffix)) - ) throw new Error(`${change.address} does not have the reviewed Asia router shape`) - if ( - change.address.startsWith('google_compute_router_nat.') && - (after.nat_ip_allocate_option !== 'AUTO_ONLY' || - after.source_subnetwork_ip_ranges_to_nat !== 'LIST_OF_SUBNETWORKS' || - after.subnetwork?.length !== 1 || - !unknownOrMatches(after.subnetwork[0]?.name, (value) => - value.endsWith( - `/regions/${region}/subnetworks/${relayGceName(config.environment)}-${region}` - )) || - JSON.stringify(after.subnetwork[0]?.source_ip_ranges_to_nat) !== - JSON.stringify(['ALL_IP_RANGES'])) - ) throw new Error(`${change.address} does not have the reviewed Asia NAT shape`) -} - -function canonical(value) { - if (!Array.isArray(value)) return JSON.stringify(value ?? []) - return JSON.stringify([...value].sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)))) -} - -function normalizeDescription(value) { - return { ...value, description: value.description ?? '' } -} - -function normalizeMatcher(value) { - const apiPrefix = 'https://www.googleapis.com/compute/v1/' - const defaultService = value.default_service - return { - ...normalizeDescription(value), - default_service: typeof defaultService === 'string' && defaultService.startsWith(apiPrefix) - ? defaultService.slice(apiPrefix.length) - : defaultService - } -} - -function requireUrlMap(change, config) { - const before = change.change.before ?? {} - const after = change.change.after ?? {} - const permitted = new Set(['host_rule', 'path_matcher', 'fingerprint']) - const changed = new Set([...Object.keys(before), ...Object.keys(after)].filter( - (key) => JSON.stringify(before[key]) !== JSON.stringify(after[key]) - )) - if ([...changed].some((key) => !permitted.has(key))) { - throw new Error('shared URL map changes outside host routing') - } - const newHosts = new Set() - const newMatchers = new Set() - for (const cellId of config.cells) { - const hostname = cellId.split('-').at(-1) - const host = `${hostname}.${CELL_SHAPES[config.environment].domain}` - const hostRules = after.host_rule?.filter( - (rule) => - rule.hosts?.length === 1 && - rule.hosts[0] === host && - rule.path_matcher === `cell-${hostname}` - ) ?? [] - if (hostRules.length !== 1) { - throw new Error(`shared URL map has no exact host for ${cellId}`) - } - const matchers = after.path_matcher?.filter( - (matcher) => - matcher.name === `cell-${hostname}` && - unknownOrMatches(matcher.default_service, (value) => - value.endsWith( - `/global/backendServices/${relayGceName(config.environment)}-${hostname}` - )) - ) ?? [] - if (matchers.length !== 1) { - throw new Error(`shared URL map has no exact backend route for ${cellId}`) - } - newHosts.add(host) - newMatchers.add(`cell-${hostname}`) - } - const preservedHostRules = (after.host_rule ?? []).filter( - (rule) => !(rule.hosts?.length === 1 && newHosts.has(rule.hosts[0])) - ) - const preservedMatchers = (after.path_matcher ?? []).filter( - (matcher) => !newMatchers.has(matcher.name) - ) - if ( - sameActions(change, ['update']) && - canonical(preservedHostRules.map(normalizeDescription)) !== - canonical((before.host_rule ?? []).map(normalizeDescription)) || - sameActions(change, ['update']) && - canonical(preservedMatchers.map(normalizeMatcher)) !== - canonical((before.path_matcher ?? []).map(normalizeMatcher)) - ) { - throw new Error('shared URL map does not preserve every existing exact route') - } -} - -export function validateRelayAsiaTopologyPlan(plan, config) { - if (!Array.isArray(plan.resource_changes)) throw new Error('Terraform plan has no resource changes') - const region = waveRegion(config.environment, config.cells) - // The root region's network predates every cell, so only an additional region's is planned. - const required = new Map(region === PRIMARY_REGION ? [] : [ - [address('google_compute_subnetwork.relay_gce_additional', region), [['create'], ['no-op']]], - [address('google_compute_router.relay_gce_additional', region), [['create'], ['no-op']]], - [address('google_compute_router_nat.relay_gce_additional', region), [['create'], ['no-op']]] - ]) - required.set('google_compute_url_map.relay_gce[0]', [['update'], ['no-op']]) - for (const cellId of config.cells) { - required.set(address('google_compute_instance_template.relay_gce_cell', cellId), [['create'], ['no-op']]) - required.set(address('google_compute_instance_group_manager.relay_gce_cell', cellId), [['create'], ['no-op']]) - required.set(address('google_compute_backend_service.relay_gce_cell', cellId), [['create'], ['no-op']]) - } - const byAddress = new Map(plan.resource_changes.map((change) => [change.address, change])) - for (const [resourceAddress, allowedActions] of required) { - const change = byAddress.get(resourceAddress) - if (!change || !allowedActions.some((expected) => sameActions(change, expected))) { - throw new Error(`${resourceAddress} is absent or has an unreviewed topology action`) - } - const cellId = config.cells.find((candidate) => resourceAddress.endsWith(`[${JSON.stringify(candidate)}]`)) - if (resourceAddress.startsWith('google_compute_instance_template.') && cellId) { - requireCellTemplate(change, config, cellId, region) - } else if (resourceAddress.startsWith('google_compute_instance_group_manager.') && cellId) { - requireCellManager(change, config, cellId) - } else if (resourceAddress.startsWith('google_compute_backend_service.') && cellId) { - requireCellBackend(change, config, cellId) - } else if ( - resourceAddress.startsWith('google_compute_subnetwork.') || - resourceAddress.startsWith('google_compute_router.') || - resourceAddress.startsWith('google_compute_router_nat.') - ) { - requireNetworkResource(change, config, region) - } else if (resourceAddress === 'google_compute_url_map.relay_gce[0]') { - requireUrlMap(change, config) - } - } - const changes = plan.resource_changes.filter((change) => !actions(change).every( - (action) => action === 'no-op' || action === 'read' - )) - for (const change of changes) { - const cellId = /^google_compute_(?:instance_template|instance_group_manager|backend_service)\.relay_gce_cell\["([^"]+)"\]$/ - .exec(change.address)?.[1] - // Usually a stale committed image; the workflow overlays live images so this stays empty. - if (cellId && !config.cells.includes(cellId)) { - throw new Error(`${change.address} changes a live cell outside the planned wave`) - } - const allowedActions = required.get(change.address) - if (!allowedActions || !allowedActions.some((expected) => sameActions(change, expected))) { - throw new Error(`${change.address} has an unreviewed topology action`) - } - } - return { environment: config.environment, cells: config.cells, changes: changes.length } -} - -if (process.argv[1] === fileURLToPath(import.meta.url)) { - const config = parseRelayAsiaTopologyPlanArguments(process.argv.slice(2)) - const plan = JSON.parse(readFileSync(config.planJson, 'utf8')) - console.log(JSON.stringify(validateRelayAsiaTopologyPlan(plan, config))) -} diff --git a/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs b/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs deleted file mode 100644 index 0a939da7fbb..00000000000 --- a/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs +++ /dev/null @@ -1,535 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import { test } from 'node:test' -import { - RELAY_CELL_BACKEND_TIMEOUT_SECONDS, - RELAY_CELL_CONNECTION_DRAIN_SECONDS, - RELAY_CELL_LOG_SAMPLE_RATE, - parseRelayAsiaTopologyPlanArguments, - validateRelayAsiaTopologyPlan -} from './validate-relay-asia-topology-plan.mjs' - -const image = `us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:${'a'.repeat(64)}` -const config = { environment: 'staging', cells: ['staging-gce-c4'], image } -const create = (address, after = {}) => ({ address, change: { actions: ['create'], after } }) -const script = [ - `printf 'ORCA_RELAY_REGION=%s\\n' 'asia-east2'`, - `printf 'ORCA_RELAY_CELL_CAPACITY=%s\\n' '6000'`, - `printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '10'`, - `printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`, - `printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`, - `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${image.split('@')[1]}'`, - `docker pull '${image}'`, - `'${image}'` -].join('\n') -const resources = [ - create('google_compute_subnetwork.relay_gce_additional["asia-east2"]', { - region: 'asia-east2', ip_cidr_range: '10.42.1.0/24', private_ip_google_access: true, - stack_type: 'IPV4_ONLY', - network: 'projects/p/global/networks/orca-cloud-staging-relay-gce' - }), - create('google_compute_router.relay_gce_additional["asia-east2"]', { - region: 'asia-east2', network: 'projects/p/global/networks/orca-cloud-staging-relay-gce' - }), - create('google_compute_router_nat.relay_gce_additional["asia-east2"]', { - region: 'asia-east2', nat_ip_allocate_option: 'AUTO_ONLY', - source_subnetwork_ip_ranges_to_nat: 'LIST_OF_SUBNETWORKS', - subnetwork: [{ - name: 'projects/p/regions/asia-east2/subnetworks/orca-cloud-staging-relay-gce-asia-east2', - source_ip_ranges_to_nat: ['ALL_IP_RANGES'] - }] - }), - create('google_compute_instance_template.relay_gce_cell["staging-gce-c4"]', { - machine_type: 'e2-standard-4', - labels: { 'orca-relay-cell': 'staging-gce-c4', 'orca-relay-region': 'asia-east2' }, - network_interface: [{ - subnetwork: 'projects/p/regions/asia-east2/subnetworks/relay', access_config: [] - }], - metadata_startup_script: script - }), - create('google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]', { - zone: 'asia-east2-a', target_size: 1, - version: [{ - name: 'primary', - instance_template: 'projects/p/global/instanceTemplates/orca-cloud-staging-relay-gce-c4-abc' - }], - update_policy: [{ replacement_method: 'RECREATE', max_surge_fixed: 0, max_unavailable_fixed: 1 }] - }), - create('google_compute_backend_service.relay_gce_cell["staging-gce-c4"]', { - timeout_sec: RELAY_CELL_BACKEND_TIMEOUT_SECONDS, - connection_draining_timeout_sec: RELAY_CELL_CONNECTION_DRAIN_SECONDS, - load_balancing_scheme: 'EXTERNAL_MANAGED', protocol: 'HTTP', port_name: 'relay', - session_affinity: 'NONE', - health_checks: ['projects/p/global/healthChecks/orca-cloud-staging-relay-gce-ready'], - backend: [{ - balancing_mode: 'UTILIZATION', max_utilization: 0.8, capacity_scaler: 1, - group: 'projects/p/zones/asia-east2-a/instanceGroups/orca-cloud-staging-relay-gce-c4' - }] - }), - { - address: 'google_compute_url_map.relay_gce[0]', - change: { - actions: ['update'], - before: { host_rule: [], path_matcher: [], fingerprint: 'old' }, - after: { - host_rule: [{ - hosts: ['c4.relay-staging.onorca.dev'], path_matcher: 'cell-c4' - }], - path_matcher: [{ - name: 'cell-c4', - default_service: 'projects/p/global/backendServices/orca-cloud-staging-relay-gce-c4' - }], - fingerprint: null - } - } - } -] - -test('accepts the exact additive staging Asia topology', () => { - assert.deepEqual(validateRelayAsiaTopologyPlan({ resource_changes: resources }, config), { - environment: 'staging', cells: ['staging-gce-c4'], changes: 7 - }) -}) - -const productionImage = image.replace('onorca-cloud-staging/', 'onorca-cloud/') -const productionConfig = { - environment: 'production', cells: ['production-gce-c30'], image: productionImage -} - -// A later cell joins a live Asia region: the network is a no-op and the existing C27 route is preserved. -function productionWavePlan(hostname = 'c30', zone = 'asia-east2-a') { - const plan = JSON.parse(JSON.stringify(resources) - .replaceAll('onorca-cloud-staging/', 'onorca-cloud/') - .replaceAll('orca-cloud-staging-relay-gce', 'orca-cloud-relay-gce') - .replaceAll('staging-gce-c4', `production-gce-${hostname}`) - .replaceAll('relay-gce-c4', `relay-gce-${hostname}`) - .replaceAll('cell-c4', `cell-${hostname}`) - .replaceAll('c4.relay-staging.onorca.dev', `${hostname}.relay.onorca.dev`) - .replaceAll('asia-east2-a', zone) - .replaceAll("'10'", "'16'")) - for (const network of plan.slice(0, 3)) { - network.change.actions = ['no-op'] - network.change.before = structuredClone(network.change.after) - } - const existingHost = { hosts: ['c27.relay.onorca.dev'], path_matcher: 'cell-c27' } - const existingMatcher = { - name: 'cell-c27', - default_service: 'projects/p/global/backendServices/orca-cloud-relay-gce-c27' - } - const urlMap = plan.at(-1).change - urlMap.before = { host_rule: [existingHost], path_matcher: [existingMatcher], fingerprint: 'old' } - urlMap.after.host_rule.unshift(structuredClone(existingHost)) - urlMap.after.path_matcher.unshift(structuredClone(existingMatcher)) - return plan -} - -test('accepts the additive production C30 wave at the 16-connection Asia pool', () => { - assert.deepEqual( - validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan() }, productionConfig), - { environment: 'production', cells: ['production-gce-c30'], changes: 4 } - ) - const staleShape = productionWavePlan() - staleShape[3].change.after.metadata_startup_script = - staleShape[3].change.after.metadata_startup_script.replace("'16'", "'10'") - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: staleShape }, productionConfig), - /reviewed Asia cell shape/ - ) - const wrongZone = productionWavePlan() - wrongZone[4].change.after.zone = 'asia-east2-b' - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: wrongZone }, productionConfig), - /fixed-one Asia MIG shape/ - ) - const liveCellTouched = productionWavePlan() - liveCellTouched.push(create('google_compute_instance_template.relay_gce_cell["production-gce-c27"]')) - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: liveCellTouched }, productionConfig), - /outside the planned wave/ - ) -}) - -test('accepts the additive production C31 wave only in asia-east2-b', () => { - const c31Config = { ...productionConfig, cells: ['production-gce-c31'] } - assert.deepEqual( - validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c31', 'asia-east2-b') }, c31Config), - { environment: 'production', cells: ['production-gce-c31'], changes: 4 } - ) - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c31') }, c31Config), - /fixed-one Asia MIG shape/ - ) -}) - -test('accepts the additive production C34 spare wave only in asia-east2-c', () => { - const c34Config = { ...productionConfig, cells: ['production-gce-c34'] } - assert.deepEqual( - validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c34', 'asia-east2-c') }, c34Config), - { environment: 'production', cells: ['production-gce-c34'], changes: 4 } - ) - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c34') }, c34Config), - /fixed-one Asia MIG shape/ - ) -}) - -test('accepts the additive production C35 spare wave only in asia-east2-a', () => { - const c35Config = { ...productionConfig, cells: ['production-gce-c35'] } - assert.deepEqual( - validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c35', 'asia-east2-a') }, c35Config), - { environment: 'production', cells: ['production-gce-c35'], changes: 4 } - ) - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c35', 'asia-east2-c') }, c35Config), - /fixed-one Asia MIG shape/ - ) -}) - -// A US cell joins the root region: no additional-region network, no region label or line, and -// the default pool emits no line, exactly as the startup template renders a root-region cell. -function usCellPlan(hostname, zone) { - const plan = productionWavePlan(hostname, zone).slice(3) - const template = plan[0].change.after - template.labels = { 'orca-relay-cell': `production-gce-${hostname}` } - template.network_interface[0].subnetwork = 'projects/p/regions/us-central1/subnetworks/relay' - template.metadata_startup_script = template.metadata_startup_script.split('\n') - .filter((line) => !/ORCA_RELAY_(?:REGION|DATABASE_POOL_MAX)=/.test(line)) - .join('\n') - return plan -} - -// C32 and C33 plan as one wave: both cells' resources and one URL map adding both hosts. -function usWavePlan(c33Zone = 'us-central1-b') { - const c32 = usCellPlan('c32', 'us-central1-a') - const c33 = usCellPlan('c33', c33Zone) - const urlMap = c32.at(-1) - const added = c33.at(-1).change.after - urlMap.change.after.host_rule.push(added.host_rule.at(-1)) - urlMap.change.after.path_matcher.push(added.path_matcher.at(-1)) - return [...c32.slice(0, -1), ...c33.slice(0, -1), urlMap] -} - -const usConfig = { ...productionConfig, cells: ['production-gce-c32', 'production-gce-c33'] } - -test('accepts the additive US C32+C33 wave at the root-region shape', () => { - assert.deepEqual( - validateRelayAsiaTopologyPlan({ resource_changes: usWavePlan() }, usConfig), - { environment: 'production', cells: ['production-gce-c32', 'production-gce-c33'], changes: 7 } - ) - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: usWavePlan('us-central1-a') }, usConfig), - /fixed-one Asia MIG shape/ - ) - // A plan missing either cell is not the reviewed wave. - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: usCellPlan('c32', 'us-central1-a') }, usConfig), - /no exact host for production-gce-c33|absent or has an unreviewed topology action/ - ) - const mutate = (index, edit) => { - const plan = usWavePlan() - edit(plan[index].change.after) - return plan - } - const asiaLine = "printf 'ORCA_RELAY_REGION=%s\\n' 'asia-east2'" - for (const index of [0, 3]) { - for (const [label, plan] of [ - ['region label', mutate(index, (after) => { after.labels['orca-relay-region'] = 'us-central1' })], - ['region line', mutate(index, (after) => { - after.metadata_startup_script = `${asiaLine}\n${after.metadata_startup_script}` - })], - ['Asia pool', mutate(index, (after) => { - after.metadata_startup_script += "\nprintf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '16'" - })], - ['Asia subnet', mutate(index, (after) => { - after.network_interface[0].subnetwork = 'projects/p/regions/asia-east2/subnetworks/relay' - })] - ]) { - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, usConfig), - /reviewed Asia cell shape/, - `${index} ${label}` - ) - } - } - // A US wave plans no additional-region network, so creating one is an unreviewed change. - const withNetwork = [...productionWavePlan('c32', 'us-central1-a').slice(0, 1), ...usWavePlan()] - withNetwork[0].change.actions = ['create'] - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: withNetwork }, usConfig), - /unreviewed topology action/ - ) -}) - -// The URL map pulls every cell's backend, MIG and template into a targeted plan. -const liveCellResources = ['instance_template', 'instance_group_manager', 'backend_service'] - .flatMap((kind) => ['production-gce-c1', 'production-gce-c27', 'production-gce-c28', 'production-gce-c29'] - .map((cellId) => `google_compute_${kind}.relay_gce_cell["${cellId}"]`)) - -test('accepts live cells the URL map pulls in only while they stay unchanged', () => { - const plan = productionWavePlan() - for (const address of liveCellResources) plan.push({ address, change: { actions: ['no-op'] } }) - assert.equal( - validateRelayAsiaTopologyPlan({ resource_changes: plan }, productionConfig).changes, - 4 - ) - for (const address of liveCellResources) { - for (const action of [['update'], ['delete'], ['create', 'delete'], ['delete', 'create']]) { - const drifted = productionWavePlan() - drifted.push({ address, change: { actions: action } }) - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: drifted }, productionConfig), - new RegExp(`${address.replaceAll(/[.[\]]/g, '\\$&')} changes a live cell outside the planned wave`) - ) - } - } -}) - -test('accepts only a reviewed Asia topology wave', () => { - const argv = (environment, cellIds, planImage, region = 'asia-east2') => [ - '--plan-json', 'plan.json', '--environment', environment, '--cell-ids', cellIds, - '--region', region, '--image', planImage - ] - for (const cellIds of [ - 'production-gce-c27,production-gce-c28,production-gce-c29', - 'production-gce-c29,production-gce-c27,production-gce-c28', - 'production-gce-c30', - 'production-gce-c31', - 'production-gce-c34', - 'production-gce-c35' - ]) { - assert.doesNotThrow( - () => parseRelayAsiaTopologyPlanArguments(argv('production', cellIds, productionImage)), - cellIds - ) - } - for (const cellIds of [ - 'production-gce-c27', - 'production-gce-c27,production-gce-c30', - 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30', - 'production-gce-c30,production-gce-c30', - 'production-gce-c30,production-gce-c31', - 'production-gce-c31,production-gce-c32', - 'production-gce-c32', - 'production-gce-c33', - 'production-gce-c32,production-gce-c33,production-gce-c34', - 'production-gce-c33,production-gce-c34', - 'production-gce-c34,production-gce-c35', - 'production-gce-c36' - ]) { - assert.throws( - () => parseRelayAsiaTopologyPlanArguments(argv('production', cellIds, productionImage)), - /exact reviewed Asia topology set/, - cellIds - ) - } - assert.throws( - () => parseRelayAsiaTopologyPlanArguments(argv('staging', 'production-gce-c30', image)), - /exact reviewed Asia topology set/ - ) - // Each wave's region comes from its reviewed zone, so the workflow must pass that region. - for (const cellIds of ['production-gce-c32,production-gce-c33', 'production-gce-c33,production-gce-c32']) { - assert.deepEqual( - parseRelayAsiaTopologyPlanArguments(argv('production', cellIds, productionImage, 'us-central1')) - .cells, - cellIds.split(',') - ) - assert.throws( - () => parseRelayAsiaTopologyPlanArguments(argv('production', cellIds, productionImage)), - /--region must be us-central1/ - ) - } - assert.throws( - () => parseRelayAsiaTopologyPlanArguments( - argv('production', 'production-gce-c31', productionImage, 'us-central1') - ), - /--region must be asia-east2/ - ) -}) - -test('accepts an idempotent empty plan', () => { - const noChanges = structuredClone(resources).map((resource) => ({ - ...resource, - change: { - ...resource.change, - actions: ['no-op'], - before: structuredClone(resource.change.after) - } - })) - assert.equal(validateRelayAsiaTopologyPlan({ resource_changes: noChanges }, config).changes, 0) -}) - -test('rejects a plan that omits any required topology resource', () => { - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: resources.slice(1) }, config), - /absent or has an unreviewed topology action/ - ) -}) - -test('rejects any US or unrelated mutation', () => { - const plan = structuredClone(resources) - plan.push(create('google_compute_subnetwork.relay_gce[0]')) - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), - /unreviewed topology action/ - ) -}) - -test('rejects delete and replacement actions', () => { - for (const invalidActions of [['delete'], ['create', 'delete']]) { - const plan = structuredClone(resources) - plan[0].change.actions = invalidActions - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), - /unreviewed topology action/ - ) - } -}) - -test('rejects a cell with different limits or image', () => { - const plan = structuredClone(resources) - plan[3].change.after.metadata_startup_script = script.replace("'3000'", "'5000'") - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), - /reviewed Asia cell shape/ - ) -}) - -test('rejects shared URL-map changes outside exact host routing', () => { - const plan = structuredClone(resources) - plan[6].change.after.default_service = 'unreviewed' - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), - /outside host routing/ - ) -}) - -test('rejects removal of an existing exact route', () => { - const plan = structuredClone(resources) - plan[6].change.before.host_rule = [{ hosts: ['c1.relay-staging.onorca.dev'] }] - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), - /preserve every existing exact route/ - ) -}) - -test('accepts provider normalization of preserved route descriptions', () => { - const plan = structuredClone(resources) - const matcher = { - name: 'cell-c1', - description: '', - default_service: - 'https://www.googleapis.com/compute/v1/projects/p/global/backendServices/orca-cloud-staging-relay-gce-c1' - } - plan[6].change.before.host_rule = [{ - description: '', hosts: ['c1.relay-staging.onorca.dev'], path_matcher: 'cell-c1' - }] - plan[6].change.before.path_matcher = [matcher] - plan[6].change.after.host_rule.unshift({ - description: null, hosts: ['c1.relay-staging.onorca.dev'], path_matcher: 'cell-c1' - }) - plan[6].change.after.path_matcher.unshift({ - ...matcher, - description: null, - default_service: 'projects/p/global/backendServices/orca-cloud-staging-relay-gce-c1' - }) - assert.equal(validateRelayAsiaTopologyPlan({ resource_changes: plan }, config).changes, 7) -}) - -test('rejects a changed preserved route backend', () => { - const plan = structuredClone(resources) - plan[6].change.before.path_matcher = [{ - name: 'cell-c1', - default_service: - 'https://www.googleapis.com/compute/v1/projects/p/global/backendServices/orca-cloud-staging-relay-gce-c1' - }] - plan[6].change.after.path_matcher.unshift({ - name: 'cell-c1', - default_service: 'projects/p/global/backendServices/orca-cloud-staging-relay-gce-c2' - }) - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), - /preserve every existing exact route/ - ) -}) - -test('rejects a different Asia subnet range', () => { - const plan = structuredClone(resources) - plan[0].change.after.ip_cidr_range = '10.99.0.0/24' - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: plan }, config), - /reviewed Asia subnet shape/ - ) -}) - -test('rejects incomplete NAT, backend, and URL routing shapes', () => { - const nat = structuredClone(resources) - nat[2].change.after.subnetwork[0].source_ip_ranges_to_nat = [] - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: nat }, config), - /reviewed Asia NAT shape/ - ) - - const backend = structuredClone(resources) - backend[5].change.after.backend[0].group = 'projects/p/zones/asia-east2-a/instanceGroups/wrong' - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: backend }, config), - /reviewed Asia backend shape/ - ) - - const route = structuredClone(resources) - route[6].change.after.path_matcher[0].default_service = - 'projects/p/global/backendServices/wrong' - assert.throws( - () => validateRelayAsiaTopologyPlan({ resource_changes: route }, config), - /no exact backend route/ - ) -}) - -// Terraform cannot export a local to JS, so this validator restates two topology values that -// the Asia workflow applies. Read the .tf source and equate all three statements of each: the -// local, the topology `check` assert that pins it, and the constant above. -test('the reviewed backend constants match the Terraform topology they validate', () => { - const terraform = readFileSync( - new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url), - 'utf8' - ) - const local = (name) => { - const found = new RegExp(`\\n\\s*${name}\\s*=\\s*(\\d+)\\n`).exec(terraform) - assert.notEqual(found, null, `relay_gce_topology has no ${name}`) - return Number(found[1]) - } - const asserted = (name) => { - const found = - new RegExp(`local\\.relay_gce_topology\\.${name}\\s*==\\s*(\\d+)`).exec(terraform) - assert.notEqual(found, null, `the topology check does not pin ${name}`) - return Number(found[1]) - } - for (const [name, constant] of [ - ['backend_timeout_seconds', RELAY_CELL_BACKEND_TIMEOUT_SECONDS], - ['connection_drain_seconds', RELAY_CELL_CONNECTION_DRAIN_SECONDS] - ]) { - assert.equal(local(name), constant, `${name} local differs from the validator constant`) - assert.equal(asserted(name), constant, `${name} check assert differs from the validator`) - } - // The sample rate is a variable, not a local, and no environment file overrides it, so the - // declared default is what every cell backend gets. Equate the default and that absence. - const variables = readFileSync( - new URL('../../infra/terraform/variables.tf', import.meta.url), - 'utf8' - ) - const declared = - /variable "relay_gce_cell_log_sample_rate" \{[\s\S]*?\n {2}default {5}= (\d+)\n/.exec(variables) - assert.notEqual(declared, null, 'relay_gce_cell_log_sample_rate declares no default') - assert.equal(Number(declared[1]), RELAY_CELL_LOG_SAMPLE_RATE) - assert.match(terraform, /sample_rate = var\.relay_gce_cell_log_sample_rate/) - for (const environment of ['production', 'staging']) { - assert.doesNotMatch( - readFileSync( - new URL(`../../infra/terraform/environments/${environment}.tfvars`, import.meta.url), - 'utf8' - ), - /relay_gce_cell_log_sample_rate/, - `${environment} overrides the reviewed log sample rate` - ) - } -}) diff --git a/cloud/dev/scripts/validate-relay-capacity-plan.mjs b/cloud/dev/scripts/validate-relay-capacity-plan.mjs deleted file mode 100644 index 5da51abe95c..00000000000 --- a/cloud/dev/scripts/validate-relay-capacity-plan.mjs +++ /dev/null @@ -1,667 +0,0 @@ -import { readFileSync } from 'node:fs' -import { pathToFileURL } from 'node:url' -import { - RELAY_CELL_CONNECTION_DRAIN_SECONDS, - RELAY_CELL_LOG_SAMPLE_RATE -} from './validate-relay-asia-topology-plan.mjs' - -const CELL_BACKEND_RESOURCE = 'google_compute_backend_service.relay_gce_cell' -const CONNECTION_DRAIN_PATH = 'connection_draining_timeout_sec' -// A backend with no logging has `log_config: []`, so gaining the block moves this one path. -const CELL_LOG_CONFIG_PATH = 'log_config.0' - -const SERVICE_ACCOUNT_EMAIL = - /^[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com$/ - -const REHOME_CONFIG = - /^ printf 'ORCA_RELAY_REHOME_(?:DIRECTOR_SERVICE_ACCOUNT|AUDIENCE)=%s\\n' '[^'\n]+'$/ - -const DATABASE_POOL_MAX = /^ printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '[0-9]+'$/ - -// Only cells listed as regional rehome sources get rehome trust lines in their startup script. -function rehomeProtocol({ regionalRehomeProtocol }) { - if (![0, 1, 3, '0', '1', '3'].includes(regionalRehomeProtocol)) { - throw new Error('same-cap Terraform plan has an invalid regional rehome protocol') - } - return Number(regionalRehomeProtocol) -} - -// Only cells off the root pool default get a pool line, so the caller states whether to expect one. -function databasePoolMax({ databasePoolMax: value }) { - if (value === undefined) return undefined - const pool = Number(value) - if (!/^[0-9]+$/.test(String(value)) || pool < 1 || pool > 100) { - throw new Error('same-cap Terraform plan has an invalid database pool max') - } - return String(pool) -} - -export function parseCapacityPlanArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of ['mode', 'cell-id', 'hard-cap', 'unobserved-bound']) { - if (!values[key]) throw new Error(`missing --${key}`) - } - if (!['cell', 'bootstrap-cell', 'same-cap-cell', 'same-cap-image'].includes(values.mode)) { - throw new Error('--mode must be cell, bootstrap-cell, same-cap-cell, or same-cap-image') - } - const integer = (key) => { - const value = Number(values[key]) - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`--${key} is invalid`) - return value - } - if (!values.image) throw new Error('missing --image') - if ( - ['bootstrap-cell', 'same-cap-cell'].includes(values.mode) && - !values['capacity-service-account'] - ) { - throw new Error('missing --capacity-service-account') - } - if ( - values.mode === 'same-cap-cell' && - (!values['rollback-image'] || - !values['rehome-director-service-account'] || - !values['rehome-audience'] || - !['0', '1', '3'].includes(values['regional-rehome-protocol'])) - ) throw new Error('same-cap validation requires rollback image and rehome trust config') - if (values.mode !== 'same-cap-cell' && values['regional-rehome-protocol'] !== undefined) { - throw new Error('--regional-rehome-protocol applies only to same-cap-cell validation') - } - if (values.mode !== 'same-cap-cell' && values['database-pool-max'] !== undefined) { - throw new Error('--database-pool-max applies only to same-cap-cell validation') - } - if (values.mode === 'same-cap-image' && !values['rollback-image']) { - throw new Error('same-cap image validation requires a rollback image') - } - if ( - values['capacity-service-account'] !== undefined && - !SERVICE_ACCOUNT_EMAIL.test(values['capacity-service-account']) - ) { - throw new Error('--capacity-service-account is invalid') - } - return { - mode: values.mode, - cellId: values['cell-id'], - hardCap: integer('hard-cap'), - unobservedBound: integer('unobserved-bound'), - image: values.image, - capacityServiceAccount: values['capacity-service-account'], - rollbackImage: values['rollback-image'], - rehomeDirectorServiceAccount: values['rehome-director-service-account'], - rehomeAudience: values['rehome-audience'], - regionalRehomeProtocol: values['regional-rehome-protocol'], - databasePoolMax: values['database-pool-max'] - } -} - -function mutations(plan) { - if (!Array.isArray(plan.resource_changes)) throw new Error('Terraform plan has no resource changes') - return plan.resource_changes.filter(({ change }) => { - const actions = change?.actions - return Array.isArray(actions) && !actions.every((action) => ['no-op', 'read'].includes(action)) - }) -} - -function sameActions(change, expected) { - return JSON.stringify(change.change?.actions) === JSON.stringify(expected) -} - -function changedPaths(before, after, path = []) { - if (Object.is(before, after)) return [] - const beforeObject = before !== null && typeof before === 'object' - const afterObject = after !== null && typeof after === 'object' - if (!beforeObject || !afterObject || Array.isArray(before) !== Array.isArray(after)) { - return [path.join('.')] - } - const keys = new Set([...Object.keys(before), ...Object.keys(after)]) - return [...keys].flatMap((key) => changedPaths(before[key], after[key], [...path, key])) -} - -function unknownPaths(value, path = []) { - if (value === true) return [path.join('.')] - if (value === null || typeof value !== 'object') return [] - return Object.entries(value).flatMap(([key, nested]) => unknownPaths(nested, [...path, key])) -} - -function valueAtPath(value, path) { - return path.split('.').reduce((current, key) => current?.[key], value) -} - -function providerDefaultPaths(change, paths) { - const empty = (value) => - value === '' || - value === 0 || - (Array.isArray(value) && value.length === 0) || - (value !== null && - typeof value === 'object' && - !Array.isArray(value) && - Object.keys(value).length === 0) - return paths.filter( - (path) => - empty(valueAtPath(change.change.before, path)) && - valueAtPath(change.change.after, path) === null - ) -} - -function canonicalResourcePaths(change, paths) { - const canonical = (value) => - typeof value === 'string' - ? value.replace('https://www.googleapis.com/compute/v1/', '') - : value - return paths.filter( - (path) => - canonical(valueAtPath(change.change.before, path)) === - canonical(valueAtPath(change.change.after, path)) - ) -} - -// What relay-gce-cells.tf declares for these MIG fields (a test pins the two together). A gcloud -// rolling action, like the stranded recovery's old one, rewrites the version label and persists -// its flags into the update policy outside Terraform; a later plan then reverts them. Moving back -// to the declared value is reconciliation, never a capacity change, so it is the only move allowed. -export const DECLARED_MANAGER_FIELDS = Object.freeze({ - 'version.0.name': 'primary', - 'update_policy.0.type': 'PROACTIVE', - 'update_policy.0.minimal_action': 'REPLACE', - 'update_policy.0.most_disruptive_allowed_action': 'REPLACE', - 'update_policy.0.replacement_method': 'RECREATE', - 'update_policy.0.max_surge_fixed': 0, - 'update_policy.0.max_unavailable_fixed': 1 -}) - -function declaredManagerReconciliationPaths(change) { - return Object.entries(DECLARED_MANAGER_FIELDS) - .filter(([path, declared]) => valueAtPath(change.change.after, path) === declared) - .map(([path]) => path) -} - -function requireOnlyPaths(change, allowed, required = [], allowedUnknown = new Set()) { - const paths = changedPaths(change.change.before, change.change.after) - const unknown = unknownPaths(change.change.after_unknown) - const unexpected = paths.filter((path) => !allowed.has(path)) - const unexpectedUnknown = unknown.filter((path) => !allowedUnknown.has(path)) - if ( - unexpected.length > 0 || - unexpectedUnknown.length > 0 || - required.some((path) => !paths.includes(path)) - ) { - throw new Error(`${change.address} changes outside the reviewed capacity fields`) - } -} - -function relayImage(script) { - const lines = script.split('\n') - const starts = lines.flatMap((line, index) => - line === 'docker run --detach \\' ? [index] : []) - const commands = starts.map((start) => { - const end = lines.findIndex((line, index) => index > start && !line.endsWith(' \\')) - return end < 0 ? [] : lines.slice(start, end + 1) - }) - const relayCommands = commands.filter((command) => - command.filter((line) => line === ' --name orca-relay \\').length === 1) - if (relayCommands.length !== 1) return null - const command = relayCommands[0] - const image = /^ '([^'\n]+@sha256:[a-f0-9]{64})'$/.exec(command.at(-1))?.[1] - const digests = [...command.join('\n').matchAll(/'([^'\n]+@sha256:[a-f0-9]{64})'/g)] - return image && digests.length === 1 ? image : null -} - -function normalizedStartupScript( - script, - stripCapacityIdentity = false, - stripRehomeConfig = false, - preserveCapacity = false, - stripDatabasePoolMax = false -) { - const image = relayImage(script) - if (!image) throw new Error('cell plan startup script has no Relay image') - const digest = image.split('@')[1] - const capacityAssignment = - /^ printf 'ORCA_RELAY_CELL_CONNECTION_(?:HARD_CAP|UNOBSERVED_BOUND)=%s\\n' '[0-9]+'$/ - const capacityIdentity = - /^ printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/ - return script - .split('\n') - .filter( - (line) => - (preserveCapacity || !capacityAssignment.test(line)) && - (!stripCapacityIdentity || !capacityIdentity.test(line)) && - (!stripRehomeConfig || !REHOME_CONFIG.test(line)) && - (!stripDatabasePoolMax || !DATABASE_POOL_MAX.test(line)) - ) - .join('\n') - .replaceAll(image, '') - .replaceAll(digest, '') -} - -function hasExactSingleAssignment(lines, pattern, expected) { - const assignments = lines.filter((line) => pattern.test(line)) - return assignments.length === 1 && assignments[0] === expected -} - -function requireDesiredStartupScript(script, config) { - const lines = typeof script === 'string' ? script.split('\n') : [] - const expected = [ - [ - /^ printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '[0-9]+'$/, - ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${config.hardCap}'` - ], - [ - /^ printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '[0-9]+'$/, - ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${config.unobservedBound}'` - ] - ] - // A same-cap cell whose template predates this line gains it on its next roll, so the - // before/after comparison ignores it; pinning the exact identity here is what reviews it, - // and what stops a roll dropping or rewriting the line it lets through. - if (['bootstrap-cell', 'same-cap-cell'].includes(config.mode)) { - expected.push([ - /^ printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/, - ` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'` - ]) - } - const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) >= 1 - if (rehomeTrusted) { - expected.push( - [ - /^ printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '[^'\n]+'$/, - ` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${config.rehomeDirectorServiceAccount}'` - ], - [ - /^ printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '[^'\n]+'$/, - ` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${config.rehomeAudience}'` - ] - ) - } - // A protocol-0 cell is not a rehome source, so gaining any rehome trust line is real drift. - const unexpectedRehome = - config.mode === 'same-cap-cell' && - !rehomeTrusted && - lines.some((line) => REHOME_CONFIG.test(line)) - const pool = config.mode === 'same-cap-cell' ? databasePoolMax(config) : undefined - if (pool !== undefined) { - expected.push([ - DATABASE_POOL_MAX, - ` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'` - ]) - } - // An unpinned cell sits on the root pool default, so gaining a pool line is real drift. - const unexpectedDatabasePoolMax = - config.mode === 'same-cap-cell' && - pool === undefined && - lines.some((line) => DATABASE_POOL_MAX.test(line)) - if ( - typeof script !== 'string' || - relayImage(script) !== config.image || - unexpectedRehome || - unexpectedDatabasePoolMax || - expected.some(([pattern, line]) => !hasExactSingleAssignment(lines, pattern, line)) - ) { - throw new Error('cell plan does not contain the reviewed image and capacity') - } -} - -// The same-cap job no longer targets this cell's backend service (the capacity role has no -// compute.backendServices.update), so a wave plan carries no backend change and this reports an -// empty list. It stays as the bound on any caller that does target one: exactly this cell's -// backend, exactly the reviewed drain and request-logging attributes, each optional because a -// cell that already has one plans no change for it. Splitting the backend out keeps `changes` -// the template-and-MIG count both callers read. -function takeCellBackendUpdate(changes, config) { - const backends = changes.filter( - ({ address }) => typeof address === 'string' && address.startsWith(`${CELL_BACKEND_RESOURCE}[`) - ) - if (config.mode !== 'same-cap-cell' || backends.length === 0) { - return { rest: changes, backendUpdate: [] } - } - const [backend] = backends - if ( - backends.length !== 1 || - backend.address !== `${CELL_BACKEND_RESOURCE}[${JSON.stringify(config.cellId)}]` || - backend.deposed !== undefined || - !sameActions(backend, ['update']) - ) { - throw new Error('cell plan may change only this cell backend drain and request logging') - } - const after = backend.change?.after - const moved = changedPaths(backend.change?.before, after) - const logging = after?.log_config?.[0] - const accepted = [CONNECTION_DRAIN_PATH, CELL_LOG_CONFIG_PATH].filter((path) => - moved.includes(path)) - if ( - accepted.length === 0 || - (moved.includes(CONNECTION_DRAIN_PATH) && - after?.[CONNECTION_DRAIN_PATH] !== RELAY_CELL_CONNECTION_DRAIN_SECONDS) || - (moved.includes(CELL_LOG_CONFIG_PATH) && - (after?.log_config?.length !== 1 || - logging?.enable !== true || - logging?.sample_rate !== RELAY_CELL_LOG_SAMPLE_RATE)) - ) { - throw new Error('cell plan may change only this cell backend drain and request logging') - } - const backendComputed = new Set(['fingerprint', 'generated_id']) - requireOnlyPaths( - backend, - new Set([ - ...accepted, - ...unknownPaths(backend.change.after_unknown).filter((path) => backendComputed.has(path)) - ]), - accepted, - backendComputed - ) - return { rest: changes.filter((change) => change !== backend), backendUpdate: accepted } -} - -function plannedResources(module) { - if (!module) return [] - return [ - ...(module.resources ?? []), - ...(module.child_modules ?? []).flatMap(plannedResources) - ] -} - -function canonicalResource(value) { - return typeof value === 'string' - ? value.replace('https://www.googleapis.com/compute/v1/', '') - : value -} - -function requireDesiredPlannedCell(plan, config) { - const resources = plannedResources(plan.planned_values?.root_module) - const templateAddress = `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` - const managerAddress = `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(config.cellId)}]` - const template = resources.find(({ address }) => address === templateAddress)?.values - const manager = resources.find(({ address }) => address === managerAddress)?.values - if (!template || !manager) throw new Error('convergence plan has no exact planned C26 state') - requireDesiredStartupScript(template.metadata_startup_script, config) - const templateReference = canonicalResource(template.self_link ?? template.id) - const managerReference = canonicalResource(manager.version?.[0]?.instance_template) - if (!templateReference || templateReference !== managerReference) { - throw new Error('convergence plan does not bind the MIG to the reviewed template') - } -} - -function validateManagerUpdate(manager) { - if (!sameActions(manager, ['update'])) { - throw new Error('cell plan has unexpected MIG actions') - } - const managerComputed = new Set([ - 'fingerprint', - 'operation', - 'status', - 'version.0.instance_template' - ]) - const managerUnknown = unknownPaths(manager.change.after_unknown) - const moved = changedPaths(manager.change.before, manager.change.after) - const reconciled = declaredManagerReconciliationPaths(manager).filter((path) => - moved.includes(path)) - // A plan that only reconciles declared fields leaves the template where it is. - requireOnlyPaths( - manager, - new Set([ - 'version.0.instance_template', - ...reconciled, - ...managerUnknown.filter((path) => managerComputed.has(path)) - ]), - reconciled.length > 0 ? [] : ['version.0.instance_template'], - managerComputed - ) -} - -function requireReplacementTemplateDependency(plan, template, manager) { - const configuredManager = plan.configuration?.root_module?.resources?.find( - ({ address }) => address === 'google_compute_instance_group_manager.relay_gce_cell' - ) - const expectedVersionExpression = [{ - instance_template: { - references: ['google_compute_instance_template.relay_gce_cell', 'each.key'] - }, - name: { constant_value: 'primary' } - }] - if ( - JSON.stringify(configuredManager?.expressions?.version) !== - JSON.stringify(expectedVersionExpression) || - template.change.after?.self_link != null || - template.change.after_unknown?.self_link !== true || - manager.change.after?.version?.[0]?.instance_template != null || - manager.change.after_unknown?.version?.[0]?.instance_template !== true - ) { - throw new Error('cell plan does not bind the MIG to the reviewed template dependency') - } -} - -function cellPlan(plan, changes, config) { - const templateAddress = `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` - const managerAddress = `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(config.cellId)}]` - const template = changes.find( - ({ address, deposed }) => address === templateAddress && deposed === undefined - ) - const manager = changes.find(({ address }) => address === managerAddress) - const obsoleteTemplates = changes.filter( - ({ address, deposed }) => address === templateAddress && typeof deposed === 'string' - ) - // A failed wave apply leaves its predecessor deposed; the wave must plan its own cleanup. - const allowsObsoleteTemplates = - obsoleteTemplates.length > 0 && - obsoleteTemplates.every((change) => sameActions(change, ['delete'])) && - (config.mode === 'same-cap-image' || - (config.mode === 'same-cap-cell' && obsoleteTemplates.length === 1)) - if ( - !template || - !manager || - changes.length !== 2 + obsoleteTemplates.length || - (obsoleteTemplates.length > 0 && !allowsObsoleteTemplates) - ) { - throw new Error('cell plan must change only the exact instance template and MIG') - } - if (!sameActions(template, ['create', 'delete']) || !sameActions(manager, ['update'])) { - throw new Error('cell plan has unexpected replacement actions') - } - const templateComputed = new Set([ - 'confidential_instance_config', - 'creation_timestamp', - 'disk.0.architecture', - 'disk.0.interface', - 'disk.0.mode', - 'disk.0.provisioned_iops', - 'disk.0.provisioned_throughput', - 'disk.0.type', - 'id', - 'metadata_fingerprint', - 'name', - 'network_interface.0.internal_ipv6_prefix_length', - 'network_interface.0.ipv6_access_type', - 'network_interface.0.ipv6_address', - 'network_interface.0.name', - 'network_interface.0.network', - 'network_interface.0.stack_type', - 'network_interface.0.subnetwork_project', - 'numeric_id', - 'region', - 'self_link', - 'self_link_unique', - 'tags_fingerprint' - ]) - const templateDefaults = [ - 'description', - 'disk.0.disk_name', - 'disk.0.guest_os_features', - 'disk.0.labels', - 'disk.0.resource_manager_tags', - 'disk.0.resource_policies', - 'disk.0.source', - 'disk.0.source_snapshot', - 'instance_description', - 'key_revocation_action_type', - 'min_cpu_platform', - 'network_interface.0.network_ip', - 'network_interface.0.nic_type', - 'network_interface.0.queue_count', - 'scheduling.0.availability_domain', - 'scheduling.0.instance_termination_action', - 'scheduling.0.min_node_cpus', - 'scheduling.0.termination_time' - ] - const templateCanonicalResources = [ - 'disk.0.source_image', - 'network_interface.0.subnetwork' - ] - const templateUnknown = unknownPaths(template.change.after_unknown) - requireOnlyPaths( - template, - new Set([ - 'metadata_startup_script', - ...templateUnknown.filter((path) => templateComputed.has(path)), - ...providerDefaultPaths(template, templateDefaults), - ...canonicalResourcePaths(template, templateCanonicalResources) - ]), - ['metadata_startup_script'], - templateComputed - ) - validateManagerUpdate(manager) - requireReplacementTemplateDependency(plan, template, manager) - const beforeScript = template.change.before?.metadata_startup_script - const script = template.change.after?.metadata_startup_script - requireDesiredStartupScript(script, config) - const sameCap = ['same-cap-cell', 'same-cap-image'].includes(config.mode) - // Only a pinned pool may move here; requireDesiredStartupScript holds the after value exactly. - const stripPool = config.mode === 'same-cap-cell' && config.databasePoolMax !== undefined - // Only a template stale enough to predate the line may move it, and only by gaining it; - // requireDesiredStartupScript holds the after value to the exact reviewed identity. - const stripCapacityIdentity = - ['bootstrap-cell', 'same-cap-cell'].includes(config.mode) - if ( - typeof beforeScript !== 'string' || - (sameCap && relayImage(beforeScript) !== config.rollbackImage) || - normalizedStartupScript( - beforeScript, - stripCapacityIdentity, - config.mode === 'same-cap-cell', - sameCap, - stripPool - ) !== normalizedStartupScript( - script, - stripCapacityIdentity, - config.mode === 'same-cap-cell', - sameCap, - stripPool - ) - ) { - throw new Error('cell plan does not contain the reviewed image and capacity') - } -} - -function convergenceCellPlan(plan, changes, config) { - const templateAddress = `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` - const managerAddress = `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(config.cellId)}]` - const manager = changes.find(({ address }) => address === managerAddress) - const obsoleteTemplates = changes.filter( - ({ address, deposed }) => address === templateAddress && typeof deposed === 'string' - ) - if ( - (!manager && obsoleteTemplates.length === 0) || - (obsoleteTemplates.length > 1 && config.mode !== 'same-cap-image') || - changes.length !== (manager ? 1 : 0) + obsoleteTemplates.length - ) { - throw new Error('cell convergence plan changes outside the exact template and MIG') - } - if (manager) validateManagerUpdate(manager) - if (obsoleteTemplates.some((change) => !sameActions(change, ['delete']))) { - throw new Error('cell convergence plan has unexpected obsolete-template actions') - } - requireDesiredPlannedCell(plan, config) -} - -export function validateCapacityPlan(plan, config) { - if (!['cell', 'bootstrap-cell', 'same-cap-cell', 'same-cap-image'].includes(config.mode)) { - throw new Error('capacity Terraform plans may change only a cell') - } - if ( - ['bootstrap-cell', 'same-cap-cell'].includes(config.mode) && - !SERVICE_ACCOUNT_EMAIL.test(config.capacityServiceAccount ?? '') - ) { - throw new Error('capacity Terraform plan has an invalid service account') - } - if (config.mode === 'same-cap-cell') { - rehomeProtocol(config) - databasePoolMax(config) - } - if ( - config.mode === 'same-cap-cell' && - (!SERVICE_ACCOUNT_EMAIL.test(config.rehomeDirectorServiceAccount ?? '') || - !/^https:\/\/[^/]+\/v1\/admin\/host-drain$/.test(config.rehomeAudience ?? '') || - !/^.+@sha256:[a-f0-9]{64}$/.test(config.rollbackImage ?? '')) - ) throw new Error('same-cap Terraform plan has invalid rehome trust config') - if ( - config.mode === 'same-cap-image' && - !/^.+@sha256:[a-f0-9]{64}$/.test(config.rollbackImage ?? '') - ) throw new Error('same-cap image Terraform plan has an invalid rollback image') - const { rest: changes, backendUpdate } = takeCellBackendUpdate(mutations(plan), config) - const backend = backendUpdate.length > 0 ? { backendUpdate } : {} - if (changes.length === 0) { - return { - mode: config.mode, - changes: 0, - ...backend, - ...(config.mode === 'same-cap-image' ? { changeKind: 'none' } : {}) - } - } - const templateAddress = `google_compute_instance_template.relay_gce_cell[${JSON.stringify(config.cellId)}]` - const replacement = changes.some( - ({ address, deposed, change }) => - address === templateAddress && - deposed === undefined && - JSON.stringify(change?.actions) === JSON.stringify(['create', 'delete']) - ) - if (replacement) cellPlan(plan, changes, config) - else convergenceCellPlan(plan, changes, config) - const obsoleteTemplates = changes.filter( - ({ address, deposed }) => address === templateAddress && typeof deposed === 'string' - ) - const obsoleteTemplateOnly = changes.every( - ({ address, deposed, change }) => - address === templateAddress && - typeof deposed === 'string' && - JSON.stringify(change?.actions) === JSON.stringify(['delete']) - ) - // Same as the backend split: `changes` stays the template-and-MIG count the same-cap job gates - // on. same-cap-image keeps its own total and names obsolete deletes in changeKind instead. - const splitsObsoleteTemplates = config.mode === 'same-cap-cell' && obsoleteTemplates.length > 0 - return { - mode: config.mode, - changes: changes.length - (splitsObsoleteTemplates ? obsoleteTemplates.length : 0), - ...(splitsObsoleteTemplates ? { obsoleteTemplates: obsoleteTemplates.length } : {}), - ...backend, - ...(config.mode === 'same-cap-image' - ? { - changeKind: replacement - ? obsoleteTemplates.length > 0 - ? 'replacement-with-obsolete-template' - : 'replacement' - : obsoleteTemplateOnly - ? 'obsolete-template-delete' - : 'manager-convergence' - } - : {}) - } -} - -export function main(argv = process.argv.slice(2)) { - const config = parseCapacityPlanArguments(argv) - const plan = JSON.parse(readFileSync(0, 'utf8')) - process.stdout.write(`${JSON.stringify({ event: 'relay_capacity_plan_verified', ...validateCapacityPlan(plan, config) })}\n`) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - try { - main() - } catch (error) { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - } -} diff --git a/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs b/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs deleted file mode 100644 index bbd6220423e..00000000000 --- a/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs +++ /dev/null @@ -1,1430 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { - RELAY_CELL_CONNECTION_DRAIN_SECONDS, - RELAY_CELL_LOG_SAMPLE_RATE -} from './validate-relay-asia-topology-plan.mjs' -import { readFileSync } from 'node:fs' -import { - DECLARED_MANAGER_FIELDS, - parseCapacityPlanArguments, - validateCapacityPlan as validateCapacityPlanRaw -} from './validate-relay-capacity-plan.mjs' - -const config = { - cellId: 'staging-gce-c3', - hardCap: 1_000, - unobservedBound: 60 -} - -function replacementConfiguration(references = [ - 'google_compute_instance_template.relay_gce_cell', - 'each.key' -]) { - return { - root_module: { - resources: [{ - address: 'google_compute_instance_group_manager.relay_gce_cell', - expressions: { - version: [{ - instance_template: { references }, - name: { constant_value: 'primary' } - }] - } - }] - } - } -} - -function validateCapacityPlan(plan, planConfig) { - const replacement = plan.resource_changes.some(({ change }) => - JSON.stringify(change?.actions) === JSON.stringify(['create', 'delete'])) - return validateCapacityPlanRaw( - replacement && !plan.configuration - ? { ...plan, configuration: replacementConfiguration() } - : plan, - planConfig - ) -} - -test('accepts only the exact canary template replacement and MIG update', () => { - const image = `us-docker.pkg.dev/project/relay/image@sha256:${'a'.repeat(64)}` - const startupScript = (cap, bound, selectedImage, extra = '') => - [ - ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`, - ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${bound}'`, - `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`, - `docker pull '${selectedImage}'`, - extra, - 'docker run --detach \\', - ' --name cloud-sql-proxy \\', - ` 'us-docker.pkg.dev/project/proxy@sha256:${'c'.repeat(64)}'`, - 'docker run --detach \\', - ' --name orca-relay \\', - ` '${selectedImage}'` - ].join('\n') - const script = startupScript(1_000, 60, image) - const template = { - address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]', - change: { - actions: ['create', 'delete'], - before: { - metadata_startup_script: startupScript( - 600, - 60, - `us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}` - ) - }, - after: { metadata_startup_script: script, self_link: null }, - after_unknown: { self_link: true } - } - } - const manager = { - address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]', - change: { - actions: ['update'], - before: { target_size: 1, version: [{ instance_template: 'old' }] }, - after: { target_size: 1, version: [{ instance_template: null }] }, - after_unknown: { version: [{ instance_template: true }] } - } - } - const cellConfig = { ...config, mode: 'cell', image } - assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, cellConfig), { - mode: 'cell', - changes: 0 - }) - assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), { - mode: 'cell', - changes: 2 - }) - const wrongTemplateManager = structuredClone(manager) - wrongTemplateManager.change.after.version[0].instance_template = - 'projects/project/global/instanceTemplates/unreviewed' - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, wrongTemplateManager] }, - cellConfig - ), - /does not bind the MIG/ - ) - assert.throws( - () => validateCapacityPlanRaw( - { - resource_changes: [template, manager], - configuration: replacementConfiguration([ - 'google_compute_instance_template.relay_gce_cell', - 'var.unreviewed_key' - ]) - }, - cellConfig - ), - /reviewed template dependency/ - ) - assert.throws( - () => validateCapacityPlanRaw( - { resource_changes: [template, manager] }, - cellConfig - ), - /reviewed template dependency/ - ) - const capacityServiceAccount = - 'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com' - const bootstrapTemplate = structuredClone(template) - const bootstrapManager = structuredClone(manager) - bootstrapTemplate.change.after.metadata_startup_script = [ - ` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityServiceAccount}'`, - script - ].join('\n') - const bootstrapConfig = { - ...cellConfig, - mode: 'bootstrap-cell', - capacityServiceAccount - } - const plannedValues = (startupScript) => ({ - root_module: { - resources: [ - { - address: bootstrapTemplate.address, - values: { - metadata_startup_script: startupScript, - self_link: 'projects/project/global/instanceTemplates/c26-reviewed' - } - }, - { - address: bootstrapManager.address, - values: { - version: [{ - instance_template: - 'https://www.googleapis.com/compute/v1/projects/project/global/instanceTemplates/c26-reviewed' - }] - } - } - ] - } - }) - assert.deepEqual( - validateCapacityPlan( - { resource_changes: [bootstrapTemplate, bootstrapManager] }, - bootstrapConfig - ), - { mode: 'bootstrap-cell', changes: 2 } - ) - const managerOnly = structuredClone(bootstrapManager) - assert.deepEqual( - validateCapacityPlan( - { - resource_changes: [managerOnly], - planned_values: plannedValues( - bootstrapTemplate.change.after.metadata_startup_script - ) - }, - bootstrapConfig - ), - { mode: 'bootstrap-cell', changes: 1 } - ) - const decoyPlannedValues = plannedValues( - bootstrapTemplate.change.after.metadata_startup_script.replaceAll( - image, - `us-docker.pkg.dev/project/relay/image@sha256:${'b'.repeat(64)}` - ) + `\n# decoy '${image}'` - ) - assert.throws( - () => validateCapacityPlan( - { resource_changes: [managerOnly], planned_values: decoyPlannedValues }, - bootstrapConfig - ), - /reviewed image and capacity/ - ) - const obsoleteTemplate = structuredClone(bootstrapTemplate) - obsoleteTemplate.deposed = 'retired-template' - obsoleteTemplate.change.actions = ['delete'] - obsoleteTemplate.change.after = null - assert.deepEqual( - validateCapacityPlan( - { - resource_changes: [managerOnly, obsoleteTemplate], - planned_values: plannedValues( - bootstrapTemplate.change.after.metadata_startup_script - ) - }, - bootstrapConfig - ), - { mode: 'bootstrap-cell', changes: 2 } - ) - assert.deepEqual( - validateCapacityPlan( - { - resource_changes: [obsoleteTemplate], - planned_values: plannedValues( - bootstrapTemplate.change.after.metadata_startup_script - ) - }, - bootstrapConfig - ), - { mode: 'bootstrap-cell', changes: 1 } - ) - const wrongManagerReference = plannedValues( - bootstrapTemplate.change.after.metadata_startup_script - ) - wrongManagerReference.root_module.resources[1].values.version[0].instance_template = - 'projects/project/global/instanceTemplates/not-reviewed' - assert.throws( - () => validateCapacityPlan( - { resource_changes: [managerOnly], planned_values: wrongManagerReference }, - bootstrapConfig - ), - /does not bind the MIG/ - ) - assert.throws( - () => validateCapacityPlan( - { resource_changes: [managerOnly] }, - bootstrapConfig - ), - /no exact planned C26 state/ - ) - // A MIG moving back to its declared label and update policy is reconciliation in every mode. - const restartedBootstrapManager = structuredClone(bootstrapManager) - restartedBootstrapManager.change.before.update_policy = [{ minimal_action: 'RESTART' }] - restartedBootstrapManager.change.after.update_policy = [{ minimal_action: 'REPLACE' }] - restartedBootstrapManager.change.before.version[0].name = - '0/2026-08-10 23:30:14.196895+00:00' - restartedBootstrapManager.change.after.version[0].name = 'primary' - assert.deepEqual( - validateCapacityPlan( - { resource_changes: [bootstrapTemplate, restartedBootstrapManager] }, - bootstrapConfig - ), - { mode: 'bootstrap-cell', changes: 2 } - ) - assert.deepEqual( - validateCapacityPlan({ resource_changes: [template, restartedBootstrapManager] }, cellConfig), - { mode: 'cell', changes: 2 } - ) - const operatorLabelManager = structuredClone(restartedBootstrapManager) - operatorLabelManager.change.before.version[0].name = 'operator-version' - delete operatorLabelManager.change.before.update_policy - delete operatorLabelManager.change.after.update_policy - assert.deepEqual( - validateCapacityPlan( - { resource_changes: [bootstrapTemplate, operatorLabelManager] }, - bootstrapConfig - ), - { mode: 'bootstrap-cell', changes: 2 } - ) - const unrecognizedPrimaryVersion = structuredClone(restartedBootstrapManager) - unrecognizedPrimaryVersion.change.after.version[0].name = 'other' - assert.throws( - () => - validateCapacityPlan( - { resource_changes: [bootstrapTemplate, unrecognizedPrimaryVersion] }, - bootstrapConfig - ), - /outside the reviewed capacity fields/ - ) - const unrecognizedRestoredPolicy = structuredClone(restartedBootstrapManager) - unrecognizedRestoredPolicy.change.after.update_policy[0].minimal_action = 'REFRESH' - assert.throws( - () => - validateCapacityPlan( - { resource_changes: [bootstrapTemplate, unrecognizedRestoredPolicy] }, - bootstrapConfig - ), - /outside the reviewed capacity fields/ - ) - // Gaining a whole policy block is not a field reverting, so it is not reconciliation. - const addedPolicy = structuredClone(restartedBootstrapManager) - delete addedPolicy.change.before.update_policy - assert.throws( - () => - validateCapacityPlan( - { resource_changes: [bootstrapTemplate, addedPolicy] }, - bootstrapConfig - ), - /outside the reviewed capacity fields/ - ) - const duplicateHardCapTemplate = structuredClone(template) - duplicateHardCapTemplate.change.after.metadata_startup_script = [ - script, - ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'` - ].join('\n') - assert.throws( - () => - validateCapacityPlan( - { resource_changes: [duplicateHardCapTemplate, structuredClone(manager)] }, - cellConfig - ), - /reviewed image and capacity/ - ) - const duplicateIdentityTemplate = structuredClone(bootstrapTemplate) - duplicateIdentityTemplate.change.after.metadata_startup_script = [ - duplicateIdentityTemplate.change.after.metadata_startup_script, - ` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' 'other-capacity@onorca-cloud-staging.iam.gserviceaccount.com'` - ].join('\n') - assert.throws( - () => - validateCapacityPlan( - { resource_changes: [duplicateIdentityTemplate, structuredClone(bootstrapManager)] }, - bootstrapConfig - ), - /reviewed image and capacity/ - ) - assert.throws( - () => - validateCapacityPlan( - { resource_changes: [bootstrapTemplate, bootstrapManager] }, - { ...bootstrapConfig, capacityServiceAccount: 'invalid' } - ), - /invalid service account/ - ) - assert.throws( - () => - validateCapacityPlan( - { resource_changes: [bootstrapTemplate, bootstrapManager] }, - cellConfig - ), - /reviewed image and capacity/ - ) - manager.change.after.target_size = 0 - assert.throws( - () => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), - /outside the reviewed capacity fields/ - ) - manager.change.after.target_size = 1 - template.change.after.metadata_startup_script = startupScript(1_000, 60, image, 'curl bad') - assert.throws( - () => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), - /reviewed image and capacity/ - ) - template.change.after.metadata_startup_script = startupScript( - 1_000, - 60, - image, - 'curl bad # ORCA_RELAY_CELL_CONNECTION_HARD_CAP=' - ) - assert.throws( - () => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), - /reviewed image and capacity/ - ) - template.change.after.metadata_startup_script = script - template.change.after_unknown = { - id: true, - self_link: true, - disk: [{ architecture: true }] - } - manager.change.after_unknown = { - fingerprint: true, - version: [{ instance_template: true }] - } - assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), { - mode: 'cell', - changes: 2 - }) - template.change.before.description = '' - template.change.after.description = null - template.change.before.disk = [{ - architecture: '', - source_image: 'projects/cos-cloud/global/images/cos-stable-1' - }] - template.change.after.disk = [{ - source_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1' - }] - template.change.after_unknown.disk = [{ architecture: true }] - assert.deepEqual(validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), { - mode: 'cell', - changes: 2 - }) - template.change.after.disk[0].source_image = - 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/different' - assert.throws( - () => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), - /outside the reviewed capacity fields/ - ) - template.change.after.disk[0].source_image = - 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-1' - manager.change.after_unknown = { target_size: true } - assert.throws( - () => validateCapacityPlan({ resource_changes: [template, manager] }, cellConfig), - /outside the reviewed capacity fields/ - ) -}) - -test('same-cap mode preserves 1000/60 while adding only the reviewed trust config', () => { - const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}` - const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}` - const directorIdentity = 'relay-director@project.iam.gserviceaccount.com' - const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com' - const audience = 'https://relay.example.com/v1/admin/host-drain' - const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [ - ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`, - ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`, - ...(capacity === null - ? [] - : [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]), - ...(trust ? [ - ` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`, - ` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'` - ] : []), - `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`, - `docker pull '${selectedImage}'`, - 'docker run --detach \\', - ' --name orca-relay \\', - ` '${selectedImage}'` - ].join('\n') - const template = { - address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]', - change: { - actions: ['create', 'delete'], - before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) }, - after: { - metadata_startup_script: startup({ selectedImage: image, trust: true }), - self_link: null - }, - after_unknown: { self_link: true } - } - } - const manager = { - address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]', - change: { - actions: ['update'], - before: { target_size: 1, version: [{ instance_template: 'old' }] }, - after: { target_size: 1, version: [{ instance_template: null }] }, - after_unknown: { version: [{ instance_template: true }] } - } - } - const sameCapConfig = { - ...config, - mode: 'same-cap-cell', - image, - rollbackImage, - capacityServiceAccount: capacityIdentity, - rehomeDirectorServiceAccount: directorIdentity, - rehomeAudience: audience, - regionalRehomeProtocol: '1' - } - assert.deepEqual( - validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig), - { mode: 'same-cap-cell', changes: 2 } - ) - // A gcloud rolling action on a stranded cell renamed its MIG version; the next roll reverts it. - const relabelledManager = structuredClone(manager) - relabelledManager.change.before.version[0].name = '0/2026-10-01 10:41:28.681518+00:00' - relabelledManager.change.after.version[0].name = 'primary' - assert.deepEqual( - validateCapacityPlan({ resource_changes: [template, relabelledManager] }, sameCapConfig), - { mode: 'same-cap-cell', changes: 2 } - ) - for (const [field, moved] of [ - ['target_size', 0], - ['base_instance_name', 'relay-other'], - ['named_port', [{ name: 'relay', port: 9090 }]] - ]) { - const unrelated = structuredClone(relabelledManager) - unrelated.change.after[field] = moved - assert.throws( - () => validateCapacityPlan({ resource_changes: [template, unrelated] }, sameCapConfig), - /outside the reviewed capacity fields/, - field - ) - } - const declaredPolicy = { - type: 'PROACTIVE', - minimal_action: 'REPLACE', - most_disruptive_allowed_action: 'REPLACE', - replacement_method: 'RECREATE', - max_surge_fixed: 0, - max_unavailable_fixed: 1 - } - for (const [field, drifted, away] of [ - ['type', 'OPPORTUNISTIC', 'OPPORTUNISTIC'], - ['minimal_action', 'RESTART', 'RESTART'], - ['most_disruptive_allowed_action', 'RESTART', 'RESTART'], - ['replacement_method', 'SUBSTITUTE', 'SUBSTITUTE'], - ['max_surge_fixed', 1, 1], - ['max_unavailable_fixed', 0, 0] - ]) { - const reverted = structuredClone(relabelledManager) - reverted.change.before.update_policy = [{ ...declaredPolicy, [field]: drifted }] - reverted.change.after.update_policy = [{ ...declaredPolicy }] - assert.deepEqual( - validateCapacityPlan({ resource_changes: [template, reverted] }, sameCapConfig), - { mode: 'same-cap-cell', changes: 2 }, - field - ) - const leaving = structuredClone(relabelledManager) - leaving.change.before.update_policy = [{ ...declaredPolicy }] - leaving.change.after.update_policy = [{ ...declaredPolicy, [field]: away }] - assert.throws( - () => validateCapacityPlan({ resource_changes: [template, leaving] }, sameCapConfig), - /outside the reviewed capacity fields/, - field - ) - } - // A second version is a canary split, never a label revert. - const secondVersion = structuredClone(relabelledManager) - secondVersion.change.after.version.push({ - name: 'primary', - instance_template: 'projects/project/global/instanceTemplates/canary', - target_size: [{ fixed: 1 }] - }) - assert.throws( - () => validateCapacityPlan({ resource_changes: [template, secondVersion] }, sameCapConfig), - /outside the reviewed capacity fields/ - ) - // A stranded rollback whose template is already in place plans only the label revert. - const plannedCell = (startupScript, templateLink) => ({ - root_module: { - resources: [ - { - address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]', - values: { metadata_startup_script: startupScript, self_link: templateLink } - }, - { - address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]', - values: { version: [{ instance_template: templateLink, name: 'primary' }] } - } - ] - } - }) - const reviewedLink = 'projects/project/global/instanceTemplates/reviewed' - const labelOnly = { - address: manager.address, - change: { - actions: ['update'], - before: { - target_size: 1, - version: [{ instance_template: reviewedLink, name: '0/2026-10-01 10:41:28.681518+00:00' }] - }, - after: { target_size: 1, version: [{ instance_template: reviewedLink, name: 'primary' }] }, - after_unknown: {} - } - } - const strandedPlan = (managerChange) => ({ - resource_changes: [managerChange], - planned_values: plannedCell(template.change.after.metadata_startup_script, reviewedLink) - }) - assert.deepEqual( - validateCapacityPlan(strandedPlan(labelOnly), sameCapConfig), - { mode: 'same-cap-cell', changes: 1 } - ) - const labelAway = structuredClone(labelOnly) - labelAway.change.before.version[0].name = 'primary' - labelAway.change.after.version[0].name = 'other' - assert.throws( - () => validateCapacityPlan(strandedPlan(labelAway), sameCapConfig), - /outside the reviewed capacity fields/ - ) - const labelAndResize = structuredClone(labelOnly) - labelAndResize.change.after.target_size = 0 - assert.throws( - () => validateCapacityPlan(strandedPlan(labelAndResize), sameCapConfig), - /outside the reviewed capacity fields/ - ) - const percentSurge = structuredClone(relabelledManager) - percentSurge.change.before.update_policy = [{ ...declaredPolicy, max_surge_percent: 0 }] - percentSurge.change.after.update_policy = [{ ...declaredPolicy, max_surge_percent: 50 }] - assert.throws( - () => validateCapacityPlan({ resource_changes: [template, percentSurge] }, sameCapConfig), - /outside the reviewed capacity fields/ - ) - // A pre-template-apply rollback resume validates drift for the image the - // cell already serves: the template leaves and re-enters the rollback image. - const resumeTemplate = structuredClone(template) - resumeTemplate.change.after.metadata_startup_script = startup({ - selectedImage: rollbackImage, - trust: true - }) - assert.deepEqual( - validateCapacityPlan( - { resource_changes: [resumeTemplate, manager] }, - { ...sameCapConfig, image: rollbackImage } - ), - { mode: 'same-cap-cell', changes: 2 } - ) - const asiaTemplate = structuredClone(template) - const asiaManager = structuredClone(manager) - asiaTemplate.address = - 'google_compute_instance_template.relay_gce_cell["production-gce-c28"]' - asiaManager.address = - 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c28"]' - asiaTemplate.change.before.metadata_startup_script = startup({ - selectedImage: rollbackImage, - cap: 3_000 - }) - asiaTemplate.change.after.metadata_startup_script = startup({ - selectedImage: image, - cap: 3_000, - trust: true - }) - assert.deepEqual( - validateCapacityPlan( - { resource_changes: [asiaTemplate, asiaManager] }, - { ...sameCapConfig, cellId: 'production-gce-c28', hardCap: 3_000 } - ), - { mode: 'same-cap-cell', changes: 2 } - ) - const changedCap = structuredClone(template) - changedCap.change.before.metadata_startup_script = startup({ - selectedImage: rollbackImage, - cap: 600 - }) - assert.throws( - () => validateCapacityPlan({ resource_changes: [changedCap, manager] }, sameCapConfig), - /reviewed image and capacity/ - ) - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, manager] }, - { ...sameCapConfig, rollbackImage: image } - ), - /reviewed image and capacity/ - ) - const wrongTrust = structuredClone(template) - wrongTrust.change.after.metadata_startup_script = startup({ - selectedImage: image, - trust: true - }).replace(directorIdentity, 'other-director@project.iam.gserviceaccount.com') - assert.throws( - () => validateCapacityPlan({ resource_changes: [wrongTrust, manager] }, sameCapConfig), - /reviewed image and capacity/ - ) - - // Production run 35684694704 died after create_before_destroy made the new template, leaving - // the Sep 18 template deposed; every later wave plan for that cell carries its delete. - const deposedTemplate = structuredClone(template) - deposedTemplate.deposed = '4cb19f83' - deposedTemplate.change.actions = ['delete'] - deposedTemplate.change.after = null - deposedTemplate.change.after_unknown = {} - assert.deepEqual( - validateCapacityPlan( - { resource_changes: [template, manager, deposedTemplate] }, - sameCapConfig - ), - { mode: 'same-cap-cell', changes: 2, obsoleteTemplates: 1 } - ) - const updatedDeposedTemplate = structuredClone(deposedTemplate) - updatedDeposedTemplate.change.actions = ['update'] - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, manager, updatedDeposedTemplate] }, - sameCapConfig - ), - /change only the exact instance template and MIG/ - ) - const foreignDeposedTemplate = structuredClone(deposedTemplate) - foreignDeposedTemplate.address = - 'google_compute_instance_template.relay_gce_cell["production-gce-c8"]' - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, manager, foreignDeposedTemplate] }, - sameCapConfig - ), - /change only the exact instance template and MIG/ - ) - const secondDeposedTemplate = { ...structuredClone(deposedTemplate), deposed: 'aa17b204' } - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, manager, deposedTemplate, secondDeposedTemplate] }, - sameCapConfig - ), - /change only the exact instance template and MIG/ - ) - // Nothing but the deposed delete left: the cell is stranded on the reviewed template, and the - // job's `changes == 0` roll must still fire. - assert.deepEqual( - validateCapacityPlan( - { - resource_changes: [deposedTemplate], - planned_values: { - root_module: { - resources: [ - { - address: template.address, - values: { - metadata_startup_script: template.change.after.metadata_startup_script, - self_link: 'projects/project/global/instanceTemplates/target' - } - }, - { - address: manager.address, - values: { - version: [{ - instance_template: 'projects/project/global/instanceTemplates/target' - }] - } - } - ] - } - } - }, - sameCapConfig - ), - { mode: 'same-cap-cell', changes: 0, obsoleteTemplates: 1 } - ) - - const imageOnly = structuredClone(template) - imageOnly.change.after.metadata_startup_script = startup({ selectedImage: image }) - const imageOnlyConfig = { - ...config, - mode: 'same-cap-image', - image, - rollbackImage - } - assert.deepEqual( - validateCapacityPlan({ resource_changes: [imageOnly, manager] }, imageOnlyConfig), - { mode: 'same-cap-image', changes: 2, changeKind: 'replacement' } - ) - assert.throws( - () => validateCapacityPlan( - { resource_changes: [imageOnly, manager] }, - { ...imageOnlyConfig, rollbackImage: image } - ), - /reviewed image and capacity/ - ) - const changedTrust = structuredClone(imageOnly) - changedTrust.change.after.metadata_startup_script += - `\n printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'` - assert.throws( - () => validateCapacityPlan({ resource_changes: [changedTrust, manager] }, imageOnlyConfig), - /reviewed image and capacity/ - ) - const plannedValues = { - root_module: { - resources: [ - { - address: imageOnly.address, - values: { - metadata_startup_script: imageOnly.change.after.metadata_startup_script, - self_link: 'projects/project/global/instanceTemplates/target' - } - }, - { - address: manager.address, - values: { - version: [{ instance_template: 'projects/project/global/instanceTemplates/target' }] - } - } - ] - } - } - const obsoleteTemplate = structuredClone(imageOnly) - obsoleteTemplate.deposed = 'obsolete' - obsoleteTemplate.change.actions = ['delete'] - assert.deepEqual( - validateCapacityPlan( - { resource_changes: [obsoleteTemplate], planned_values: plannedValues }, - imageOnlyConfig - ), - { mode: 'same-cap-image', changes: 1, changeKind: 'obsolete-template-delete' } - ) - assert.deepEqual( - validateCapacityPlan( - { resource_changes: [imageOnly, manager, obsoleteTemplate] }, - imageOnlyConfig - ), - { mode: 'same-cap-image', changes: 3, changeKind: 'replacement-with-obsolete-template' } - ) - const anotherObsoleteTemplate = { - ...structuredClone(obsoleteTemplate), - deposed: 'another-obsolete' - } - assert.deepEqual( - validateCapacityPlan( - { resource_changes: [imageOnly, manager, obsoleteTemplate, anotherObsoleteTemplate] }, - imageOnlyConfig - ), - { mode: 'same-cap-image', changes: 4, changeKind: 'replacement-with-obsolete-template' } - ) - assert.deepEqual( - validateCapacityPlan( - { - resource_changes: [obsoleteTemplate, anotherObsoleteTemplate], - planned_values: plannedValues - }, - imageOnlyConfig - ), - { mode: 'same-cap-image', changes: 2, changeKind: 'obsolete-template-delete' } - ) - assert.deepEqual( - validateCapacityPlan( - { - resource_changes: [manager, obsoleteTemplate, anotherObsoleteTemplate], - planned_values: plannedValues - }, - imageOnlyConfig - ), - { mode: 'same-cap-image', changes: 3, changeKind: 'manager-convergence' } - ) - const invalidObsoleteTemplate = structuredClone(anotherObsoleteTemplate) - invalidObsoleteTemplate.change.actions = ['update'] - assert.throws( - () => validateCapacityPlan( - { resource_changes: [imageOnly, manager, obsoleteTemplate, invalidObsoleteTemplate] }, - imageOnlyConfig - ), - /change only the exact instance template and MIG/ - ) - assert.deepEqual( - validateCapacityPlan( - { resource_changes: [manager], planned_values: plannedValues }, - imageOnlyConfig - ), - { mode: 'same-cap-image', changes: 1, changeKind: 'manager-convergence' } - ) -}) - -test('protocol-0 same-cap cells roll without rehome trust lines', () => { - const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}` - const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}` - const directorIdentity = 'relay-director@project.iam.gserviceaccount.com' - const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com' - const audience = 'https://relay.example.com/v1/admin/host-drain' - const startup = ({ selectedImage, trust = false }) => [ - ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`, - ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`, - ` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`, - ` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`, - ...(trust ? [ - ` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`, - ` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'` - ] : []), - `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`, - `docker pull '${selectedImage}'`, - 'docker run --detach \\', - ' --name orca-relay \\', - ` '${selectedImage}'` - ].join('\n') - const template = { - address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]', - change: { - actions: ['create', 'delete'], - before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) }, - after: { metadata_startup_script: startup({ selectedImage: image }), self_link: null }, - after_unknown: { self_link: true } - } - } - const manager = { - address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]', - change: { - actions: ['update'], - before: { target_size: 1, version: [{ instance_template: 'old' }] }, - after: { target_size: 1, version: [{ instance_template: null }] }, - after_unknown: { version: [{ instance_template: true }] } - } - } - const asiaConfig = { - cellId: 'production-gce-c27', - hardCap: 3_000, - unobservedBound: 60, - mode: 'same-cap-cell', - image, - rollbackImage, - capacityServiceAccount: capacityIdentity, - rehomeDirectorServiceAccount: directorIdentity, - rehomeAudience: audience, - regionalRehomeProtocol: '0' - } - assert.deepEqual( - validateCapacityPlan({ resource_changes: [template, manager] }, asiaConfig), - { mode: 'same-cap-cell', changes: 2 } - ) - const gainsTrust = structuredClone(template) - gainsTrust.change.after.metadata_startup_script = startup({ - selectedImage: image, - trust: true - }) - assert.throws( - () => validateCapacityPlan({ resource_changes: [gainsTrust, manager] }, asiaConfig), - /reviewed image and capacity/ - ) - // Under protocol 1 that same script is the reviewed roll: trust is added, not drift. - assert.deepEqual( - validateCapacityPlan( - { resource_changes: [gainsTrust, manager] }, - { ...asiaConfig, regionalRehomeProtocol: '1' } - ), - { mode: 'same-cap-cell', changes: 2 } - ) - // A protocol-1 cell whose script has no rehome lines is the pre-existing failure, unchanged. - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, manager] }, - { ...asiaConfig, regionalRehomeProtocol: '1' } - ), - /reviewed image and capacity/ - ) - for (const protocol of [undefined, '', '2', 'yes']) { - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, manager] }, - { ...asiaConfig, regionalRehomeProtocol: protocol } - ), - /invalid regional rehome protocol/ - ) - } -}) - -test('a same-cap roll may gain the pinned capacity identity but never move it', () => { - const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}` - const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}` - const directorIdentity = 'relay-director@project.iam.gserviceaccount.com' - const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com' - const audience = 'https://relay.example.com/v1/admin/host-drain' - const startup = ({ selectedImage, capacity }) => [ - ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`, - ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`, - ...(capacity === null - ? [] - : [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]), - `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`, - `docker pull '${selectedImage}'`, - 'docker run --detach \\', - ' --name orca-relay \\', - ` '${selectedImage}'` - ].join('\n') - const plan = (beforeCapacity, afterCapacity) => ({ - resource_changes: [ - { - address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]', - change: { - actions: ['create', 'delete'], - before: { - metadata_startup_script: startup({ - selectedImage: rollbackImage, - capacity: beforeCapacity - }) - }, - after: { - metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }), - self_link: null - }, - after_unknown: { self_link: true } - } - }, - { - address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]', - change: { - actions: ['update'], - before: { target_size: 1, version: [{ instance_template: 'old' }] }, - after: { target_size: 1, version: [{ instance_template: null }] }, - after_unknown: { version: [{ instance_template: true }] } - } - } - ] - }) - const config = { - cellId: 'production-gce-c17', - hardCap: 600, - unobservedBound: 60, - mode: 'same-cap-cell', - image, - rollbackImage, - capacityServiceAccount: capacityIdentity, - rehomeDirectorServiceAccount: directorIdentity, - rehomeAudience: audience, - regionalRehomeProtocol: '0' - } - // A template old enough to predate the line gains it, which is the only move allowed. - assert.deepEqual( - validateCapacityPlan(plan(null, capacityIdentity), config), - { mode: 'same-cap-cell', changes: 2 } - ) - assert.deepEqual( - validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config), - { mode: 'same-cap-cell', changes: 2 } - ) - for (const [before, after] of [ - [capacityIdentity, null], - [null, null], - [capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'], - [null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'] - ]) { - assert.throws( - () => validateCapacityPlan(plan(before, after), config), - /reviewed image and capacity/, - `${before} -> ${after}` - ) - } - // Without the pin there is nothing reviewing the line the comparison now ignores. - assert.throws( - () => validateCapacityPlan(plan(null, capacityIdentity), { - ...config, - capacityServiceAccount: undefined - }), - /invalid service account/ - ) - assert.throws( - () => validateCapacityPlan(plan(null, capacityIdentity), { - ...config, - capacityServiceAccount: 'not-an-email' - }), - /invalid service account/ - ) -}) - -test('the capacity identity argument is required by same-cap-cell mode', () => { - const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}` - const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}` - const base = [ - '--mode', 'same-cap-cell', - '--cell-id', 'production-gce-c17', - '--hard-cap', '600', - '--unobserved-bound', '60', - '--image', image, - '--rollback-image', rollbackImage, - '--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com', - '--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain', - '--regional-rehome-protocol', '0' - ] - assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/) - assert.throws( - () => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']), - /--capacity-service-account is invalid/ - ) - assert.equal( - parseCapacityPlanArguments([ - ...base, - '--capacity-service-account', - 'orca-cloud-gha-cap@project.iam.gserviceaccount.com' - ]).capacityServiceAccount, - 'orca-cloud-gha-cap@project.iam.gserviceaccount.com' - ) -}) - -test('the rehome protocol argument is required by same-cap-cell mode alone', () => { - const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}` - const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}` - const sameCapArguments = (...extra) => [ - '--mode', 'same-cap-cell', - '--cell-id', 'production-gce-c27', - '--hard-cap', '3000', - '--unobserved-bound', '60', - '--image', image, - '--rollback-image', rollbackImage, - '--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com', - '--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com', - '--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain', - ...extra - ] - assert.equal( - parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '0')) - .regionalRehomeProtocol, - '0' - ) - assert.equal( - parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '3')).regionalRehomeProtocol, - '3' - ) - assert.throws( - () => parseCapacityPlanArguments(sameCapArguments()), - /requires rollback image and rehome trust config/ - ) - for (const protocol of ['', '2', 'true']) { - assert.throws( - () => parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', protocol)), - /requires rollback image and rehome trust config/ - ) - } - assert.throws( - () => parseCapacityPlanArguments([ - '--mode', 'bootstrap-cell', - '--cell-id', 'staging-gce-c3', - '--hard-cap', '1000', - '--unobserved-bound', '60', - '--image', image, - '--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com', - '--regional-rehome-protocol', '0' - ]), - /applies only to same-cap-cell validation/ - ) -}) - -test('the reviewed database pool is pinned for the cells that emit one', () => { - const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}` - const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}` - const directorIdentity = 'relay-director@project.iam.gserviceaccount.com' - const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com' - const audience = 'https://relay.example.com/v1/admin/host-drain' - const startup = ({ selectedImage, pool }) => [ - ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`, - ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`, - ` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`, - ...(pool === undefined - ? [] - : [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]), - ` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`, - ` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`, - `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`, - `docker pull '${selectedImage}'`, - 'docker run --detach \\', - ' --name orca-relay \\', - ` '${selectedImage}'` - ].join('\n') - const rollPlan = (before, after) => ({ - resource_changes: [ - { - address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]', - change: { - actions: ['create', 'delete'], - before: { - metadata_startup_script: startup({ selectedImage: rollbackImage, pool: before }) - }, - after: { - metadata_startup_script: startup({ selectedImage: image, pool: after }), - self_link: null - }, - after_unknown: { self_link: true } - } - }, - { - address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]', - change: { - actions: ['update'], - before: { target_size: 1, version: [{ instance_template: 'old' }] }, - after: { target_size: 1, version: [{ instance_template: null }] }, - after_unknown: { version: [{ instance_template: true }] } - } - } - ] - }) - const asiaConfig = { - cellId: 'production-gce-c27', - hardCap: 3_000, - unobservedBound: 60, - mode: 'same-cap-cell', - image, - rollbackImage, - capacityServiceAccount: capacityIdentity, - rehomeDirectorServiceAccount: directorIdentity, - rehomeAudience: audience, - regionalRehomeProtocol: '1' - } - // The live template still says 10 while the reviewed plan says 16; only the pin bridges that. - assert.deepEqual( - validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: '16' }), - { mode: 'same-cap-cell', changes: 2 } - ) - assert.throws( - () => validateCapacityPlan(rollPlan('10', '16'), asiaConfig), - /reviewed image and capacity/ - ) - assert.throws( - () => validateCapacityPlan(rollPlan('10', '12'), { ...asiaConfig, databasePoolMax: '16' }), - /reviewed image and capacity/ - ) - // A cell on the root pool default emits no line at all, and gaining one is real drift. - assert.deepEqual( - validateCapacityPlan(rollPlan(undefined, undefined), asiaConfig), - { mode: 'same-cap-cell', changes: 2 } - ) - assert.throws( - () => validateCapacityPlan(rollPlan(undefined, '16'), asiaConfig), - /reviewed image and capacity/ - ) - // A line already on the live template is still unreviewed without the pin, even standing still. - assert.throws( - () => validateCapacityPlan(rollPlan('16', '16'), asiaConfig), - /reviewed image and capacity/ - ) - // A pin must also fail closed when the plan drops the line it names. - assert.throws( - () => validateCapacityPlan(rollPlan('16', undefined), { ...asiaConfig, databasePoolMax: '16' }), - /reviewed image and capacity/ - ) - for (const pool of ['', '0', '101', 'ten']) { - assert.throws( - () => validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: pool }), - /invalid database pool max/ - ) - } -}) - -test('the database pool argument is accepted by same-cap-cell mode alone', () => { - const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}` - const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}` - const sameCapArguments = (...extra) => [ - '--mode', 'same-cap-cell', - '--cell-id', 'production-gce-c27', - '--hard-cap', '3000', - '--unobserved-bound', '60', - '--image', image, - '--rollback-image', rollbackImage, - '--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com', - '--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com', - '--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain', - '--regional-rehome-protocol', '1', - ...extra - ] - assert.equal( - parseCapacityPlanArguments(sameCapArguments('--database-pool-max', '16')).databasePoolMax, - '16' - ) - assert.equal(parseCapacityPlanArguments(sameCapArguments()).databasePoolMax, undefined) - assert.throws( - () => parseCapacityPlanArguments([ - '--mode', 'bootstrap-cell', - '--cell-id', 'staging-gce-c3', - '--hard-cap', '1000', - '--unobserved-bound', '60', - '--image', image, - '--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com', - '--database-pool-max', '16' - ]), - /applies only to same-cap-cell validation/ - ) -}) - -test('a same-cap roll may carry only this cell backend drain and request logging', () => { - const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}` - const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}` - const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com' - const startup = (selectedImage) => [ - " printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '1000'", - " printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'", - ` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`, - `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`, - `docker pull '${selectedImage}'`, - 'docker run --detach \\', - ' --name orca-relay \\', - ` '${selectedImage}'` - ].join('\n') - const template = { - address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]', - change: { - actions: ['create', 'delete'], - before: { metadata_startup_script: startup(rollbackImage) }, - after: { metadata_startup_script: startup(image), self_link: null }, - after_unknown: { self_link: true } - } - } - const manager = { - address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]', - change: { - actions: ['update'], - before: { target_size: 1, version: [{ instance_template: 'old' }] }, - after: { target_size: 1, version: [{ instance_template: null }] }, - after_unknown: { version: [{ instance_template: true }] } - } - } - // The exact shape a live US cell's backend plans: 300 s drain and no log_config at all. - const loggingAfter = [{ - enable: true, - optional_fields: null, - optional_mode: null, - sample_rate: RELAY_CELL_LOG_SAMPLE_RATE - }] - const backendChange = ({ drain = true, logging = true }) => ({ - address: 'google_compute_backend_service.relay_gce_cell["staging-gce-c3"]', - change: { - actions: ['update'], - before: { - connection_draining_timeout_sec: drain ? 300 : RELAY_CELL_CONNECTION_DRAIN_SECONDS, - log_config: logging ? [] : loggingAfter, - timeout_sec: 86_400, - fingerprint: 'before' - }, - after: { - connection_draining_timeout_sec: RELAY_CELL_CONNECTION_DRAIN_SECONDS, - log_config: loggingAfter, - timeout_sec: 86_400, - fingerprint: null - }, - after_unknown: { fingerprint: true } - } - }) - const backend = backendChange({}) - const drainOnly = backendChange({ logging: false }) - const loggingOnly = backendChange({ drain: false }) - const sameCapConfig = { - ...config, - mode: 'same-cap-cell', - image, - rollbackImage, - capacityServiceAccount: capacityIdentity, - rehomeDirectorServiceAccount: 'relay-director@project.iam.gserviceaccount.com', - rehomeAudience: 'https://relay.onorca.dev/v1/admin/host-drain', - regionalRehomeProtocol: '0' - } - const refused = /only this cell backend drain and request logging/ - // Both attributes ride along with the roll without inflating the template-and-MIG count - // the apply's stranded branch and the resume's drift branch both read. - assert.deepEqual( - validateCapacityPlan({ resource_changes: [template, manager, backend] }, sameCapConfig), - { - mode: 'same-cap-cell', - changes: 2, - backendUpdate: ['connection_draining_timeout_sec', 'log_config.0'] - } - ) - // Each is independently optional: a cell that already has one plans no change for it. - assert.deepEqual( - validateCapacityPlan({ resource_changes: [template, manager, drainOnly] }, sameCapConfig), - { mode: 'same-cap-cell', changes: 2, backendUpdate: ['connection_draining_timeout_sec'] } - ) - assert.deepEqual( - validateCapacityPlan({ resource_changes: [template, manager, loggingOnly] }, sameCapConfig), - { mode: 'same-cap-cell', changes: 2, backendUpdate: ['log_config.0'] } - ) - // Once both are applied the backend is simply absent from the plan. - assert.deepEqual( - validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig), - { mode: 'same-cap-cell', changes: 2 } - ) - // A cell whose template and MIG have converged but whose backend has not is still clean. - assert.deepEqual( - validateCapacityPlan({ resource_changes: [backend] }, sameCapConfig), - { - mode: 'same-cap-cell', - changes: 0, - backendUpdate: ['connection_draining_timeout_sec', 'log_config.0'] - } - ) - assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, sameCapConfig), { - mode: 'same-cap-cell', - changes: 0 - }) - const extraAttribute = structuredClone(backend) - extraAttribute.change.after.timeout_sec = 3_600 - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, manager, extraAttribute] }, - sameCapConfig - ), - /changes outside the reviewed capacity fields/ - ) - const otherCell = structuredClone(backend) - otherCell.address = 'google_compute_backend_service.relay_gce_cell["production-gce-c27"]' - assert.throws( - () => validateCapacityPlan({ resource_changes: [template, manager, otherCell] }, sameCapConfig), - refused - ) - const unreviewedDrain = structuredClone(backend) - unreviewedDrain.change.after.connection_draining_timeout_sec = - RELAY_CELL_CONNECTION_DRAIN_SECONDS + 1 - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, manager, unreviewedDrain] }, - sameCapConfig - ), - refused - ) - const sampledLogging = structuredClone(backend) - sampledLogging.change.after.log_config[0].sample_rate = RELAY_CELL_LOG_SAMPLE_RATE / 2 - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, manager, sampledLogging] }, - sameCapConfig - ), - refused - ) - const disabledLogging = structuredClone(backend) - disabledLogging.change.after.log_config[0].enable = false - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, manager, disabledLogging] }, - sameCapConfig - ), - refused - ) - const replaced = structuredClone(backend) - replaced.change.actions = ['create', 'delete'] - assert.throws( - () => validateCapacityPlan({ resource_changes: [template, manager, replaced] }, sameCapConfig), - refused - ) - // Only the same-cap wave targets a backend service; every other mode still refuses one. - assert.throws( - () => validateCapacityPlan( - { resource_changes: [template, manager, backend] }, - { ...config, mode: 'cell', image } - ), - /only the exact instance template and MIG/ - ) -}) - -// The reconciliation allowance is only safe while it names exactly what Terraform declares. -test('the declared MIG fields match relay-gce-cells.tf', () => { - const terraform = readFileSync( - new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url), - 'utf8' - ) - const manager = terraform - .split('resource "google_compute_instance_group_manager" "relay_gce_cell" {')[1] - ?.split('\n}')[0] ?? '' - const block = (name) => manager.split(` ${name} {`)[1]?.split('\n }')[0] ?? '' - const topology = terraform.split(' relay_gce_topology = {')[1]?.split('\n }')[0] ?? '' - const attribute = (source, name) => { - const raw = new RegExp(`\\n\\s+${name}\\s+= (.+)`).exec(source)?.[1] - assert.notEqual(raw, undefined, `relay-gce-cells.tf declares no ${name}`) - const local = /^local\.relay_gce_topology\.(\w+)$/.exec(raw)?.[1] - if (local) return Number(new RegExp(`${local}\\s+= (\\d+)`).exec(topology)?.[1]) - return JSON.parse(raw) - } - const declared = Object.fromEntries( - Object.keys(DECLARED_MANAGER_FIELDS).map((path) => { - const [blockName, , field] = path.split('.') - return [path, attribute(block(blockName), field)] - }) - ) - assert.deepEqual(declared, { ...DECLARED_MANAGER_FIELDS }) -}) diff --git a/cloud/dev/scripts/verify-relay-capacity-transition.mjs b/cloud/dev/scripts/verify-relay-capacity-transition.mjs deleted file mode 100644 index 9e0ce7a71ac..00000000000 --- a/cloud/dev/scripts/verify-relay-capacity-transition.mjs +++ /dev/null @@ -1,537 +0,0 @@ -import { pathToFileURL } from 'node:url' -import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' - -const CAPACITY_PROTOCOL = 2 -const POLL_INTERVAL_MS = 5_000 - -function integer(value, name) { - const parsed = Number(value) - if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${name} is invalid`) - return parsed -} - -function signedInteger(value, name) { - if (typeof value !== 'number' || !Number.isSafeInteger(value)) { - throw new Error(`${name} is invalid`) - } - return value -} - -export function parseCapacityTransitionArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of [ - 'director-origin', - 'cell-origin', - 'cell-id', - 'heartbeat', - 'admission', - 'draining', - 'activity' - ]) { - if (!values[key]) throw new Error(`missing --${key}`) - } - if (!['fresh', 'stale', 'either'].includes(values.heartbeat)) { - throw new Error('--heartbeat must be fresh, stale, or either') - } - if ( - !['general', 'migration-only', 'general-or-migration-only', 'non-general', 'either'].includes( - values.admission - ) - ) { - throw new Error( - '--admission must be general, migration-only, general-or-migration-only, non-general, or either' - ) - } - if (!['required', 'forbidden', 'either'].includes(values.draining)) { - throw new Error('--draining must be required, forbidden, or either') - } - if (!['quiescent', 'restart-safe', 'allowed'].includes(values.activity)) { - throw new Error('--activity must be quiescent, restart-safe, or allowed') - } - const runtime = values.runtime ?? 'required' - if (!['required', 'unavailable'].includes(runtime)) { - throw new Error('--runtime must be required or unavailable') - } - if ( - values.activity === 'restart-safe' && - runtime === 'required' && - (values.admission !== 'migration-only' || values.draining !== 'required') - ) { - throw new Error('restart-safe activity requires migration-only admission and draining') - } - if ( - runtime === 'unavailable' && - (values.heartbeat !== 'stale' || - values.admission !== 'migration-only' || - values.draining !== 'either' || - values.activity !== 'restart-safe') - ) { - throw new Error('unavailable runtime requires stale migration-only durable state') - } - const origin = new URL(values['director-origin']) - const cellOrigin = new URL(values['cell-origin']) - if ( - origin.protocol !== 'https:' || - origin.origin !== values['director-origin'] || - cellOrigin.protocol !== 'https:' || - cellOrigin.origin !== values['cell-origin'] - ) { - throw new Error('origins must be canonical HTTPS origins') - } - const hardCap = values['hard-cap'] === undefined - ? undefined - : integer(values['hard-cap'], '--hard-cap') - const unobservedBound = values['unobserved-bound'] === undefined - ? undefined - : integer(values['unobserved-bound'], '--unobserved-bound') - if ((hardCap === undefined) !== (unobservedBound === undefined)) { - throw new Error('capacity expectations must be paired') - } - if (runtime === 'unavailable' && hardCap !== undefined) { - throw new Error('unavailable runtime cannot prove live capacity') - } - const expectedImageDigests = values['expected-image-digests']?.split(',') ?? [] - if ( - new Set(expectedImageDigests).size !== expectedImageDigests.length || - expectedImageDigests.some((digest) => !/^sha256:[a-f0-9]{64}$/.test(digest)) - ) { - throw new Error('--expected-image-digests is invalid') - } - if (runtime === 'unavailable' && expectedImageDigests.length > 0) { - throw new Error('unavailable runtime cannot prove a live image') - } - const regionalRehomeProtocol = values['regional-rehome-protocol'] === undefined - ? undefined - : integer(values['regional-rehome-protocol'], '--regional-rehome-protocol') - if (regionalRehomeProtocol !== undefined && ![0, 1, 3].includes(regionalRehomeProtocol)) { - throw new Error('--regional-rehome-protocol must be 0, 1, or 3') - } - if (runtime === 'unavailable' && regionalRehomeProtocol !== undefined) { - throw new Error('unavailable runtime cannot prove the regional rehome protocol') - } - const paceWindowMs = values['pace-window-ms'] === undefined - ? undefined - : integer(values['pace-window-ms'], '--pace-window-ms') - const liveRestartSafe = values.activity === 'restart-safe' && runtime === 'required' - if ((paceWindowMs !== undefined) !== liveRestartSafe) { - throw new Error('--pace-window-ms is required exactly for restart-safe activity on a live runtime') - } - return { - directorOrigin: origin.origin, - cellOrigin: cellOrigin.origin, - cellId: values['cell-id'], - heartbeat: values.heartbeat, - admission: values.admission, - draining: values.draining, - activity: values.activity, - runtime, - expectedImageDigests, - ...(regionalRehomeProtocol === undefined ? {} : { regionalRehomeProtocol }), - ...(paceWindowMs === undefined ? {} : { paceWindowMs }), - hardCap, - unobservedBound, - timeoutMs: integer(values['timeout-ms'] ?? 180_000, '--timeout-ms') - } -} - -async function responseJson(response, label) { - const body = await response.json().catch(() => ({})) - if (!response.ok) throw new Error(`${label} returned ${response.status}`) - return body -} - -async function cellRuntime(fetchImpl, config, token) { - let response - try { - response = await fetchImpl(`${config.cellOrigin}/v1/admin/runtime-status`, { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify({ v: 1 }), - signal: AbortSignal.timeout(30_000) - }) - } catch (error) { - if (config.runtime === 'unavailable') return null - throw error - } - if ([502, 503, 504].includes(response.status)) { - await response.arrayBuffer().catch(() => undefined) - return null - } - return await responseJson(response, 'cell runtime status') -} - -function offlineRollbackMatches(status, config) { - if (status.admissionState !== 'migration-only') { - throw new Error('capacity transition admission does not match the required state') - } - const durableCounts = [ - status.activityLeases, - status.activityRequestUnits, - status.reservedRequests, - status.restartBlockingActivityLeases, - status.restartBlockingActivityRequestUnits, - status.outgoingMigrations, - status.incomingMigrations, - status.connectionCapacity?.pendingControlReservations - ] - const restartBlockingReservedRequests = signedInteger( - status.restartBlockingReservedRequests, - 'restart-blocking reserved requests' - ) - // Only a positive remainder is unexplained; the other gates reject real work. - return heartbeatMatches(status, config.heartbeat) && - durableCounts.every((value) => integer(value, 'durable activity count') === 0) && - restartBlockingReservedRequests <= 0 -} - -function directorActivityMatches(status, config) { - const durable = [ - status.activityLeases, - status.reservedRequests, - status.outgoingMigrations, - status.incomingMigrations - ] - // Reconnect reservations survive replacement; draining prevents activation on this process. - const transient = [ - status.connectionCapacity?.observedConnections, - status.connectionCapacity?.inFlightConnections, - status.connectionCapacity?.reservedConnectionUnits, - status.connectionCapacity?.enforcedConnectionUnits, - status.connectionCapacity?.pendingControlReservations - ] - const restartSafe = config.activity !== 'restart-safe' || (() => { - // Leases lag hosts that left or cannot be placed, so the runtime's counters gate instead; - // still validated so a malformed director fails closed. - integer(status.restartBlockingActivityLeases, 'restart-blocking activity leases') - integer( - status.restartBlockingActivityRequestUnits, - 'restart-blocking activity request units' - ) - // Open migrations are pinned to this cell's incarnation, which a restart replaces. - return integer(status.outgoingMigrations, 'outgoing migrations') === 0 && - integer(status.incomingMigrations, 'incoming migrations') === 0 - })() - const quiescent = [...durable, ...transient] - .filter((value) => value !== undefined) - .every((value) => integer(value, 'activity count') === 0) - if ( - (config.admission === 'general' && status.admissionState !== 'general') || - (config.admission === 'migration-only' && status.admissionState !== 'migration-only') || - // A failed same-cap canary leaves its cell migration-only; the documented - // rollback recovery must accept that state alongside a completed general roll. - (config.admission === 'general-or-migration-only' && - !['general', 'migration-only'].includes(status.admissionState)) || - (config.admission === 'non-general' && - !['existing-only', 'migration-only'].includes(status.admissionState)) - ) { - throw new Error('capacity transition admission does not match the required state') - } - return config.activity === 'allowed' || - (config.activity === 'restart-safe' ? restartSafe : quiescent) -} - -function capacityMatches(status, config) { - if (config.hardCap === undefined) return true - const capacity = status.connectionCapacity - return ( - capacity?.hardCap === config.hardCap && - capacity.unobservedBound === config.unobservedBound && - capacity.controlRebindReserve === 100 && - capacity.ordinaryConnectionLimit === config.hardCap - 100 && - capacity.normalAdmissionPause === config.hardCap - 100 - config.unobservedBound - ) -} - -function heartbeatMatches(status, expectation) { - if (expectation === 'either') return true - const fresh = status.connectionCapacity?.heartbeatFresh ?? status.runtime?.heartbeatFresh - return fresh === (expectation === 'fresh') -} - -function aggregateCount(value) { - const parsed = Number(value) - return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null -} - -function signedAggregateCount(value) { - return typeof value === 'number' && Number.isSafeInteger(value) ? value : null -} - -function capacityObservation(capacity) { - if (capacity === null || capacity === undefined) return null - return { - hardCap: aggregateCount(capacity.hardCap), - controlRebindReserve: aggregateCount(capacity.controlRebindReserve), - ordinaryConnectionLimit: aggregateCount(capacity.ordinaryConnectionLimit), - unobservedBound: aggregateCount(capacity.unobservedBound), - normalAdmissionPause: aggregateCount(capacity.normalAdmissionPause), - observedConnections: aggregateCount(capacity.observedConnections), - inFlightConnections: aggregateCount(capacity.inFlightConnections), - reservedConnectionUnits: aggregateCount(capacity.reservedConnectionUnits), - enforcedConnectionUnits: aggregateCount(capacity.enforcedConnectionUnits), - pendingControlReservations: aggregateCount(capacity.pendingControlReservations), - heartbeatFresh: typeof capacity.heartbeatFresh === 'boolean' - ? capacity.heartbeatFresh - : null - } -} - -function transitionObservation(runtime, status) { - return { - runtimeAvailable: runtime !== null, - admissionState: ['general', 'migration-only', 'existing-only'].includes(status.admissionState) - ? status.admissionState - : null, - draining: typeof runtime?.draining === 'boolean' ? runtime.draining : null, - runtime: runtime === null - ? null - : { - totalConnections: aggregateCount(runtime.runtime?.totalConnections), - preAuthConnections: aggregateCount(runtime.runtime?.preAuthConnections), - inFlightConnections: aggregateCount(runtime.runtime?.inFlightConnections), - reservedConnectionUnits: aggregateCount(runtime.runtime?.reservedConnectionUnits), - enforcedConnectionUnits: aggregateCount(runtime.runtime?.enforcedConnectionUnits), - controls: aggregateCount(runtime.runtime?.controls), - splices: aggregateCount(runtime.runtime?.splices), - pendingSplices: aggregateCount(runtime.runtime?.pendingSplices), - queuedBytes: aggregateCount(runtime.runtime?.queuedBytes) - }, - director: { - activityLeases: aggregateCount(status.activityLeases), - activityRequestUnits: aggregateCount(status.activityRequestUnits), - reservedRequests: aggregateCount(status.reservedRequests), - restartBlockingActivityLeases: aggregateCount(status.restartBlockingActivityLeases), - restartBlockingActivityRequestUnits: - aggregateCount(status.restartBlockingActivityRequestUnits), - restartBlockingReservedRequests: - signedAggregateCount(status.restartBlockingReservedRequests), - outgoingMigrations: aggregateCount(status.outgoingMigrations), - incomingMigrations: aggregateCount(status.incomingMigrations) - }, - runtimeCapacity: capacityObservation(runtime?.connectionCapacity), - directorCapacity: capacityObservation(status.connectionCapacity), - runtimeHeartbeatFresh: typeof status.runtime?.heartbeatFresh === 'boolean' - ? status.runtime.heartbeatFresh - : null - } -} - -// Director records of hosts that left or cannot be placed; reported, never gated on. -function strandedObservation(observation) { - const director = observation.director - return director === undefined - ? null - : { - restartBlockingActivityLeases: director.restartBlockingActivityLeases, - restartBlockingActivityRequestUnits: director.restartBlockingActivityRequestUnits, - restartBlockingReservedRequests: director.restartBlockingReservedRequests, - outgoingMigrations: director.outgoingMigrations, - incomingMigrations: director.incomingMigrations - } -} - -function runtimeQuiescent(runtime, config) { - if ( - runtime.role !== 'cell' || - runtime.cellId !== config.cellId || - runtime.cellUrl !== config.cellOrigin || - // Legacy pre-rehome images omit the field; the exact digest binds absence to protocol 0. - (config.regionalRehomeProtocol !== undefined && - (runtime.regionalRehomeProtocol ?? 0) !== config.regionalRehomeProtocol) || - (config.expectedImageDigests?.length > 0 && - !config.expectedImageDigests.includes(runtime.imageDigest)) - ) { - throw new Error('capacity transition runtime does not match the cell') - } - if ( - (config.draining === 'required' && runtime.draining !== true) || - (config.draining === 'forbidden' && runtime.draining === true) - ) { - return false - } - const counts = [runtime.runtime?.totalConnections, runtime.runtime?.preAuthConnections] - if (counts.some((value) => value === undefined)) { - throw new Error('capacity transition runtime is incomplete') - } - if (runtime.connectionCapacity !== null && runtime.connectionCapacity !== undefined) { - if (runtime.runtime?.enforcedConnectionUnits === undefined) { - throw new Error('capacity transition runtime is incomplete') - } - counts.push(runtime.runtime.enforcedConnectionUnits) - } - const quiescent = counts.every((value) => integer(value, 'runtime connection count') === 0) - // Total and pre-auth connections also count unauthenticated redials, which never stop while - // hosts have nowhere else to go and lose nothing on a restart. - const liveIdle = (values) => - values.every((value) => integer(value, 'live runtime count') === 0) - // Draining refuses every control and host proof, so with no session left an in-flight or - // reserved unit can only belong to a handshake the cell is about to refuse. - const restartSafe = config.activity !== 'restart-safe' || ( - liveIdle([ - runtime.runtime?.controls, - runtime.runtime?.splices, - runtime.runtime?.pendingSplices, - runtime.runtime?.queuedBytes - ]) && - // Still validated on a draining cell so a malformed runtime fails closed. - (liveIdle([runtime.runtime?.inFlightConnections, runtime.runtime?.reservedConnectionUnits]) || - runtime.draining === true) - ) - if ( - config.heartbeat === 'fresh' && - !capacityMatches({ connectionCapacity: runtime.connectionCapacity }, config) - ) { - return false - } - return config.activity === 'allowed' || - (config.activity === 'restart-safe' ? restartSafe : quiescent) -} - -export async function verifyCapacityTransition(config, overrides = {}) { - if ( - config.activity === 'restart-safe' && - config.runtime !== 'unavailable' && - (config.admission !== 'migration-only' || config.draining !== 'required') - ) { - throw new Error('restart-safe activity requires migration-only admission and draining') - } - const fetchImpl = overrides.fetch ?? fetch - const progress = overrides.progress ?? - ((line) => process.stdout.write(`${JSON.stringify(line)}\n`)) - const wait = overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) - const now = overrides.now ?? Date.now - const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN - if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') - const health = await responseJson( - await fetchAdminOnceMore( - fetchImpl, - `${config.directorOrigin}/health`, - {}, - { wait, timeoutMs: 15_000 } - ), - 'director health' - ) - if (health.ok !== true || health.connectionCapacityProtocol !== CAPACITY_PROTOCOL) { - throw new Error('director is not capacity-protocol compatible') - } - const startedAt = now() - const deadline = startedAt + config.timeoutMs - // A paced drain can refill the runtime until its window ends, so the second of two consecutive - // quiet samples must be taken at or after that end. This runs after the cell accepted the drain, - // so the cell's window ends no later than this one. - const paceWindowEndsAt = startedAt + (config.paceWindowMs ?? 0) - let restartSafeSamples = 0 - let lastObservation = { runtimeAvailable: false } - for (;;) { - // Before the read, so a sample taken inside the window cannot pass. - const sampledAt = now() - const runtime = await cellRuntime(fetchImpl, config, token) - lastObservation = { runtimeAvailable: runtime !== null } - if ((runtime === null) === (config.runtime === 'unavailable')) { - const result = await responseJson( - await fetchAdminOnceMore( - fetchImpl, - `${config.directorOrigin}/v1/admin/cell-status`, - { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify({ v: 1, cellId: config.cellId }) - }, - { wait } - ), - 'cell status' - ) - const status = result.status - if ( - status?.cellId !== config.cellId || - status.cellUrl !== config.cellOrigin || - status.runtime?.cellUrl !== config.cellOrigin - ) { - throw new Error('capacity transition director status does not match the cell') - } - lastObservation = transitionObservation(runtime, status) - const restartBlockingReservedRequests = - config.activity === 'restart-safe' - ? signedInteger( - status.restartBlockingReservedRequests, - 'restart-blocking reserved requests' - ) - : null - const matches = runtime === null - ? offlineRollbackMatches(status, config) - : runtimeQuiescent(runtime, config) && - directorActivityMatches(status, config) && - capacityMatches(status, config) && - heartbeatMatches(status, config.heartbeat) - if ( - matches && - (config.activity !== 'restart-safe' || - (restartSafeSamples >= 1 && sampledAt >= paceWindowEndsAt)) - ) { - return { - cellId: status.cellId, - admissionState: status.admissionState, - assignments: integer(status.assignments, 'assignments'), - hardCap: runtime === null ? null : status.connectionCapacity?.hardCap ?? null, - unobservedBound: - runtime === null ? null : status.connectionCapacity?.unobservedBound ?? null, - heartbeatFresh: - status.connectionCapacity?.heartbeatFresh ?? status.runtime?.heartbeatFresh ?? false, - imageDigest: runtime?.imageDigest ?? null, - ...(config.activity === 'restart-safe' - ? { restartBlockingReservedRequests, stranded: strandedObservation(lastObservation) } - : {}) - } - } - restartSafeSamples = matches ? restartSafeSamples + 1 : 0 - } else { - restartSafeSamples = 0 - } - if (config.activity === 'restart-safe') { - const paceWindowRemainingMs = Math.max(paceWindowEndsAt - sampledAt, 0) - lastObservation = { - ...lastObservation, - restartSafeSamples, - paceWindowRemainingMs - } - progress({ - event: 'relay_capacity_transition_restart_progress', - restartSafeSamples, - paceWindowRemainingMs, - totalConnections: lastObservation.runtime?.totalConnections ?? null, - preAuthConnections: lastObservation.runtime?.preAuthConnections ?? null, - runtimeAvailable: lastObservation.runtimeAvailable, - admissionState: lastObservation.admissionState ?? null, - draining: lastObservation.draining ?? null, - runtime: lastObservation.runtime ?? null, - directorHeartbeatFresh: lastObservation.directorCapacity?.heartbeatFresh ?? - lastObservation.runtimeHeartbeatFresh ?? null, - stranded: strandedObservation(lastObservation) - }) - } - if (now() >= deadline) { - throw new Error( - `capacity transition verification timed out: ${JSON.stringify(lastObservation)}` - ) - } - await wait(POLL_INTERVAL_MS) - } -} - -export async function main(argv = process.argv.slice(2)) { - const result = await verifyCapacityTransition(parseCapacityTransitionArguments(argv)) - process.stdout.write(`${JSON.stringify({ event: 'relay_capacity_transition_verified', ...result })}\n`) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/verify-relay-capacity-transition.test.mjs b/cloud/dev/scripts/verify-relay-capacity-transition.test.mjs deleted file mode 100644 index 8d861b3bd57..00000000000 --- a/cloud/dev/scripts/verify-relay-capacity-transition.test.mjs +++ /dev/null @@ -1,1511 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { - parseCapacityTransitionArguments, - verifyCapacityTransition -} from './verify-relay-capacity-transition.mjs' - -const imageDigest = `sha256:${'a'.repeat(64)}` -const config = { - directorOrigin: 'https://relay.example.com', - cellOrigin: 'https://c3.relay.example.com', - cellId: 'staging-gce-c3', - heartbeat: 'fresh', - admission: 'non-general', - draining: 'required', - activity: 'quiescent', - expectedImageDigests: [imageDigest], - hardCap: 1_000, - unobservedBound: 60, - timeoutMs: 1 -} - -function response(body) { - return Response.json(body) -} - -function harness({ - heartbeatFresh = true, - active = 0, - preAuthConnections = 0, - inFlightConnections = 0, - reservedConnectionUnits = 0, - enforcedConnectionUnits = active, - activityLeases = active, - activityRequestUnits = activityLeases, - reservedRequests = activityRequestUnits, - restartBlockingActivityLeases = activityLeases, - restartBlockingActivityRequestUnits = activityRequestUnits, - restartBlockingReservedRequests = reservedRequests, - outgoingMigrations = 0, - incomingMigrations = 0, - controls = 0, - splices = 0, - pendingSplices = 0, - queuedBytes = 0, - pendingControlReservations = 0, - runtimeHardCap = 1_000, - directorHardCap = 1_000, - runtimeImageDigest = imageDigest, - protocol = 2, - regionalRehomeProtocol = 1, - legacy = false, - admission = config.admission, - draining = config.draining -} = {}) { - return async (url) => { - const parsed = new URL(url) - if (parsed.pathname === '/health') { - return response({ ok: true, connectionCapacityProtocol: protocol }) - } - if (parsed.pathname === '/v1/admin/runtime-status') { - return response({ - role: 'cell', - cellId: config.cellId, - cellUrl: config.cellOrigin, - imageDigest: runtimeImageDigest, - ...(regionalRehomeProtocol === null ? {} : { regionalRehomeProtocol }), - draining: draining === 'required', - connectionCapacity: legacy - ? null - : { - hardCap: runtimeHardCap, - unobservedBound: 60, - controlRebindReserve: 100, - ordinaryConnectionLimit: runtimeHardCap - 100, - normalAdmissionPause: runtimeHardCap - 160 - }, - runtime: { - totalConnections: active, - preAuthConnections, - controls, - splices, - pendingSplices, - queuedBytes, - ...(legacy ? {} : { - inFlightConnections, - reservedConnectionUnits, - enforcedConnectionUnits - }) - } - }) - } - return response({ - status: { - cellId: config.cellId, - cellUrl: config.cellOrigin, - admissionState: admission === 'non-general' ? 'migration-only' : admission, - runtime: { heartbeatFresh, cellUrl: config.cellOrigin }, - assignments: 900, - activityLeases, - activityRequestUnits, - restartBlockingActivityLeases, - restartBlockingActivityRequestUnits, - restartBlockingReservedRequests, - reservedRequests, - outgoingMigrations, - incomingMigrations, - connectionCapacity: { - hardCap: directorHardCap, - unobservedBound: 60, - controlRebindReserve: 100, - ordinaryConnectionLimit: directorHardCap - 100, - normalAdmissionPause: directorHardCap - 160, - observedConnections: 0, - inFlightConnections: 0, - reservedConnectionUnits: 0, - enforcedConnectionUnits: 0, - pendingControlReservations, - heartbeatFresh - } - } - }) - } -} - -test('parses a paired reviewed capacity', () => { - assert.deepEqual( - parseCapacityTransitionArguments([ - '--director-origin', 'https://relay.example.com', - '--cell-origin', 'https://c3.relay.example.com', - '--cell-id', 'staging-gce-c3', - '--heartbeat', 'stale', - '--admission', 'migration-only', - '--draining', 'required', - '--activity', 'restart-safe', - '--expected-image-digests', imageDigest, - '--hard-cap', '1000', - '--unobserved-bound', '60', - '--pace-window-ms', '300000' - ]), - { - ...config, - heartbeat: 'stale', - admission: 'migration-only', - activity: 'restart-safe', - runtime: 'required', - expectedImageDigests: [imageDigest], - paceWindowMs: 300_000, - timeoutMs: 180_000 - } - ) -}) - -test('accepts either exact predecessor image without weakening the live state checks', async () => { - const predecessor = `sha256:${'b'.repeat(64)}` - const compatible = `sha256:${'c'.repeat(64)}` - const predecessorConfig = parseCapacityTransitionArguments([ - '--director-origin', 'https://relay.example.com', - '--cell-origin', 'https://c3.relay.example.com', - '--cell-id', 'staging-gce-c3', - '--heartbeat', 'fresh', - '--admission', 'general', - '--draining', 'forbidden', - '--activity', 'allowed', - '--expected-image-digests', `${predecessor},${compatible}`, - '--hard-cap', '600', - '--unobserved-bound', '60' - ]) - assert.deepEqual(predecessorConfig.expectedImageDigests, [predecessor, compatible]) - assert.deepEqual( - { - heartbeat: predecessorConfig.heartbeat, - admission: predecessorConfig.admission, - draining: predecessorConfig.draining, - hardCap: predecessorConfig.hardCap, - unobservedBound: predecessorConfig.unobservedBound - }, - { - heartbeat: 'fresh', - admission: 'general', - draining: 'forbidden', - hardCap: 600, - unobservedBound: 60 - } - ) - const liveState = { - runtimeHardCap: 600, - directorHardCap: 600, - runtimeImageDigest: compatible, - admission: 'general', - draining: 'forbidden' - } - assert.equal( - (await verifyCapacityTransition(predecessorConfig, { - fetch: harness(liveState), - token: 'masked-token' - })).imageDigest, - compatible - ) - await assert.rejects( - verifyCapacityTransition(predecessorConfig, { - fetch: harness({ - ...liveState, - runtimeImageDigest: `sha256:${'d'.repeat(64)}` - }), - token: 'masked-token' - }), - /runtime does not match the cell/ - ) -}) - -test('requires the exact regional rehome protocol when requested', async () => { - const rehomeConfig = { - ...config, - regionalRehomeProtocol: 1 - } - await verifyCapacityTransition(rehomeConfig, { - token: 'token', - fetch: harness({ regionalRehomeProtocol: 1 }) - }) - await assert.rejects( - verifyCapacityTransition(rehomeConfig, { - token: 'token', - fetch: harness({ regionalRehomeProtocol: 0 }), - now: () => 1, - wait: async () => undefined - }), - /runtime does not match/ - ) - assert.equal( - parseCapacityTransitionArguments([ - '--director-origin', 'https://relay.example.com', - '--cell-origin', 'https://c3.relay.example.com', - '--cell-id', 'staging-gce-c3', - '--heartbeat', 'fresh', - '--admission', 'general', - '--draining', 'forbidden', - '--activity', 'allowed', - '--regional-rehome-protocol', '1' - ]).regionalRehomeProtocol, - 1 - ) -}) - -test('binds an absent rehome protocol to 0 for legacy pre-rehome images', async () => { - // A rolled-back cell runs an image that omits the field entirely; the - // documented contract binds absence to protocol 0. - await verifyCapacityTransition( - { ...config, regionalRehomeProtocol: 0 }, - { token: 'token', fetch: harness({ regionalRehomeProtocol: null }) } - ) - await verifyCapacityTransition( - { ...config, regionalRehomeProtocol: 0 }, - { token: 'token', fetch: harness({ regionalRehomeProtocol: 0 }) } - ) - await assert.rejects( - verifyCapacityTransition( - { ...config, regionalRehomeProtocol: 1 }, - { - token: 'token', - fetch: harness({ regionalRehomeProtocol: null }), - now: () => 1, - wait: async () => undefined - } - ), - /runtime does not match/ - ) -}) - -test('restart-safe mode requires the exact isolated drain state', () => { - assert.throws( - () => parseCapacityTransitionArguments([ - '--director-origin', 'https://relay.example.com', - '--cell-origin', 'https://c3.relay.example.com', - '--cell-id', 'staging-gce-c3', - '--heartbeat', 'either', - '--admission', 'general', - '--draining', 'required', - '--activity', 'restart-safe' - ]), - /requires migration-only admission and draining/ - ) -}) - -test('offline rollback requires two stale zero-durable-activity samples', async () => { - const offlineConfig = { - ...config, - heartbeat: 'stale', - admission: 'migration-only', - draining: 'either', - activity: 'restart-safe', - runtime: 'unavailable', - expectedImageDigests: [], - hardCap: undefined, - unobservedBound: undefined, - timeoutMs: 10_000 - } - const base = harness({ - heartbeatFresh: false, - admission: 'migration-only', - draining: 'forbidden', - activityLeases: 0, - activityRequestUnits: 0, - reservedRequests: 0, - restartBlockingActivityLeases: 0, - restartBlockingActivityRequestUnits: 0, - restartBlockingReservedRequests: -1 - }) - let runtimeReads = 0 - let directorReads = 0 - let now = 0 - const result = await verifyCapacityTransition(offlineConfig, { - fetch: async (url, options) => { - const pathname = new URL(url).pathname - if (pathname === '/v1/admin/runtime-status') { - runtimeReads += 1 - return Response.json({ error: 'backend_unavailable' }, { status: 503 }) - } - if (pathname === '/v1/admin/cell-status') directorReads += 1 - return await base(url, options) - }, - token: 'masked-token', - now: () => now, - wait: async (milliseconds) => { - now += milliseconds - } - }) - assert.equal(result.cellId, config.cellId) - assert.equal(result.hardCap, null) - assert.equal(result.restartBlockingReservedRequests, -1) - assert.equal(runtimeReads, 2) - assert.equal(directorReads, 2) -}) - -test('offline rollback rejects a reachable cell or durable work', async () => { - const offlineConfig = { - ...config, - heartbeat: 'stale', - admission: 'migration-only', - draining: 'either', - activity: 'restart-safe', - runtime: 'unavailable', - expectedImageDigests: [], - hardCap: undefined, - unobservedBound: undefined, - timeoutMs: 0 - } - await assert.rejects( - verifyCapacityTransition(offlineConfig, { - fetch: harness({ heartbeatFresh: false, admission: 'migration-only' }), - token: 'masked-token' - }), - /timed out/ - ) - const active = harness({ - heartbeatFresh: false, - admission: 'migration-only', - activityLeases: 1 - }) - await assert.rejects( - verifyCapacityTransition(offlineConfig, { - fetch: async (url, options) => - new URL(url).pathname === '/v1/admin/runtime-status' - ? Response.json({ error: 'backend_unavailable' }, { status: 503 }) - : await active(url, options), - token: 'masked-token' - }), - /timed out/ - ) - await assert.rejects( - verifyCapacityTransition(offlineConfig, { - fetch: async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') { - return Response.json({ error: 'backend_unavailable' }, { status: 503 }) - } - const result = await active(url, options) - if (new URL(url).pathname !== '/v1/admin/cell-status') return result - const body = await result.json() - body.status.cellId = 'production-gce-other' - return response(body) - }, - token: 'masked-token' - }), - /does not match the cell/ - ) -}) - -test('offline rollback arguments cannot claim live capacity', () => { - assert.throws( - () => parseCapacityTransitionArguments([ - '--director-origin', 'https://relay.example.com', - '--cell-origin', 'https://c3.relay.example.com', - '--cell-id', 'staging-gce-c3', - '--heartbeat', 'stale', - '--admission', 'migration-only', - '--draining', 'either', - '--activity', 'restart-safe', - '--runtime', 'unavailable', - '--hard-cap', '600', - '--unobserved-bound', '60' - ]), - /cannot prove live capacity/ - ) -}) - -test('accepts a quiescent matching cell without exposing assignments', async () => { - assert.deepEqual( - await verifyCapacityTransition(config, { - fetch: harness(), - token: 'masked-token' - }), - { - cellId: config.cellId, - admissionState: 'migration-only', - assignments: 900, - hardCap: 1_000, - unobservedBound: 60, - heartbeatFresh: true, - imageDigest - } - ) -}) - -test('migration-only admission rejects the irreversible existing-only state', async () => { - const migrationConfig = { ...config, admission: 'migration-only' } - await verifyCapacityTransition(migrationConfig, { - fetch: harness({ admission: 'migration-only' }), - token: 'masked-token' - }) - await assert.rejects( - verifyCapacityTransition(migrationConfig, { - fetch: harness({ admission: 'existing-only' }), - token: 'masked-token' - }), - /admission does not match/ - ) -}) - -test('requires the exact runtime image and both cell origins', async () => { - await assert.rejects( - verifyCapacityTransition(config, { - fetch: harness({ runtimeImageDigest: `sha256:${'b'.repeat(64)}` }), - token: 'masked-token' - }), - /runtime does not match the cell/ - ) - const base = harness() - for (const location of ['runtime', 'director']) { - await assert.rejects( - verifyCapacityTransition(config, { - fetch: async (url, options) => { - const result = await base(url, options) - const path = new URL(url).pathname - if ( - (location === 'runtime' && path !== '/v1/admin/runtime-status') || - (location === 'director' && path !== '/v1/admin/cell-status') - ) return result - const body = await result.json() - if (location === 'runtime') body.cellUrl = 'https://other.example.com' - else body.status.cellUrl = 'https://other.example.com' - return response(body) - }, - token: 'masked-token' - }), - /does not match the cell/ - ) - } -}) - -test('accepts a quiescent legacy runtime before its first cap transition', async () => { - assert.equal( - ( - await verifyCapacityTransition( - { ...config, hardCap: undefined, unobservedBound: undefined, heartbeat: 'either' }, - { fetch: harness({ legacy: true }), token: 'masked-token' } - ) - ).cellId, - config.cellId - ) -}) - -test('uses the legacy runtime heartbeat when capacity telemetry is not registered', async () => { - const result = await verifyCapacityTransition( - { - ...config, - hardCap: undefined, - unobservedBound: undefined, - heartbeat: 'fresh', - draining: 'forbidden', - activity: 'allowed' - }, - { fetch: harness({ legacy: true, draining: 'forbidden' }), token: 'masked-token' } - ) - assert.equal(result.heartbeatFresh, true) -}) - -test('rejects old directors and active cells', async () => { - await assert.rejects( - verifyCapacityTransition(config, { fetch: harness({ protocol: 1 }), token: 'masked' }), - /not capacity-protocol compatible/ - ) - let now = 0 - await assert.rejects( - verifyCapacityTransition(config, { - fetch: harness({ active: 1 }), - token: 'masked', - now: () => now, - wait: async (milliseconds) => { - now += milliseconds - } - }), - /timed out/ - ) -}) - -test('accepts active controls only when the transition explicitly allows them', async () => { - const activeConfig = { - ...config, - admission: 'general', - draining: 'forbidden', - activity: 'allowed' - } - const result = await verifyCapacityTransition(activeConfig, { - fetch: harness({ active: 900, admission: 'general', draining: 'forbidden' }), - token: 'masked' - }) - assert.equal(result.admissionState, 'general') -}) - -test('accepts only rejected reconnect traffic at the restart gate', async () => { - const restartConfig = { - ...config, - admission: 'migration-only', - activity: 'restart-safe', - timeoutMs: 10_000 - } - const reconnecting = harness({ - active: 10, - activityLeases: 839, - restartBlockingActivityLeases: 0, - restartBlockingActivityRequestUnits: 0, - restartBlockingReservedRequests: 0, - pendingControlReservations: 839 - }) - let now = 0 - let runtimeReads = 0 - const fetch = async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1 - return await reconnecting(url, options) - } - assert.equal((await verifyCapacityTransition(restartConfig, { - fetch, - token: 'masked-token', - now: () => now, - wait: async (milliseconds) => { - now += milliseconds - } - })).cellId, config.cellId) - assert.equal(runtimeReads, 2) - await assert.rejects( - verifyCapacityTransition({ ...config, timeoutMs: 0 }, { - fetch: reconnecting, - token: 'masked-token' - }), - /timed out/ - ) -}) - -test('restart-safe settling resets after data-plane admission appears', async () => { - const restartConfig = { - ...config, - admission: 'migration-only', - activity: 'restart-safe', - timeoutMs: 20_000 - } - const safe = harness({ active: 1, activityLeases: 0 }) - const unsafe = harness({ active: 1, activityLeases: 0, controls: 1 }) - let runtimeReads = 0 - let now = 0 - const result = await verifyCapacityTransition(restartConfig, { - fetch: async (url, options) => { - if (new URL(url).pathname !== '/v1/admin/runtime-status') { - return await safe(url, options) - } - runtimeReads += 1 - return await (runtimeReads === 2 ? unsafe : safe)(url, options) - }, - token: 'masked-token', - now: () => now, - wait: async (milliseconds) => { - now += milliseconds - } - }) - assert.equal(result.cellId, config.cellId) - assert.equal(runtimeReads, 4) -}) - -test('restart-safe settling resets after a migration opens', async () => { - const restartConfig = { - ...config, - admission: 'migration-only', - activity: 'restart-safe', - timeoutMs: 20_000 - } - const safe = harness({ - activityLeases: 839, - restartBlockingActivityLeases: 0, - restartBlockingActivityRequestUnits: 0, - restartBlockingReservedRequests: 0 - }) - const unsafe = harness({ - activityLeases: 839, - restartBlockingActivityLeases: 0, - restartBlockingActivityRequestUnits: 0, - restartBlockingReservedRequests: 0, - incomingMigrations: 1 - }) - let directorReads = 0 - let runtimeReads = 0 - let now = 0 - const result = await verifyCapacityTransition(restartConfig, { - fetch: async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1 - if (new URL(url).pathname !== '/v1/admin/cell-status') { - return await safe(url, options) - } - directorReads += 1 - return await (directorReads === 2 ? unsafe : safe)(url, options) - }, - token: 'masked-token', - now: () => now, - wait: async (milliseconds) => { - now += milliseconds - } - }) - assert.equal(result.cellId, config.cellId) - assert.equal(runtimeReads, 4) -}) - -test('restart gate rejects malformed restart aggregates', async (t) => { - const restartConfig = { - ...config, - admission: 'migration-only', - activity: 'restart-safe', - timeoutMs: 0 - } - for (const field of [ - 'restartBlockingActivityLeases', - 'restartBlockingActivityRequestUnits' - ]) { - for (const value of [undefined, -1]) { - await t.test(`${field} ${value === undefined ? 'missing' : 'negative'}`, async () => { - const base = harness({ [field]: value }) - const fetch = value === undefined - ? async (url, options) => { - const result = await base(url, options) - if (new URL(url).pathname !== '/v1/admin/cell-status') return result - const body = await result.json() - delete body.status[field] - return response(body) - } - : base - await assert.rejects( - verifyCapacityTransition(restartConfig, { - fetch, - token: 'masked-token' - }), - /is invalid/ - ) - }) - } - } - for (const value of [undefined, null, '0', false]) { - await t.test(`restartBlockingReservedRequests ${String(value)}`, async () => { - const base = harness({ - preAuthConnections: 1, - restartBlockingActivityLeases: 1, - restartBlockingReservedRequests: value - }) - await assert.rejects( - verifyCapacityTransition(restartConfig, { - fetch: async (url, options) => { - const result = await base(url, options) - if ( - value !== undefined || - new URL(url).pathname !== '/v1/admin/cell-status' - ) return result - const body = await result.json() - delete body.status.restartBlockingReservedRequests - return response(body) - }, - token: 'masked-token' - }), - /is invalid/ - ) - }) - } -}) - -test('offline rollback validates restart reservation accounting before other blockers', async (t) => { - const offlineConfig = { - ...config, - heartbeat: 'stale', - admission: 'migration-only', - draining: 'either', - activity: 'restart-safe', - runtime: 'unavailable', - expectedImageDigests: [], - hardCap: undefined, - unobservedBound: undefined, - timeoutMs: 0 - } - for (const value of [undefined, null, '0', false]) { - await t.test(String(value), async () => { - const base = harness({ - heartbeatFresh: false, - activityLeases: 1, - restartBlockingReservedRequests: value - }) - await assert.rejects( - verifyCapacityTransition(offlineConfig, { - fetch: async (url, options) => { - const pathname = new URL(url).pathname - if (pathname === '/v1/admin/runtime-status') { - return Response.json({ error: 'backend_unavailable' }, { status: 503 }) - } - const result = await base(url, options) - if (value !== undefined || pathname !== '/v1/admin/cell-status') return result - const body = await result.json() - delete body.status.restartBlockingReservedRequests - return response(body) - }, - token: 'masked-token' - }), - /is invalid/ - ) - }) - } -}) - -test('negative restart reservation accounting is safe and remains observable', async () => { - const result = await verifyCapacityTransition( - { - ...config, - admission: 'migration-only', - activity: 'restart-safe', - timeoutMs: 10_000 - }, - { - fetch: harness({ restartBlockingReservedRequests: -1 }), - token: 'masked-token', - now: () => 0, - wait: async () => undefined - } - ) - assert.equal(result.restartBlockingReservedRequests, -1) -}) - -test('negative restart reservation accounting cannot mask real work', async () => { - await assert.rejects( - verifyCapacityTransition( - { - ...config, - admission: 'migration-only', - activity: 'restart-safe', - timeoutMs: 0 - }, - { - fetch: harness({ - controls: 1, - restartBlockingReservedRequests: -1 - }), - token: 'masked-token' - } - ), - /"restartBlockingReservedRequests":-1/ - ) -}) - -test('restart gate rejects live or durable cell work', async (t) => { - const restartConfig = { - ...config, - admission: 'migration-only', - activity: 'restart-safe', - timeoutMs: 0 - } - const unsafeStates = [ - ['control', { active: 1, activityLeases: 0, controls: 1 }], - ['splice', { active: 1, activityLeases: 0, splices: 1 }], - ['pending splice', { active: 1, activityLeases: 0, pendingSplices: 1 }], - ['queued data', { active: 1, activityLeases: 0, queuedBytes: 1 }], - ['control beside a reserved data unit', { - active: 2, - activityLeases: 0, - controls: 1, - reservedConnectionUnits: 1, - enforcedConnectionUnits: 3 - }], - ['outgoing migration', { outgoingMigrations: 1 }], - ['incoming migration', { incomingMigrations: 1 }] - ] - for (const [name, state] of unsafeStates) { - await t.test(name, async () => { - await assert.rejects( - verifyCapacityTransition(restartConfig, { - fetch: harness(state), - token: 'masked-token' - }), - // A zero timeout can never collect two samples; the blocker must stop the first one. - /timed out.*"restartSafeSamples":0/ - ) - }) - } -}) - -test('restart timeout reports only aggregate blockers', async () => { - const base = harness({ - controls: 2, - restartBlockingActivityLeases: 3, - restartBlockingActivityRequestUnits: 4, - restartBlockingReservedRequests: 5, - incomingMigrations: 6 - }) - await assert.rejects( - verifyCapacityTransition( - { - ...config, - admission: 'migration-only', - draining: 'required', - activity: 'restart-safe', - timeoutMs: 0 - }, - { fetch: base, token: 'masked-token' } - ), - (error) => { - assert.match(error.message, /"controls":2/) - assert.match(error.message, /"restartBlockingActivityLeases":3/) - assert.match(error.message, /"incomingMigrations":6/) - assert.doesNotMatch(error.message, /user|host|secret|token/i) - return true - } - ) -}) - -test('restart timeout reports one of two safe settling samples', async () => { - const base = harness() - await assert.rejects( - verifyCapacityTransition( - { - ...config, - admission: 'migration-only', - draining: 'required', - activity: 'restart-safe', - timeoutMs: 0 - }, - { fetch: base, token: 'masked-token' } - ), - (error) => { - assert.match(error.message, /"restartSafeSamples":1,"paceWindowRemainingMs":0/) - return true - } - ) -}) - -test('quiescent timeout reports connection and capacity aggregates', async () => { - const base = harness({ - active: 2, - enforcedConnectionUnits: 3, - activityLeases: 0, - activityRequestUnits: 0, - reservedRequests: 0, - restartBlockingActivityLeases: 0, - restartBlockingActivityRequestUnits: 0, - restartBlockingReservedRequests: 0, - pendingControlReservations: 4, - heartbeatFresh: false - }) - await assert.rejects( - verifyCapacityTransition({ ...config, timeoutMs: 0 }, { fetch: base, token: 'masked-token' }), - (error) => { - assert.match(error.message, /"totalConnections":2/) - assert.match(error.message, /"enforcedConnectionUnits":3/) - assert.match(error.message, /"pendingControlReservations":4/) - assert.match(error.message, /"heartbeatFresh":false/) - return true - } - ) -}) - -test('timeout distinguishes runtime and director capacity views', async () => { - const base = harness({ runtimeHardCap: 999 }) - await assert.rejects( - verifyCapacityTransition({ ...config, timeoutMs: 0 }, { fetch: base, token: 'masked-token' }), - (error) => { - assert.match(error.message, /"runtimeCapacity":\{"hardCap":999/) - assert.match(error.message, /"directorCapacity":\{"hardCap":1000/) - return true - } - ) -}) - -test('offline timeout reports every durable activity aggregate', async () => { - const base = harness({ - activityLeases: 1, - activityRequestUnits: 2, - reservedRequests: 3, - pendingControlReservations: 4, - heartbeatFresh: false - }) - await assert.rejects( - verifyCapacityTransition( - { - ...config, - admission: 'migration-only', - draining: 'either', - activity: 'restart-safe', - heartbeat: 'stale', - runtime: 'unavailable', - hardCap: undefined, - unobservedBound: undefined, - timeoutMs: 0 - }, - { - fetch: async (url, options) => - new URL(url).pathname === '/v1/admin/runtime-status' - ? Response.json({ error: 'backend_unavailable' }, { status: 503 }) - : await base(url, options), - token: 'masked-token' - } - ), - (error) => { - assert.match(error.message, /"activityLeases":1/) - assert.match(error.message, /"activityRequestUnits":2/) - assert.match(error.message, /"reservedRequests":3/) - assert.match(error.message, /"pendingControlReservations":4/) - return true - } - ) -}) - -test('waits for a drained runtime to become quiescent', async () => { - let runtimeReads = 0 - const base = harness() - const fetch = async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status' && runtimeReads++ === 0) { - return Response.json({ - role: 'cell', - cellId: config.cellId, - cellUrl: config.cellOrigin, - imageDigest, - draining: true, - connectionCapacity: { - hardCap: 1_000, - unobservedBound: 60, - controlRebindReserve: 100, - ordinaryConnectionLimit: 900, - normalAdmissionPause: 840 - }, - runtime: { totalConnections: 1, preAuthConnections: 0, enforcedConnectionUnits: 1 } - }) - } - return await base(url, options) - } - let now = 0 - const result = await verifyCapacityTransition( - { ...config, timeoutMs: 10_000 }, - { - fetch, - token: 'masked', - now: () => now, - wait: async (milliseconds) => { - now += milliseconds - } - } - ) - assert.equal(result.cellId, config.cellId) - assert.equal(runtimeReads, 2) -}) - -test('waits for transient cell routing after a replacement becomes stable', async () => { - for (const status of [502, 503, 504]) { - let runtimeReads = 0 - let unavailableResponse - const base = harness() - const fetch = async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status' && runtimeReads++ === 0) { - unavailableResponse = Response.json({ error: 'backend_unavailable' }, { status }) - return unavailableResponse - } - return await base(url, options) - } - let now = 0 - const result = await verifyCapacityTransition( - { ...config, timeoutMs: 10_000 }, - { - fetch, - token: 'masked', - now: () => now, - wait: async (milliseconds) => { - now += milliseconds - } - } - ) - assert.equal(result.cellId, config.cellId) - assert.equal(runtimeReads, 2) - assert.equal(unavailableResponse.bodyUsed, true) - } -}) - -test('persistent cell unavailability fails closed at the deadline', async () => { - let runtimeReads = 0 - let directorStatusReads = 0 - let waits = 0 - let now = 0 - const base = harness() - await assert.rejects( - verifyCapacityTransition( - { ...config, timeoutMs: 10_000 }, - { - fetch: async (url, options) => { - const pathname = new URL(url).pathname - if (pathname === '/v1/admin/runtime-status') { - runtimeReads += 1 - return Response.json({ error: 'backend_unavailable' }, { status: 503 }) - } - if (pathname === '/v1/admin/cell-status') directorStatusReads += 1 - return await base(url, options) - }, - token: 'masked', - now: () => now, - wait: async (milliseconds) => { - waits += 1 - now += milliseconds - } - } - ), - /capacity transition verification timed out: \{"runtimeAvailable":false\}/ - ) - assert.equal(runtimeReads, 3) - assert.equal(directorStatusReads, 0) - assert.equal(waits, 2) -}) - -test('general-or-migration-only admits both recovery states and nothing else', async () => { - for (const [admission, accepted] of [ - ['general', true], - ['migration-only', true], - ['existing-only', false] - ]) { - const attempt = verifyCapacityTransition( - { - ...config, - admission: 'general-or-migration-only', - draining: 'either', - activity: 'allowed' - }, - { fetch: harness({ admission, draining: 'either' }), token: 'masked' } - ) - if (accepted) { - await attempt - } else { - await assert.rejects(attempt, /admission does not match/) - } - } -}) - -test('does not retry a rejected cell admin token', async () => { - let waits = 0 - const base = harness() - await assert.rejects( - verifyCapacityTransition(config, { - fetch: async (url, options) => - new URL(url).pathname === '/v1/admin/runtime-status' - ? Response.json({ error: 'invalid_token' }, { status: 401 }) - : await base(url, options), - token: 'masked', - wait: async () => { - waits += 1 - } - }), - /cell runtime status returned 401/ - ) - assert.equal(waits, 0) -}) - -test('retries a transient 503 on the director cell-status read', async () => { - const base = harness() - const statusCalls = [] - const result = await verifyCapacityTransition(config, { - token: 'masked-token', - wait: async () => {}, - fetch: async (url, options) => { - const path = new URL(url).pathname - if (path !== '/v1/admin/cell-status') return await base(url, options) - statusCalls.push(path) - if (statusCalls.length === 1) return new Response('warming up', { status: 503 }) - return await base(url, options) - } - }) - assert.equal(statusCalls.length, 2) - assert.equal(result.cellId, config.cellId) -}) - -test('fails when both director cell-status attempts return a transient 503', async () => { - const base = harness() - let statusCalls = 0 - await assert.rejects( - verifyCapacityTransition(config, { - token: 'masked-token', - wait: async () => {}, - fetch: async (url, options) => { - const path = new URL(url).pathname - if (path !== '/v1/admin/cell-status') return await base(url, options) - statusCalls += 1 - return new Response('warming up', { status: 503 }) - } - }), - /cell status returned 503/ - ) - assert.equal(statusCalls, 2) -}) - -test('retries a transient 503 on the director health preflight', async () => { - const base = harness() - let healthCalls = 0 - const result = await verifyCapacityTransition(config, { - token: 'masked-token', - wait: async () => {}, - fetch: async (url, options) => { - const path = new URL(url).pathname - if (path !== '/health') return await base(url, options) - healthCalls += 1 - if (healthCalls === 1) return new Response('warming up', { status: 503 }) - return await base(url, options) - } - }) - assert.equal(healthCalls, 2) - assert.equal(result.cellId, config.cellId) -}) - -test('fails when both director health attempts return a transient 503', async () => { - const base = harness() - let healthCalls = 0 - await assert.rejects( - verifyCapacityTransition(config, { - token: 'masked-token', - wait: async () => {}, - fetch: async (url, options) => { - const path = new URL(url).pathname - if (path !== '/health') return await base(url, options) - healthCalls += 1 - return new Response('warming up', { status: 503 }) - } - }), - /director health returned 503/ - ) - assert.equal(healthCalls, 2) -}) - -// Production-gce-c28 on 2026-10-01: drained to an empty runtime while four hosts with no free -// slot anywhere kept redialling and held director leases on it. -const strandedCell = { - activityLeases: 10, - activityRequestUnits: 10, - reservedRequests: 4, - restartBlockingActivityLeases: 4, - restartBlockingActivityRequestUnits: 4, - restartBlockingReservedRequests: -2, - pendingControlReservations: 6 -} -const pacedRestartConfig = { - ...config, - admission: 'migration-only', - activity: 'restart-safe', - paceWindowMs: 300_000, - timeoutMs: 1_200_000 -} - -function fakeClock(start = 0) { - let now = start - return { - now: () => now, - wait: async (milliseconds) => { - now += milliseconds - } - } -} - -test('requires the drain pace window exactly for a live restart-safe wait', () => { - const base = [ - '--director-origin', 'https://relay.example.com', - '--cell-origin', 'https://c3.relay.example.com', - '--cell-id', 'staging-gce-c3', - '--heartbeat', 'either', - '--admission', 'migration-only', - '--draining', 'required' - ] - assert.equal( - parseCapacityTransitionArguments([ - ...base, '--activity', 'restart-safe', '--pace-window-ms', '300000' - ]).paceWindowMs, - 300_000 - ) - assert.throws( - () => parseCapacityTransitionArguments([...base, '--activity', 'restart-safe']), - /required exactly for restart-safe/ - ) - assert.throws( - () => parseCapacityTransitionArguments([ - ...base, '--activity', 'allowed', '--pace-window-ms', '300000' - ]), - /required exactly for restart-safe/ - ) - assert.throws( - () => parseCapacityTransitionArguments([ - ...base, '--activity', 'restart-safe', '--pace-window-ms', '-1' - ]), - /--pace-window-ms is invalid/ - ) -}) - -test('passes after one pace window of an empty runtime and reports stranded leases', async () => { - const lines = [] - let runtimeReads = 0 - const base = harness(strandedCell) - const result = await verifyCapacityTransition(pacedRestartConfig, { - fetch: async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1 - return await base(url, options) - }, - token: 'masked-token', - progress: (line) => lines.push(line), - // A late start catches a window measured from zero instead of from the verifier's start. - ...fakeClock(1_000_000) - }) - // One sample at each end of a full five-minute pace window. - assert.equal(runtimeReads, 61) - assert.deepEqual(result.stranded, { - restartBlockingActivityLeases: 4, - restartBlockingActivityRequestUnits: 4, - restartBlockingReservedRequests: -2, - outgoingMigrations: 0, - incomingMigrations: 0 - }) - // Every sample but the passing one prints its progress with the stranded counts. - assert.equal(lines.length, 60) - assert.ok(lines.every((line) => - line.event === 'relay_capacity_transition_restart_progress' && - line.stranded.restartBlockingActivityLeases === 4)) - assert.deepEqual(lines.slice(0, 2).map((line) => line.restartSafeSamples), [1, 2]) - assert.deepEqual( - lines.slice(0, 2).map((line) => line.paceWindowRemainingMs), - [300_000, 295_000] - ) -}) - -test('a non-zero runtime sample restarts the quiet count', async () => { - const quiet = harness(strandedCell) - const busy = harness({ ...strandedCell, controls: 1 }) - let runtimeReads = 0 - const lines = [] - await verifyCapacityTransition(pacedRestartConfig, { - fetch: async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1 - return await (runtimeReads === 30 ? busy : quiet)(url, options) - }, - token: 'masked-token', - progress: (line) => lines.push(line), - ...fakeClock() - }) - // A mid-window reset adds no time; the quiet after it still reaches the window's end. - assert.equal(runtimeReads, 61) - assert.deepEqual(lines.slice(28, 31).map((line) => line.restartSafeSamples), [29, 0, 1]) - await assert.rejects( - verifyCapacityTransition( - { ...pacedRestartConfig, timeoutMs: 295_000 }, - { fetch: quiet, token: 'masked-token', progress: () => {}, ...fakeClock() } - ), - /"restartSafeSamples":60,"paceWindowRemainingMs":5000/ - ) -}) - -// c27 on 2026-10-13: the 20-minute paced drain emptied the runtime as its window closed, and the -// old gate then waited a second full window of quiet samples before the restart. -test('a runtime that empties as the pace window ends passes after two quiet samples', async () => { - let runtimeReads = 0 - const lines = [] - await verifyCapacityTransition(pacedRestartConfig, { - fetch: async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1 - return await harness({ ...strandedCell, controls: runtimeReads <= 60 ? 1 : 0 })(url, options) - }, - token: 'masked-token', - progress: (line) => lines.push(line), - ...fakeClock() - }) - // Busy through 295 s; quiet at 300 s and 305 s, the first eligible pair. - assert.equal(runtimeReads, 62) - assert.deepEqual(lines.slice(59).map((line) => line.restartSafeSamples), [0, 1]) -}) - -test('restart-safe fails on live runtime work or an open migration regardless of leases', async (t) => { - for (const blocker of [ - { controls: 1 }, - { splices: 1 }, - { pendingSplices: 1 }, - { queuedBytes: 1 }, - { outgoingMigrations: 1 }, - { incomingMigrations: 1 } - ]) { - await t.test(Object.keys(blocker)[0], async () => { - await assert.rejects( - verifyCapacityTransition(pacedRestartConfig, { - fetch: harness({ ...strandedCell, ...blocker }), - token: 'masked-token', - progress: () => {}, - ...fakeClock() - }), - /timed out.*"restartSafeSamples":0/ - ) - }) - } -}) - -test('unauthenticated redials do not restart the quiet count', async () => { - let runtimeReads = 0 - const lines = [] - const result = await verifyCapacityTransition(pacedRestartConfig, { - fetch: async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1 - return await harness({ - ...strandedCell, - active: runtimeReads % 4, - enforcedConnectionUnits: 0, - preAuthConnections: runtimeReads % 2 - })(url, options) - }, - token: 'masked-token', - progress: (line) => lines.push(line), - ...fakeClock() - }) - assert.equal(runtimeReads, 61) - assert.equal(result.stranded.restartBlockingActivityLeases, 4) - assert.deepEqual( - lines.slice(0, 4).map((line) => [line.totalConnections, line.preAuthConnections]), - [[1, 1], [2, 0], [3, 1], [0, 0]] - ) -}) - -// c29 2026-10-01: a phone redialling the drained cell holds one physical socket and one -// reserved host-data unit until the cell refuses it; a host control redial holds an in-flight unit. -const refusedPhoneDial = { - active: 1, - preAuthConnections: 1, - reservedConnectionUnits: 1, - enforcedConnectionUnits: 2 -} -const refusedControlDial = { inFlightConnections: 1, enforcedConnectionUnits: 1 } - -test('a draining cell reaches the pace window through refused redials', async () => { - let runtimeReads = 0 - const lines = [] - const result = await verifyCapacityTransition(pacedRestartConfig, { - fetch: async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1 - const redial = runtimeReads % 3 === 0 - ? refusedPhoneDial - : runtimeReads % 3 === 1 ? refusedControlDial : {} - return await harness({ ...strandedCell, ...redial })(url, options) - }, - token: 'masked-token', - progress: (line) => lines.push(line), - ...fakeClock() - }) - assert.equal(runtimeReads, 61) - assert.equal(result.cellId, config.cellId) - assert.deepEqual(lines.slice(0, 3).map((line) => line.restartSafeSamples), [1, 2, 3]) -}) - -test('a refused redial cannot hide a persistent control on a draining cell', async () => { - let runtimeReads = 0 - const lines = [] - await assert.rejects( - verifyCapacityTransition(pacedRestartConfig, { - fetch: async (url, options) => { - if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1 - const redial = runtimeReads % 2 === 0 ? refusedPhoneDial : {} - return await harness({ ...strandedCell, ...redial, controls: 1 })(url, options) - }, - token: 'masked-token', - progress: (line) => lines.push(line), - ...fakeClock() - }), - /timed out.*"restartSafeSamples":0/ - ) - assert.ok(lines.length > 61) - assert.ok(lines.every((line) => line.restartSafeSamples === 0 && line.runtime.controls === 1)) -}) - -test('a runtime that is not draining never passes restart-safe', async (t) => { - for (const [name, state] of [ - ['idle', {}], - ['refused phone dial', refusedPhoneDial], - ['refused control dial', refusedControlDial] - ]) { - await t.test(name, async () => { - await assert.rejects( - verifyCapacityTransition(pacedRestartConfig, { - fetch: harness({ ...strandedCell, ...state, draining: 'forbidden' }), - token: 'masked-token', - progress: () => {}, - ...fakeClock() - }), - /timed out.*"draining":false.*"restartSafeSamples":0/ - ) - }) - } -}) - -test('quiescent transitions still reject refused redials', async (t) => { - for (const [name, state] of [ - ['refused phone dial', refusedPhoneDial], - ['refused control dial', refusedControlDial] - ]) { - await t.test(name, async () => { - await assert.rejects( - verifyCapacityTransition(config, { fetch: harness(state), token: 'masked-token' }), - /timed out/ - ) - }) - } -}) - -test('restart progress prints every counter the gate reads', async () => { - const lines = [] - await assert.rejects( - verifyCapacityTransition( - { ...pacedRestartConfig, timeoutMs: 0 }, - { - fetch: harness({ - ...strandedCell, - active: 7, - preAuthConnections: 1, - inFlightConnections: 2, - reservedConnectionUnits: 3, - enforcedConnectionUnits: 12, - controls: 4, - splices: 5, - pendingSplices: 6, - queuedBytes: 8, - incomingMigrations: 9, - outgoingMigrations: 10 - }), - token: 'masked-token', - progress: (line) => lines.push(line), - ...fakeClock() - } - ), - /timed out/ - ) - assert.equal(lines.length, 1) - assert.deepEqual(lines[0], { - event: 'relay_capacity_transition_restart_progress', - restartSafeSamples: 0, - paceWindowRemainingMs: 300_000, - totalConnections: 7, - preAuthConnections: 1, - runtimeAvailable: true, - admissionState: 'migration-only', - draining: true, - runtime: { - totalConnections: 7, - preAuthConnections: 1, - inFlightConnections: 2, - reservedConnectionUnits: 3, - enforcedConnectionUnits: 12, - controls: 4, - splices: 5, - pendingSplices: 6, - queuedBytes: 8 - }, - directorHeartbeatFresh: true, - stranded: { - restartBlockingActivityLeases: 4, - restartBlockingActivityRequestUnits: 4, - restartBlockingReservedRequests: -2, - outgoingMigrations: 10, - incomingMigrations: 9 - } - }) -}) - -test('a draining cell still fails closed on malformed handshake counters', async (t) => { - for (const [name, state] of [ - ['negative in-flight', { inFlightConnections: -1 }], - ['fractional reserved', { reservedConnectionUnits: 0.5 }], - ['non-numeric reserved', { reservedConnectionUnits: 'unknown' }] - ]) { - await t.test(name, async () => { - await assert.rejects( - verifyCapacityTransition(pacedRestartConfig, { - fetch: harness({ ...strandedCell, ...state }), - token: 'masked-token', - progress: () => {}, - ...fakeClock() - }), - /live runtime count is invalid/ - ) - }) - } -}) diff --git a/cloud/dev/scripts/verify-relay-legacy-bootstrap.mjs b/cloud/dev/scripts/verify-relay-legacy-bootstrap.mjs deleted file mode 100644 index 1c51ac28f36..00000000000 --- a/cloud/dev/scripts/verify-relay-legacy-bootstrap.mjs +++ /dev/null @@ -1,292 +0,0 @@ -import { createHash } from 'node:crypto' -import { readFileSync } from 'node:fs' -import { pathToFileURL } from 'node:url' - -const CAPACITY_PROTOCOL = 2 -const LEGACY_RUNTIME_KEYS = ['cellId', 'cellUrl', 'imageDigest', 'role', 'v'] -const METRIC_COUNTS = [ - 'totalConnections', - 'preAuthConnections', - 'controls', - 'splices', - 'pendingSplices', - 'queuedBytes' -] - -function integer(value, name) { - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} is invalid`) - return value -} - -export function parseLegacyBootstrapArguments(argv) { - const values = {} - for (let index = 0; index < argv.length; index += 2) { - const key = argv[index] - const value = argv[index + 1] - if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments') - values[key.slice(2)] = value - } - for (const key of [ - 'director-origin', - 'cell-origin', - 'cell-id', - 'admission', - 'expected-image-digest', - 'metrics-after', - 'metrics-file' - ]) { - if (!values[key]) throw new Error(`missing --${key}`) - } - if (!['general', 'migration-only'].includes(values.admission)) { - throw new Error('--admission must be general or migration-only') - } - const directorOrigin = new URL(values['director-origin']) - const cellOrigin = new URL(values['cell-origin']) - if ( - directorOrigin.protocol !== 'https:' || - directorOrigin.origin !== values['director-origin'] || - cellOrigin.protocol !== 'https:' || - cellOrigin.origin !== values['cell-origin'] - ) { - throw new Error('origins must be canonical HTTPS origins') - } - if (!/^sha256:[a-f0-9]{64}$/.test(values['expected-image-digest'])) { - throw new Error('--expected-image-digest is invalid') - } - const metricsAfter = Date.parse(values['metrics-after']) - if (!Number.isFinite(metricsAfter)) throw new Error('--metrics-after is invalid') - const runtimeStartedAfter = values['runtime-started-after'] === undefined - ? undefined - : Date.parse(values['runtime-started-after']) - if (runtimeStartedAfter !== undefined && !Number.isFinite(runtimeStartedAfter)) { - throw new Error('--runtime-started-after is invalid') - } - const previousIncarnationDigest = values['previous-incarnation-digest'] - if ( - previousIncarnationDigest !== undefined && - !/^[a-f0-9]{64}$/.test(previousIncarnationDigest) - ) { - throw new Error('--previous-incarnation-digest is invalid') - } - const hardCap = values['hard-cap'] === undefined - ? undefined - : integer(Number(values['hard-cap']), '--hard-cap') - const unobservedBound = values['unobserved-bound'] === undefined - ? undefined - : integer(Number(values['unobserved-bound']), '--unobserved-bound') - if ((hardCap === undefined) !== (unobservedBound === undefined)) { - throw new Error('capacity expectations must be paired') - } - const capacityState = values['capacity-state'] ?? (hardCap === undefined ? 'absent' : 'stale') - if (!['absent', 'stale', 'absent-or-stale'].includes(capacityState)) { - throw new Error('--capacity-state is invalid') - } - if ((capacityState === 'absent') !== (hardCap === undefined)) { - throw new Error('capacity state and expectations do not match') - } - return { - directorOrigin: directorOrigin.origin, - cellOrigin: cellOrigin.origin, - cellId: values['cell-id'], - admission: values.admission, - expectedImageDigest: values['expected-image-digest'], - metricsAfter, - metricsFile: values['metrics-file'], - runtimeStartedAfter, - previousIncarnationDigest, - hardCap, - unobservedBound, - capacityState - } -} - -async function responseJson(response, label) { - const body = await response.json().catch(() => ({})) - if (!response.ok) throw new Error(`${label} returned ${response.status}`) - return body -} - -function requireLegacyRuntime(runtime, config) { - if (JSON.stringify(Object.keys(runtime).sort()) !== JSON.stringify(LEGACY_RUNTIME_KEYS)) { - throw new Error('cell runtime does not match the reviewed legacy contract') - } - if ( - runtime.v !== 1 || - runtime.role !== 'cell' || - runtime.cellId !== config.cellId || - runtime.cellUrl !== config.cellOrigin || - runtime.imageDigest !== config.expectedImageDigest - ) { - throw new Error('legacy cell runtime identity does not match') - } -} - -function requireDirectorQuiescence(status, config, now) { - const capacity = status.connectionCapacity - const staleCapacityTransition = config.capacityState !== 'absent' && capacity !== null - if ( - status.cellId !== config.cellId || - status.cellUrl !== config.cellOrigin || - status.enabled !== true || - status.admissionState !== config.admission || - status.runtime?.ready !== true || - (status.runtime.heartbeatFresh !== true && - !(staleCapacityTransition && status.runtime.heartbeatFresh === false)) || - status.runtime.cellUrl !== config.cellOrigin - ) { - throw new Error('legacy cell is not fresh, ready, and in the required admission state') - } - if (config.capacityState === 'absent' && capacity !== null) { - throw new Error('legacy cell has unexpected connection capacity') - } - if ( - config.capacityState !== 'absent' && - !(config.capacityState === 'absent-or-stale' && capacity === null) && - (capacity?.hardCap !== config.hardCap || - capacity.unobservedBound !== config.unobservedBound || - capacity.controlRebindReserve !== 100 || - capacity.ordinaryConnectionLimit !== config.hardCap - 100 || - capacity.normalAdmissionPause !== config.hardCap - 100 - config.unobservedBound || - capacity.heartbeatFresh !== false) - ) { - throw new Error('legacy cell connection capacity does not match') - } - integer(status.runtime.startedAt, 'runtime started at') - integer(status.runtime.lastHeartbeatAt, 'runtime heartbeat at') - if (!/^[0-9a-f-]{36}$/.test(status.runtime.cellIncarnation)) { - throw new Error('runtime cell incarnation is invalid') - } - const incarnationDigest = createHash('sha256') - .update(status.runtime.cellIncarnation) - .digest('hex') - if (incarnationDigest === config.previousIncarnationDigest) { - throw new Error('legacy fallback heartbeat incarnation did not change') - } - if ( - config.runtimeStartedAfter !== undefined && - (integer(status.runtime.startedAt, 'runtime started at') < config.runtimeStartedAfter || - status.runtime.startedAt > now + 30_000) - ) { - throw new Error('legacy fallback heartbeat predates the replacement') - } - const activity = [ - status.reservedRequests, - status.activityLeases, - status.activityRequestUnits, - status.outgoingMigrations, - status.incomingMigrations, - status.runtime.observedRequests - ] - if (capacity !== null) { - activity.push( - capacity.observedConnections, - capacity.inFlightConnections, - capacity.reservedConnectionUnits, - capacity.enforcedConnectionUnits, - capacity.pendingControlReservations - ) - } - if (activity.some((value) => integer(value, 'director activity count') !== 0)) { - throw new Error('legacy fallback has durable activity') - } - integer(status.assignments, 'assignments') - return incarnationDigest -} - -function requireFreshZeroMetrics(metrics, config, now) { - if (!Array.isArray(metrics)) throw new Error('legacy runtime metrics are invalid') - const samples = metrics.filter((entry) => { - const timestamp = Date.parse(entry?.timestamp) - return Number.isFinite(timestamp) && timestamp >= config.metricsAfter && timestamp <= now + 30_000 - }) - const timestamps = new Set(samples.map((entry) => entry.timestamp)) - if (samples.length < 2 || timestamps.size < 2) { - throw new Error('legacy runtime metrics need two post-boundary samples') - } - const latest = Math.max(...samples.map((entry) => Date.parse(entry.timestamp))) - if (latest < now - 90_000) throw new Error('legacy runtime metrics are stale') - for (const sample of samples) { - if (sample.cellId !== config.cellId || sample.metricVersion !== 1) { - throw new Error('legacy runtime metrics do not match the cell') - } - if (METRIC_COUNTS.some((field) => integer(sample[field], field) !== 0)) { - throw new Error('legacy runtime metrics are not quiescent') - } - } - return samples.length -} - -export async function verifyLegacyBootstrap(config, overrides = {}) { - const fetchImpl = overrides.fetch ?? fetch - const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN - const now = overrides.now?.() ?? Date.now() - const metrics = overrides.metrics ?? JSON.parse(readFileSync(config.metricsFile, 'utf8')) - if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') - const publicChecks = await Promise.all([ - responseJson( - await fetchImpl(`${config.directorOrigin}/health`, { - signal: AbortSignal.timeout(15_000) - }), - 'director health' - ), - responseJson( - await fetchImpl(`${config.cellOrigin}/health`, { signal: AbortSignal.timeout(15_000) }), - 'cell health' - ), - responseJson( - await fetchImpl(`${config.cellOrigin}/ready`, { signal: AbortSignal.timeout(15_000) }), - 'cell readiness' - ) - ]) - if ( - publicChecks[0].ok !== true || - publicChecks[0].connectionCapacityProtocol !== CAPACITY_PROTOCOL || - publicChecks[1].ok !== true || - publicChecks[2].ok !== true - ) { - throw new Error('legacy fallback public checks failed') - } - const headers = { authorization: `Bearer ${token}`, 'content-type': 'application/json' } - const [runtime, result] = await Promise.all([ - responseJson( - await fetchImpl(`${config.cellOrigin}/v1/admin/runtime-status`, { - method: 'POST', - headers, - body: JSON.stringify({ v: 1 }), - signal: AbortSignal.timeout(30_000) - }), - 'cell runtime status' - ), - responseJson( - await fetchImpl(`${config.directorOrigin}/v1/admin/cell-status`, { - method: 'POST', - headers, - body: JSON.stringify({ v: 1, cellId: config.cellId }), - signal: AbortSignal.timeout(30_000) - }), - 'cell status' - ) - ]) - requireLegacyRuntime(runtime, config) - const incarnationDigest = requireDirectorQuiescence(result.status, config, now) - const metricSamples = requireFreshZeroMetrics(metrics, config, now) - return { - cellId: config.cellId, - admissionState: result.status.admissionState, - assignments: integer(result.status.assignments, 'assignments'), - metricSamples, - incarnationDigest - } -} - -export async function main(argv = process.argv.slice(2)) { - const result = await verifyLegacyBootstrap(parseLegacyBootstrapArguments(argv)) - process.stdout.write(`${JSON.stringify({ event: 'relay_legacy_bootstrap_verified', ...result })}\n`) -} - -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) - process.exitCode = 1 - }) -} diff --git a/cloud/dev/scripts/verify-relay-legacy-bootstrap.test.mjs b/cloud/dev/scripts/verify-relay-legacy-bootstrap.test.mjs deleted file mode 100644 index 0cb3cb5a25a..00000000000 --- a/cloud/dev/scripts/verify-relay-legacy-bootstrap.test.mjs +++ /dev/null @@ -1,395 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { relayWorkflowUrl } from './relay-repository.mjs' -import { - parseLegacyBootstrapArguments, - verifyLegacyBootstrap -} from './verify-relay-legacy-bootstrap.mjs' - -const now = Date.parse('2026-08-10T22:10:00Z') -const digest = `sha256:${'a'.repeat(64)}` -const config = { - directorOrigin: 'https://relay.example.com', - cellOrigin: 'https://c3.relay.example.com', - cellId: 'staging-gce-c3', - admission: 'general', - expectedImageDigest: digest, - metricsAfter: now - 120_000, - metricsFile: '/unused', - runtimeStartedAfter: undefined, - previousIncarnationDigest: undefined, - hardCap: undefined, - unobservedBound: undefined, - capacityState: 'absent' -} - -const metrics = [30, 60].map((secondsAgo) => ({ - timestamp: new Date(now - secondsAgo * 1_000).toISOString(), - cellId: config.cellId, - metricVersion: 1, - totalConnections: 0, - preAuthConnections: 0, - controls: 0, - splices: 0, - pendingSplices: 0, - queuedBytes: 0 -})) - -function response(body, status = 200) { - return Response.json(body, { status }) -} - -function harness({ - runtime = {}, - status = {}, - ready = true, - cell = config, - runtimeHeartbeatFresh = true -} = {}) { - return async (url) => { - const path = new URL(url).pathname - if (path === '/health') { - return response( - url.startsWith(config.directorOrigin) - ? { ok: true, connectionCapacityProtocol: 2 } - : { ok: true } - ) - } - if (path === '/ready') return ready ? response({ ok: true }) : response({ error: 'no' }, 503) - if (path === '/v1/admin/runtime-status') { - return response({ - v: 1, - role: 'cell', - cellId: cell.cellId, - cellUrl: cell.cellOrigin, - imageDigest: digest, - ...runtime - }) - } - return response({ - status: { - cellId: cell.cellId, - cellUrl: cell.cellOrigin, - enabled: true, - admissionState: 'general', - assignments: 12, - reservedRequests: 0, - activityLeases: 0, - activityRequestUnits: 0, - outgoingMigrations: 0, - incomingMigrations: 0, - connectionCapacity: null, - runtime: { - cellUrl: cell.cellOrigin, - cellIncarnation: '00000000-0000-4000-8000-000000000001', - startedAt: now - 90_000, - ready: true, - observedRequests: 0, - lastHeartbeatAt: now - 1_000, - heartbeatFresh: runtimeHeartbeatFresh - }, - ...status - } - }) - } -} - -test('parses the exact legacy bootstrap evidence boundary', () => { - assert.deepEqual( - parseLegacyBootstrapArguments([ - '--director-origin', config.directorOrigin, - '--cell-origin', config.cellOrigin, - '--cell-id', config.cellId, - '--admission', 'general', - '--expected-image-digest', digest, - '--metrics-after', new Date(config.metricsAfter).toISOString(), - '--metrics-file', '/tmp/metrics.json' - ]), - { ...config, metricsFile: '/tmp/metrics.json' } - ) -}) - -test('accepts the exact old runtime shape with two fresh zero samples', async () => { - const result = await verifyLegacyBootstrap(config, { - fetch: harness(), - metrics, - now: () => now, - token: 'masked-token' - }) - assert.deepEqual( - { ...result, incarnationDigest: undefined }, - { - cellId: config.cellId, - admissionState: 'general', - assignments: 12, - metricSamples: 2, - incarnationDigest: undefined - } - ) - assert.match(result.incarnationDigest, /^[a-f0-9]{64}$/) -}) - -test('rejects a new or unknown runtime shape on the legacy-only path', async () => { - await assert.rejects( - verifyLegacyBootstrap(config, { - fetch: harness({ runtime: { draining: false } }), - metrics, - now: () => now, - token: 'masked-token' - }), - /reviewed legacy contract/ - ) -}) - -test('rejects active durable state and active runtime metrics', async () => { - await assert.rejects( - verifyLegacyBootstrap(config, { - fetch: harness({ status: { activityLeases: 1 } }), - metrics, - now: () => now, - token: 'masked-token' - }), - /durable activity/ - ) - await assert.rejects( - verifyLegacyBootstrap(config, { - fetch: harness(), - metrics: metrics.map((entry, index) => ({ - ...entry, - controls: index === 0 ? 1 : 0 - })), - now: () => now, - token: 'masked-token' - }), - /not quiescent/ - ) - await assert.rejects( - verifyLegacyBootstrap(config, { - fetch: harness(), - metrics: metrics.map((entry) => ({ ...entry, pendingSplices: null })), - now: () => now, - token: 'masked-token' - }), - /pendingSplices is invalid/ - ) -}) - -test('rejects stale, pre-boundary, or single runtime samples', async () => { - for (const invalidMetrics of [ - metrics.map((entry) => ({ ...entry, timestamp: new Date(now - 180_000).toISOString() })), - [metrics[0]], - metrics.map((entry) => ({ ...entry, timestamp: new Date(now - 100_000).toISOString() })) - ]) { - await assert.rejects( - verifyLegacyBootstrap(config, { - fetch: harness(), - metrics: invalidMetrics, - now: () => now, - token: 'masked-token' - }), - /samples|stale/ - ) - } -}) - -test('rejects the wrong legacy image and an unready cell', async () => { - await assert.rejects( - verifyLegacyBootstrap(config, { - fetch: harness({ runtime: { imageDigest: `sha256:${'b'.repeat(64)}` } }), - metrics, - now: () => now, - token: 'masked-token' - }), - /identity does not match/ - ) - await assert.rejects( - verifyLegacyBootstrap(config, { - fetch: harness({ ready: false }), - metrics, - now: () => now, - token: 'masked-token' - }), - /readiness returned 503/ - ) -}) - -test('binds the director heartbeat to the replacement boundary', async () => { - const replacementConfig = { ...config, runtimeStartedAfter: now - 60_000 } - await assert.rejects( - verifyLegacyBootstrap(replacementConfig, { - fetch: harness(), - metrics, - now: () => now, - token: 'masked-token' - }), - /heartbeat predates/ - ) - await verifyLegacyBootstrap(replacementConfig, { - fetch: harness({ status: { runtime: { - cellUrl: config.cellOrigin, - cellIncarnation: '00000000-0000-4000-8000-000000000002', - startedAt: now - 30_000, - ready: true, - observedRequests: 0, - lastHeartbeatAt: now - 1_000, - heartbeatFresh: true - } } }), - metrics, - now: () => now, - token: 'masked-token' - }) -}) - -test('accepts a drained migration-only legacy target', async () => { - const migrationConfig = { ...config, admission: 'migration-only' } - const result = await verifyLegacyBootstrap(migrationConfig, { - fetch: harness({ status: { admissionState: 'migration-only' } }), - metrics, - now: () => now, - token: 'masked-token' - }) - assert.equal(result.admissionState, 'migration-only') -}) - -test('accepts exact stale capacity during the director-first transition', async () => { - const capacity = { - hardCap: 600, - unobservedBound: 60, - controlRebindReserve: 100, - ordinaryConnectionLimit: 500, - normalAdmissionPause: 440, - observedConnections: 0, - inFlightConnections: 0, - reservedConnectionUnits: 0, - enforcedConnectionUnits: 0, - pendingControlReservations: 0, - heartbeatFresh: false - } - const transition = { - ...config, - admission: 'migration-only', - hardCap: 600, - unobservedBound: 60, - capacityState: 'stale' - } - await verifyLegacyBootstrap(transition, { - fetch: harness({ status: { admissionState: 'migration-only', connectionCapacity: capacity } }), - metrics, - now: () => now, - token: 'masked-token' - }) - await assert.rejects( - verifyLegacyBootstrap(transition, { - fetch: harness({ status: { - admissionState: 'migration-only', - connectionCapacity: { ...capacity, heartbeatFresh: true } - } }), - metrics, - now: () => now, - token: 'masked-token' - }), - /capacity does not match/ - ) -}) - -test('resumes either legacy cell after the director update', async () => { - const capacity = { - hardCap: 600, - unobservedBound: 60, - controlRebindReserve: 100, - ordinaryConnectionLimit: 500, - normalAdmissionPause: 440, - observedConnections: 0, - inFlightConnections: 0, - reservedConnectionUnits: 0, - enforcedConnectionUnits: 0, - pendingControlReservations: 0, - heartbeatFresh: false - } - for (const number of [2, 3]) { - const cell = { - ...config, - cellId: `staging-gce-c${number}`, - cellOrigin: `https://c${number}.relay.example.com`, - admission: 'migration-only', - hardCap: 600, - unobservedBound: 60, - capacityState: 'absent-or-stale' - } - const cellMetrics = metrics.map((entry) => ({ ...entry, cellId: cell.cellId })) - for (const connectionCapacity of [null, capacity]) { - await verifyLegacyBootstrap(cell, { - fetch: harness({ - cell, - runtimeHeartbeatFresh: connectionCapacity === null, - status: { admissionState: 'migration-only', connectionCapacity } - }), - metrics: cellMetrics, - now: () => now, - token: 'masked-token' - }) - } - } -}) - -test('requires a fresh director heartbeat before capacity is configured', async () => { - await assert.rejects( - verifyLegacyBootstrap(config, { - fetch: harness({ runtimeHeartbeatFresh: false }), - metrics, - now: () => now, - token: 'masked-token' - }), - /fresh, ready/ - ) -}) - -test('requires a new director heartbeat incarnation after restart', async () => { - const before = await verifyLegacyBootstrap(config, { - fetch: harness(), - metrics, - now: () => now, - token: 'masked-token' - }) - await assert.rejects( - verifyLegacyBootstrap( - { ...config, previousIncarnationDigest: before.incarnationDigest }, - { fetch: harness(), metrics, now: () => now, token: 'masked-token' } - ), - /incarnation did not change/ - ) -}) - -test('rejects same-instance samples written before restart completion', async () => { - await assert.rejects( - verifyLegacyBootstrap( - { ...config, metricsAfter: now - 20_000 }, - { fetch: harness(), metrics, now: () => now, token: 'masked-token' } - ), - /post-boundary samples/ - ) -}) - -test('the bootstrap workflow binds zero metrics to a replacement C3 instance', async () => { - const { readFile } = await import('node:fs/promises') - const workflow = await readFile( - relayWorkflowUrl('bootstrap-relay-staging-capacity.yml'), - 'utf8' - ) - assert.match(workflow, /resource\.labels\.instance_id=.*\$\{instance_id\}/) - assert.match(workflow, /timestamp>=.*\$\{after\}/) - assert.doesNotMatch(workflow, /legacy_c3_instance_id.*!=/) - const c2Proof = workflow.indexOf('staging-gce-c2 general "${legacy_pre_boundary}"') - const isolate = workflow.indexOf('--mode isolate', c2Proof) - const drainedProof = workflow.indexOf('staging-gce-c3 migration-only', isolate) - const recreate = workflow.indexOf('recreate-instances', drainedProof) - const replacementProof = workflow.indexOf('"${legacy_c3_old_incarnation}"', recreate) - const stable = workflow.indexOf('wait-until', recreate) - const metricsBoundary = workflow.indexOf('legacy_c3_metrics_boundary=', stable) - assert.ok(c2Proof < isolate && isolate < drainedProof && drainedProof < recreate) - assert.ok(recreate < stable && stable < metricsBoundary && metricsBoundary < replacementProof) - assert.match(workflow, /recreate-instances[\s\S]*?--instances "\$\{legacy_c3_instance\}"/) - assert.match(workflow, /incarnation_args=\(--previous-incarnation-digest/) - assert.match(workflow, /trap restore_legacy_c3_fallback EXIT/) - assert.match(workflow, /--mode restore-fallback[\s\S]*--general-cell-ids staging-gce-c2/) -}) diff --git a/cloud/dev/scripts/watch-relay-reserve-cell.mjs b/cloud/dev/scripts/watch-relay-reserve-cell.mjs deleted file mode 100644 index 7e608756604..00000000000 --- a/cloud/dev/scripts/watch-relay-reserve-cell.mjs +++ /dev/null @@ -1,875 +0,0 @@ -#!/usr/bin/env node -// Watches one cell already in reserve mode and, on the first stop rule it trips, flips it back to -// db through the flag tool's own write (CAS, read-back, re-registration wait). Step 5 design §9. -// -// watch: one segment of the watch (the workflow re-authenticates between segments, since an -// ID token lasts an hour). State carries over in --state-file. Writes nothing. -// outcome: the job's state and verdict, from the state file. -// flip-back: the flag tool's admitMode=db write (or its dry run), then exits non-zero with the -// watch's verdict. Why it holds no flag-workflow group: docs/relay-workflows.md. -// alarm: a refused watch or a flip back that did not land: an error annotation, and in write -// mode a log line the reserve-watch alert pages on. -import { execFile } from 'node:child_process' -import { appendFile, readFile, writeFile } from 'node:fs/promises' -import { parseArgs, promisify } from 'node:util' -import { pathToFileURL } from 'node:url' -import { - BREAK_GLASS_CONFIRMATION, - operateCellFlags, - parseCellFlagsRequest, - readCurrentObject, - readRuntime, - WRITE_CONFIRMATION -} from './operate-relay-cell-flags.mjs' -import { - DIRECTOR_DRAIN_FIELDS, - DIRECTOR_METRICS_LIMIT, - directorMetricsSample, - readDirectorDrainReturn, - readDirectorRequestsByCodePerMinute, - readLogEntries -} from './relay-same-cap-shadow-gate.mjs' -import { - DIRECTOR_METRICS_INTERVAL_MS, - drainReturnByMinute, - minuteKey, - minutesOf, - withoutDrainDeferrals -} from './relay-same-cap-shadow-gate-verdict.mjs' -import { PRODUCTION_LEASE } from './cloud-sql-rollout-lock-census.mjs' -import { RELAY_REPOSITORY_ROOT } from './relay-repository.mjs' -import { - evaluateWatchSample, - initialWatchState, - measuredBackground, - measuredPhoneBackground, - phoneSinceOf, - PHONE_WATCH_RULES, - RESERVE_WATCH_RULES, - UNATTRIBUTED_503S -} from './relay-reserve-watch-rules.mjs' - -const execFileAsync = promisify(execFile) -const MINUTE_MS = 60_000 -export const SAMPLE_INTERVAL_MS = MINUTE_MS -export const MAX_WATCH_MINUTES = 300 -// Below the hour an ID token lasts, with room for the segment's last sample. -export const MAX_SEGMENT_MINUTES = 55 -const METRIC_DELAY_MS = RESERVE_WATCH_RULES.director503.publishDelayMs -const BACKGROUND_MINUTES = 10 -// A flipped-at older than this is a typo or a stale dispatch, not this flip. -const MAX_FLIPPED_AT_AGE_MS = 24 * 60 * MINUTE_MS -export const FLIP_BACK_ATTEMPTS = 5 -const FLIP_BACK_BACKOFF_MS = 2_000 -// One slow runtime-status reply is not a dead cell: retried this often before break glass. -export const FLIP_BACK_RUNTIME_ATTEMPTS = 4 -const SAMPLE_READ_BUDGET_MS = 40_000 -const LOG_ENTRY_LIMIT = 1_000 - -function integerIn(name, raw, min, max) { - const value = Number(raw) - if (!/^[0-9]+$/.test(raw ?? '') || value < min || value > max) { - throw new Error(`--${name} must be an integer from ${min} to ${max}`) - } - return value -} - -// The flip back's request. Its generation is a placeholder: the flip back re-reads the current one. -// A phone-rule trip turns off only the phone path; anything else flips the whole cell back. Both -// remove the phonePath key, so a cell rolled back to an image without the switch takes the write. -export function dbFlipRequest(values) { - const set = values.flip === 'phonePath' ? 'phonePath=default' : 'admitMode=db,phonePath=default' - return parseCellFlagsRequest({ ...values, set, 'expected-generation': '0' }) -} - -// `now` is the watch's start: a later segment of a long watch must not age flipped-at out. -function parseFlippedAt(raw, now) { - if (raw === '') return null - const at = Date.parse(raw) - if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d{1,3})?)?(Z|[+-]\d{2}:\d{2})$/.test(raw) || !Number.isFinite(at)) { - throw new Error('--flipped-at must be an ISO time with a zone, e.g. 2026-10-14T12:00:00Z') - } - if (at > now) throw new Error('--flipped-at is in the future') - if (at < now - MAX_FLIPPED_AT_AGE_MS) throw new Error('--flipped-at is more than 24 hours ago') - return at -} - -export function parseWatchArguments(values, now = Date.now()) { - // The flag tool's own checks, so the watch is of exactly the cell the flip back would write; - // a write-mode watch refuses a missing confirmation now, not hours later. - const request = dbFlipRequest(values) - if (!/^[a-z][a-z0-9-]{0,62}$/.test(values['director-service'] ?? '')) { - throw new Error('--director-service is not a Cloud Run service name') - } - const background = values['background-503-per-minute'] ?? '' - if (background !== '' && !/^(0|[1-9][0-9]{0,6})(\.[0-9]+)?$/.test(background)) { - throw new Error('--background-503-per-minute must be empty (measure it) or a non-negative number') - } - // The background must predate the flip: a watch starts after it, so its own past is not one. - const flippedAt = parseFlippedAt(values['flipped-at'] ?? '', now) - if (background === '' && flippedAt === null) { - throw new Error('--flipped-at (the background is the 10 minutes before it) or --background-503-per-minute is required') - } - if (!values['state-file']) throw new Error('--state-file is required') - return { - request, - // The flag tool's own inputs, for turning the phone path off mid-watch. - flipValues: values, - directorService: values['director-service'], - durationMinutes: integerIn('duration-minutes', values['duration-minutes'], 5, MAX_WATCH_MINUTES), - segmentMinutes: integerIn('segment-minutes', values['segment-minutes'], 1, MAX_SEGMENT_MINUTES), - unverifiableLimit: integerIn('unverifiable-limit', values['unverifiable-limit'], 1, 30), - background503PerMinute: background === '' ? null : Number(background), - flippedAt, - stateFile: values['state-file'], - githubOutput: values['github-output'] || null - } -} - -const failedRead = (error) => ({ ok: false, error: error instanceof Error ? error.message : String(error) }) - -export async function readCellSample(fetchImpl, request, idToken, wait) { - try { - const runtime = await readRuntime(fetchImpl, request, idToken, wait) - if (runtime.role !== 'cell' || runtime.cellId !== request.cellId) { - return failedRead(`origin answers as ${runtime.role}/${runtime.cellId}`) - } - if (!runtime.flagsApplied) return failedRead('runtime-status reports no flagsApplied') - const controls = runtime.runtime?.controls - if (!Number.isInteger(controls)) return failedRead('runtime-status reports no control count') - return { - ok: true, - admitModeConfigured: runtime.flagsApplied.flags?.admitMode ?? 'db', - admitModeEffective: runtime.admitModeEffective, - phonePathConfigured: runtime.flagsApplied.flags?.phonePath ?? 'db', - resumeAuditDropped: runtime.runtime?.resumeAuditDropped, - draining: runtime.draining === true, - controls, - enforcedUnits: runtime.runtime?.enforcedConnectionUnits, - hardCap: runtime.connectionCapacity?.hardCap - } - } catch (error) { - return failedRead(error) - } -} - -const minuteFloor = (at) => Math.floor(at / MINUTE_MS) * MINUTE_MS -const failed503Read = (error) => ({ ok: false, error, perMinute: {}, minutes: [], publishedThrough: null, unmetered: [] }) - -// The `minutes` whole minutes that end at the minute boundary `endedAt`: the director's non-drain -// 503s, the roll ladder's measure its line comes from. Cloud Run's 503 count less what the -// director's runtime metrics attribute to drain-return deferrals and to a host re-dialling inside -// its own retry interval or while its last dial is in flight (10-18: ~98% of 503s), since those -// grow with traffic, not trouble. `publishedThrough` is the newest minute any response code has a -// point for (the director always serves something, so no front at all is no evidence); a minute -// with no metrics line from any instance is `unmetered`, since subtracting nothing there is a guess. -// Coverage is any instance, not each: one instance's missing lines under-subtract, biased to a trip. -export async function readDirector503Minutes(reader, { projectId, directorService, endedAt, minutes, metrics: logged }) { - const span = { startedAt: new Date(endedAt - minutes * MINUTE_MS), endedAt: new Date(endedAt) } - const config = { projectId, directorService } - let read - try { - read = await readDirectorRequestsByCodePerMinute(reader, { config, span }) - } catch (error) { - // A point with an unparseable end time has no minute. - return failed503Read(`director 503 metric unreadable: ${error instanceof Error ? error.message : error}`) - } - if (read.failed) return failed503Read('director 503 metric read failed') - // The sample's combined log read passes these in; a background read takes its own. - const metrics = logged ?? (await readDirectorDrainReturn(reader, { config, window: directorMetricsWindow(endedAt, minutes) })) - const failedPart = metrics.find((part) => part.failed && part.error) - if (failedPart) return failed503Read(`director runtime metrics read failed: ${failedPart.error}`) - const lineTimes = metrics.flatMap((part) => part.samples.map((sample) => Date.parse(sample.timestamp))) - if (lineTimes.some((at) => !Number.isFinite(at))) return failed503Read('director runtime metrics line with an unparseable timestamp') - const scheduled = drainReturnByMinute(metrics, DIRECTOR_METRICS_LIMIT) - if (scheduled.truncated) return failed503Read('director runtime metrics read failed, truncated, or short') - const keys = minutesOf(span) - const metered = new Set(lineTimes.flatMap((at) => [minuteKey(at - DIRECTOR_METRICS_INTERVAL_MS), minuteKey(at - 1)])) - const nonDrain = withoutDrainDeferrals({ perMinute: read.byCode['503'] ?? {} }, scheduled, keys) - const published = Object.values(read.byCode).flatMap((perMinute) => Object.keys(perMinute)).sort() - return { - ok: true, - perMinute: Object.fromEntries(keys.map((minute, index) => [minute, nonDrain.series[index]])), - minutes: keys, - publishedThrough: published.at(-1) ?? null, - unmetered: keys.filter((minute) => !metered.has(minute)), - instances: director503ByInstance(metrics, keys, nonDrain.series) - } -} - -// A line covers the 30 s before it, so the last minute's tail is in the next minute's lines. -const directorMetricsWindow = (endedAt, minutes) => ({ - startedAt: new Date(endedAt - minutes * MINUTE_MS), - endedAt: new Date(endedAt + DIRECTOR_METRICS_INTERVAL_MS) -}) - -// The same non-drain measure per director instance, from each instance's own runtime-metrics -// counts of the /v1/assign 503s it answered, less its own scheduled ones. 10-21 07:55Z read 40, -// 169, 5 on the stalled instance this way, against 54, 155, 5 in Cloud Run's request log. -// `directors` is, per minute, how many instances logged a line ending in it. -function director503ByInstance(metrics, keys, fleet) { - const samples = metrics.flatMap((part) => part.samples) - const perInstance = {} - for (const instance of new Set(samples.map((sample) => sample.instanceId).filter(Boolean))) { - const own = drainReturnByMinute([{ samples: samples.filter((sample) => sample.instanceId === instance) }], DIRECTOR_METRICS_LIMIT) - const { series } = withoutDrainDeferrals({ perMinute: own.assign503sPerMinute }, own, keys) - perInstance[instance] = Object.fromEntries(keys.map((minute, index) => [minute, series[index]])) - } - // Cloud Run's own 503s, or a route the director does not count; an instance that stopped logging - // lands here too. - perInstance[UNATTRIBUTED_503S] = Object.fromEntries( - keys.map((minute, index) => [minute, fleet[index] - Object.values(perInstance).reduce((sum, byMinute) => sum + (byMinute[minute] ?? 0), 0)]) - ) - const logged = new Map(keys.map((minute) => [minute, new Set()])) - for (const sample of samples) { - if (sample.instanceId) logged.get(minuteKey(Date.parse(sample.timestamp) - 1))?.add(sample.instanceId) - } - const directors = Object.fromEntries([...logged].map(([minute, instances]) => [minute, instances.size])) - return { ok: true, perMinute: perInstance, directors } -} - -const CELL_METRICS_EVENT = 'orca_relay_runtime_metrics' -const PHONE_SIGNALS = ['credentialRefusals', 'confirmFailures', 'attachDeadlines', 'acceptDatabaseErrors', 'renewalFailures', 'completedAccepts'] -const sumOf = (counts) => Object.values(counts ?? {}).reduce((sum, count) => sum + Number(count), 0) - -// The cell's own runtime metrics lines (one per 30 s) for the `minutes` whole minutes ending at -// `endedAt`, per minute: what the phone rules judge. A line counts toward the minute it ends in; -// a minute no line covers is `unmetered`, never zero. -function cellPhoneQuery({ cellId, endedAt, minutes }) { - return { - filter: [ - 'resource.type="gce_instance"', - `jsonPayload.event="${CELL_METRICS_EVENT}"`, - `jsonPayload.cellId="${cellId}"`, - ...timestampClauses(endedAt - minutes * MINUTE_MS, endedAt + DIRECTOR_METRICS_INTERVAL_MS) - ].join(' AND '), - fields: [ - 'jsonPayload.phoneRejectionsByCodeDelta', 'jsonPayload.resumeConfirmFailuresByCodeDelta', 'jsonPayload.phoneAttachDeadlinesDelta', - 'jsonPayload.phoneAcceptDatabaseErrorsDelta', 'jsonPayload.resumeRenewalFailuresDelta', 'jsonPayload.clientAcceptCompletedDelta' - ], - match: (entry) => entry.resource?.type === 'gce_instance' - } -} - -export async function readCellPhoneMinutes(reader, request) { - return parseCellPhoneMinutes(await readQuery(reader, cellPhoneQuery(request)), request) -} - -function parseCellPhoneMinutes(read, { endedAt, minutes }) { - const span = { startedAt: new Date(endedAt - minutes * MINUTE_MS), endedAt: new Date(endedAt) } - if (read.failed) return failedRead(read.error) - if (read.entries.length >= LOG_ENTRY_LIMIT) return failedRead('cell runtime metrics read truncated') - const keys = minutesOf(span) - const series = Object.fromEntries(PHONE_SIGNALS.map((signal) => [signal, keys.map(() => 0)])) - const metered = new Set() - for (const entry of read.entries) { - const at = Date.parse(entry.timestamp) - if (!Number.isFinite(at)) return failedRead('cell runtime metrics line with an unparseable timestamp') - const payload = entry.jsonPayload ?? {} - if (payload.phoneAttachDeadlinesDelta === undefined) { - return { ...failedRead('cell runtime metrics carry no phone counters (an image before step 6)'), permanent: true } - } - metered.add(minuteKey(at - DIRECTOR_METRICS_INTERVAL_MS)).add(minuteKey(at - 1)) - const index = keys.indexOf(minuteKey(at - 1)) - if (index < 0) continue - series.credentialRefusals[index] += Number(payload.phoneRejectionsByCodeDelta?.['4401'] ?? 0) - series.confirmFailures[index] += sumOf(payload.resumeConfirmFailuresByCodeDelta) - series.attachDeadlines[index] += Number(payload.phoneAttachDeadlinesDelta) - series.acceptDatabaseErrors[index] += Number(payload.phoneAcceptDatabaseErrorsDelta ?? 0) - series.renewalFailures[index] += Number(payload.resumeRenewalFailuresDelta ?? 0) - series.completedAccepts[index] += Number(payload.clientAcceptCompletedDelta ?? 0) - } - return { ok: true, minutes: keys, series, unmetered: keys.filter((minute) => !metered.has(minute)) } -} - -const LEDGER_SUMMARY_EVENT = 'orca_relay_reserve_ledger_summary' -const SEAT_BEHIND_ROW_EVENT = 'orca_relay_reserve_ledger_seat_behind_row' - -function directorLogFilter(directorService, eventClause, sinceAt) { - return [ - 'resource.type="cloud_run_revision"', - `resource.labels.service_name="${directorService}"`, - eventClause, - `timestamp>="${new Date(sinceAt).toISOString()}"` - ].join(' AND ') -} - -const atOrAfter = (entry, sinceAt) => Date.parse(entry.timestamp) >= sinceAt -const isEvent = (entry, event) => entry.resource?.type === 'cloud_run_revision' && entry.jsonPayload?.event === event - -function ledgerQuery({ directorService, at }) { - const sinceAt = at - RESERVE_WATCH_RULES.ledgerSummaryMaxAgeMs - return { - filter: directorLogFilter(directorService, `jsonPayload.event="${LEDGER_SUMMARY_EVENT}"`, sinceAt), - fields: ['jsonPayload.pending', 'jsonPayload.dropped'], - match: (entry) => isEvent(entry, LEDGER_SUMMARY_EVENT) && atOrAfter(entry, sinceAt) - } -} - -export async function readLedgerSample(reader, request) { - return parseLedgerSample(await readQuery(reader, ledgerQuery(request))) -} - -// Newest summary per director instance; queued plus dropped rows are the ones not in Postgres. -function parseLedgerSample(read) { - if (read.failed) return failedRead(read.error) - if (read.entries.length >= LOG_ENTRY_LIMIT) return failedRead('ledger summary read truncated') - if (read.entries.length === 0) return failedRead('no placing director logged a ledger summary') - if (read.entries.some((entry) => !Number.isFinite(Date.parse(entry.timestamp)))) { - return failedRead('ledger summary with an unparseable timestamp') - } - const newest = new Map() - for (const entry of read.entries) { - const instance = entry.labels?.instanceId ?? 'unknown' - const previous = newest.get(instance) - if (!previous || Date.parse(entry.timestamp) > Date.parse(previous.timestamp)) newest.set(instance, entry) - } - let unwrittenRows = 0 - for (const entry of newest.values()) { - const pending = Number(entry.jsonPayload?.pending) - const dropped = Number(entry.jsonPayload?.dropped) - if (!Number.isFinite(pending) || !Number.isFinite(dropped)) return failedRead('ledger summary without counts') - unwrittenRows += pending + dropped - } - return { ok: true, unwrittenRows, directors: newest.size } -} - -// The same read also takes the ledger summary every placing director logs each minute, as a -// positive control: with no fresh summary in it, no seat-behind-row line is blindness, not zero. -function seatBehindRowQuery({ directorService, cellId, at }) { - const sinceAt = at - RESERVE_WATCH_RULES.duplicateSeat.lookbackMs - return { - filter: directorLogFilter( - directorService, - `((jsonPayload.event="${SEAT_BEHIND_ROW_EVENT}" AND jsonPayload.cellId="${cellId}") OR jsonPayload.event="${LEDGER_SUMMARY_EVENT}")`, - sinceAt - ), - fields: ['jsonPayload.epoch', 'jsonPayload.rowEpoch'], - match: (entry) => - atOrAfter(entry, sinceAt) && - ((isEvent(entry, SEAT_BEHIND_ROW_EVENT) && entry.jsonPayload?.cellId === cellId) || isEvent(entry, LEDGER_SUMMARY_EVENT)) - } -} - -export async function readSeatBehindRowSample(reader, request) { - return parseSeatBehindRowSample(await readQuery(reader, seatBehindRowQuery(request)), request) -} - -function parseSeatBehindRowSample(read, { cellId, at }) { - if (read.failed) return failedRead(read.error) - if (read.entries.length >= LOG_ENTRY_LIMIT) return failedRead('seat-behind-row read truncated') - const reports = [] - let controlSeen = false - for (const entry of read.entries) { - const reportedAt = Date.parse(entry.timestamp) - if (!Number.isFinite(reportedAt)) return failedRead('seat-behind-row line with an unparseable timestamp') - if (entry.jsonPayload?.event === LEDGER_SUMMARY_EVENT) { - controlSeen ||= reportedAt >= at - RESERVE_WATCH_RULES.ledgerSummaryMaxAgeMs - continue - } - const { epoch, rowEpoch } = entry.jsonPayload ?? {} - if (epoch === undefined || rowEpoch === undefined) return failedRead('seat-behind-row line without epochs') - // Everything the line carries but its time, which differs on every report of one seat. A - // proxy for the host: two hosts sharing it read as one seat, so it can only over-count. - reports.push({ at: reportedAt, key: `${entry.labels?.instanceId ?? 'unknown'}/${cellId}/${epoch}:${rowEpoch}` }) - } - if (!controlSeen) return failedRead('seat-behind-row read holds no fresh ledger summary (positive control)') - return { ok: true, reports } -} - -const timestampClauses = (startedAt, endedAt) => [ - `timestamp>="${new Date(startedAt).toISOString()}"`, - `timestamp<"${new Date(endedAt).toISOString()}"` -] -const projectionOf = (queries) => - `json(${[...new Set(['timestamp', 'resource.type', 'labels.instanceId', 'jsonPayload.event', 'jsonPayload.cellId', ...queries.flatMap((query) => query.fields)])].join(',')})` -const readQuery = (reader, query) => readLogEntries(reader, { filter: query.filter, projection: projectionOf([query]), limit: LOG_ENTRY_LIMIT }) - -function directorMetricsQuery({ directorService, endedAt, minutes }) { - const window = directorMetricsWindow(endedAt, minutes) - return { - filter: [ - 'resource.type="cloud_run_revision"', - `resource.labels.service_name="${directorService}"`, - `jsonPayload.event="${CELL_METRICS_EVENT}"`, - ...timestampClauses(window.startedAt, window.endedAt) - ].join(' AND '), - fields: DIRECTOR_DRAIN_FIELDS.map((field) => `jsonPayload.${field}`), - match: (entry) => isEvent(entry, CELL_METRICS_EVENT), - minSamples: Math.floor((window.endedAt - window.startedAt) / DIRECTOR_METRICS_INTERVAL_MS) - } -} - -// The queries one sample sends; `phoneRequest` is false while the phone path is not watched. -export const sampleLogQueries = ({ phoneRequest, director, seatRequest }) => ({ - phone: phoneRequest && cellPhoneQuery(phoneRequest), - directorMetrics: directorMetricsQuery(director), - ledger: ledgerQuery({ directorService: director.directorService, at: seatRequest.at }), - seatBehindRow: seatBehindRowQuery(seatRequest) -}) - -// Every Cloud Logging query a sample needs, as one read: the project allows 60 reads a minute -// across every watch and tool, and on 10-21 18:39Z twelve watches at five reads a sample drew 429s -// until c21 and c22 tripped unverifiable. Each query keeps its own filter and takes back the lines -// that are its own; the combined read truncated or failed fails every one of them. -export async function readSampleLogs(reader, queries) { - const named = Object.entries(queries).filter(([, query]) => query) - const read = await readLogEntries(reader, { - filter: named.map(([, query]) => `(${query.filter})`).join(' OR '), - projection: projectionOf(named.map(([, query]) => query)), - limit: LOG_ENTRY_LIMIT - }) - const failure = read.failed - ? read.error - : read.entries.length >= LOG_ENTRY_LIMIT - ? 'sample log read truncated' - : read.entries.some((entry) => !Number.isFinite(Date.parse(entry.timestamp))) - ? 'sample log line with an unparseable timestamp' - : null - return Object.fromEntries( - named.map(([name, query]) => [name, failure ? { failed: true, error: failure, entries: [] } : { failed: false, entries: read.entries.filter(query.match) }]) - ) -} - -function createReader(dependencies, projectId) { - const now = dependencies.now - return { - runGcloud: dependencies.runGcloud, - fetch: dependencies.fetchImpl, - retryDelayMs: dependencies.retryDelayMs ?? 5_000, - readTimeoutMs: 20_000, - now, - // Every sample gets its own budget, so a slow read costs one sample, not the watch. - deadlineAt: now() + SAMPLE_READ_BUDGET_MS, - projectId - } -} - -/** - * Same-cap and capacity rolls hold the Cloud SQL rollout lease for the whole run; it names no - * cell, so any live holder refuses the watch's start. Read only: holding it for hours would block - * every rollout. An unreadable lease refuses too (fail closed, as the lease action does). - */ -export async function rolloutLeaseRefusal(readRolloutLease, now) { - try { - const existing = await readRolloutLease() - if (!existing || existing.record.expires_at <= now) return null - return ( - `a rollout holds the Cloud SQL rollout lease until ${new Date(existing.record.expires_at).toISOString()} ` + - `(${existing.holder}); start the watch after it ends` - ) - } catch (error) { - return `the Cloud SQL rollout lease is unreadable (${error instanceof Error ? error.message : error}); refusing to start` - } -} - -// The lease action's own reader, resolved from the repository root because this tree's depth -// differs between the two repositories' copies. -async function readProductionRolloutLease() { - const action = new URL('.github/actions/cloud-sql-rollout-lease/', RELAY_REPOSITORY_ROOT) - const [{ CloudSqlRolloutLease, describeHolder }, { createAccessTokenSource }] = await Promise.all([ - import(new URL('storage-lease.mjs', action).href), - import(new URL('gcloud-access-token.mjs', action).href) - ]) - const lease = new CloudSqlRolloutLease({ - bucket: PRODUCTION_LEASE.bucket, - objectName: PRODUCTION_LEASE.object, - accessToken: createAccessTokenSource() - }) - const existing = await lease.read() - return existing && { record: existing.record, holder: describeHolder(existing.record) } -} - -// Only a missing file is a fresh watch. Anything else that is not a state fails the segment, which -// the outcome step reads as failed and flips back: a corrupt file must not restart or refuse the watch. -export async function loadState(path) { - const text = await readFile(path, 'utf8').catch((error) => (error?.code === 'ENOENT' ? null : Promise.reject(error))) - if (text === null) return null - const state = JSON.parse(text) - const isObject = state !== null && typeof state === 'object' && !Array.isArray(state) - if (!isObject || typeof state.state !== 'string') throw new Error(`state file ${path} holds no watch state`) - if (state.state === 'watching' && (typeof state.cellId !== 'string' || !Number.isFinite(state.startedAt))) { - throw new Error(`state file ${path} has no cell or start`) - } - return state -} - -// A segment's config. Later segments check flipped-at against the watch's start; bad inputs before -// the first sample are refused, so no flip back follows a typo. -export async function parseWatchSegment(values, now = Date.now()) { - const existing = values['state-file'] ? await loadState(values['state-file']) : null - try { - return parseWatchArguments(values, Number.isFinite(existing?.startedAt) ? existing.startedAt : now) - } catch (error) { - if (values['state-file'] && existing === null) { - await writeFile(values['state-file'], JSON.stringify({ state: 'refused', verdict: error.message })) - } - throw error - } -} - -export async function watchSegment(config, dependencies) { - const now = dependencies.now ?? Date.now - const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) - const log = dependencies.log ?? console.log - const deps = { ...dependencies, now, sleep, fetchImpl: dependencies.fetchImpl ?? fetch } - const { request, directorService } = config - let state = await loadState(config.stateFile) - if (state === null) { - const refusal = await rolloutLeaseRefusal(dependencies.readRolloutLease, now()) - if (refusal) { - state = { state: 'refused', verdict: refusal } - await writeFile(config.stateFile, JSON.stringify(state)) - if (config.githubOutput) await writeOutcome(config.githubOutput, state) - return state - } - state = initialWatchState({ - cellId: request.cellId, - startedAt: now(), - durationMinutes: config.durationMinutes, - background503PerMinute: config.background503PerMinute, - flippedAt: config.flippedAt - }) - } - if (state.cellId !== request.cellId) throw new Error(`state file is for ${state.cellId}, not ${request.cellId}`) - const segmentEndsAt = now() + config.segmentMinutes * MINUTE_MS - while (state.state === 'watching' && now() < segmentEndsAt) { - const at = now() - const baselineEndsAt = state.flippedAt === null ? null : minuteFloor(state.flippedAt) - if (!state.background && baselineEndsAt !== null && at >= baselineEndsAt + METRIC_DELAY_MS) { - // The minutes just before the flip, once published; retried every sample until one reads. - const read = await readDirector503Minutes(createReader(deps, request.projectId), { - projectId: request.projectId, - directorService, - endedAt: baselineEndsAt, - minutes: BACKGROUND_MINUTES - }) - state = { ...state, background: read.ok ? measuredBackground(read) : null } - } - const reader = createReader(deps, request.projectId) - // Serialised: concurrent Logging reads are what earn a 429. - const cell = await readCellSample(deps.fetchImpl, request, dependencies.idToken, sleep) - const phoneWatched = cell.ok && cell.phonePathConfigured === 'memory' && state.phoneOffAt === null - const phoneSince = phoneSinceOf(cell, state) - const phoneBaselineEndsAt = phoneSince === null ? null : minuteFloor(phoneSince) - if (phoneWatched && !state.phoneBackground && phoneBaselineEndsAt !== null && at >= phoneBaselineEndsAt + METRIC_DELAY_MS) { - const read = await readCellPhoneMinutes(reader, { - cellId: request.cellId, - endedAt: phoneBaselineEndsAt, - minutes: BACKGROUND_MINUTES - }) - // A restart or roll in those minutes leaves no background to judge against: no phone - // rule could ever pass, so refuse rather than turn the phone path off for certain. - const incomplete = read.ok ? read.unmetered.length > 0 && `no metrics for ${read.unmetered.join(', ')}` : read.permanent && read.error - if (incomplete) { - state = { - ...state, - state: 'refused', - verdict: - `${request.cellId}: the phone background, the 10 minutes before ${new Date(phoneBaselineEndsAt).toISOString()}, ` + - `is incomplete (${incomplete}), as after a cell restart or roll; start a new watch with a later --flipped-at` - } - await writeFile(config.stateFile, JSON.stringify(state)) - break - } - state = { ...state, phoneBackground: measuredPhoneBackground(read) } - } - // Complete minutes only, held back past the publishing delay. - const endedAt = minuteFloor(at - METRIC_DELAY_MS) - const phoneRequest = { cellId: request.cellId, endedAt, minutes: PHONE_WATCH_RULES.readMinutes } - const director = { directorService, endedAt, minutes: RESERVE_WATCH_RULES.director503.readMinutes } - const seatRequest = { directorService, cellId: request.cellId, at } - const queries = sampleLogQueries({ phoneRequest: phoneWatched && phoneRequest, director, seatRequest }) - const logs = await readSampleLogs(reader, queries) - const directorMetrics = { - failed: logs.directorMetrics.failed, - error: logs.directorMetrics.error, - samples: logs.directorMetrics.entries.map(directorMetricsSample), - minSamples: queries.directorMetrics.minSamples - } - const sample = { - at, - cell, - ...(phoneWatched ? { phone: parseCellPhoneMinutes(logs.phone, phoneRequest) } : {}), - director503: await readDirector503Minutes(reader, { projectId: request.projectId, ...director, metrics: [directorMetrics] }), - ledger: parseLedgerSample(logs.ledger), - seatBehindRow: parseSeatBehindRowSample(logs.seatBehindRow, seatRequest) - } - const result = evaluateWatchSample(state, sample, { unverifiableLimit: config.unverifiableLimit }) - state = result.state - log(result.line) - for (const kind of result.pages) { - await raiseAlarm( - { mode: request.mode, cellId: request.cellId, projectId: request.projectId, kind, reason: `sustained for ${PHONE_WATCH_RULES.pageAfterMs / MINUTE_MS} min` }, - { runGcloud: dependencies.runGcloud, log } - ) - } - if (state.state === 'watching' && state.phoneTrip) state = await turnPhonePathOff(config, state, at, { ...deps, log }) - await writeFile(config.stateFile, JSON.stringify(state)) - if (state.state !== 'watching') break - await sleep(Math.max(0, at + SAMPLE_INTERVAL_MS - now())) - } - if (config.githubOutput) await writeOutcome(config.githubOutput, state) - return state -} - -// Writes phonePath=db (the flag tool's CAS write, a plan in dry-run) and keeps watching the -// reserve rules. A write that fails keeps the trip, so the next sample retries it, and pages once -// after PHONE_OFF_PAGE_AFTER_FAILURES in a row: the cell stays watched either way. -export const PHONE_OFF_PAGE_AFTER_FAILURES = 3 -async function turnPhonePathOff(config, state, at, dependencies) { - try { - const result = await flipBack({ ...config.flipValues, flip: 'phonePath' }, dependencies) - dependencies.log(JSON.stringify({ event: 'orca_relay_reserve_watch_phone_path_off', cellId: state.cellId, verdict: state.phoneTrip, written: result.written })) - return { ...state, phoneOffAt: at, phoneTrip: null, phoneOffFailures: 0 } - } catch (error) { - const reason = error instanceof Error ? error.message : String(error) - const failures = (state.phoneOffFailures ?? 0) + 1 - dependencies.log(JSON.stringify({ event: 'orca_relay_reserve_watch_phone_path_off_failed', cellId: state.cellId, failures, reason })) - if (failures === PHONE_OFF_PAGE_AFTER_FAILURES) { - await raiseAlarm( - { - mode: config.request.mode, - cellId: state.cellId, - projectId: config.request.projectId, - kind: 'flip-back-failed', - flip: 'phonePath', - reason: `${state.phoneTrip}; phonePath=db failed ${failures} times: ${reason}` - }, - { runGcloud: dependencies.runGcloud, log: dependencies.log } - ) - } - return { ...state, phoneOffFailures: failures } - } -} - -function writeOutcome(githubOutput, { state, verdict, flip }) { - return appendFile( - githubOutput, - `state=${state}\nverdict=${(verdict ?? '').replace(/[\r\n]+/g, ' ')}\nflip=${flip ?? 'admitMode'}\n` - ) -} - -// The job's outcome. A watch that ended mid-segment (a crash, a timeout) records failed, which the -// workflow flips back in write mode: missing data is never healthy. -export async function recordOutcome(stateFile, githubOutput) { - const state = await loadState(stateFile).catch(() => null) - const outcome = - state && state.state !== 'watching' ? state : { state: 'failed', verdict: 'the watch stopped before a verdict' } - await writeOutcome(githubOutput, outcome) - return outcome -} - -function breakGlassInputs(request, generation) { - return { - mode: 'write', - 'cell-id': request.cellId, - set: 'admitMode=default', - 'break-glass': `${BREAK_GLASS_CONFIRMATION} ${request.cellId}`, - 'expected-generation': generation, - confirmation: `${WRITE_CONFIRMATION} ${request.cellId}` - } -} - -async function readRuntimeRetrying(fetchImpl, request, idToken, sleep, log) { - for (let attempt = 1; ; attempt += 1) { - try { - return await readRuntime(fetchImpl, request, idToken, sleep) - } catch (error) { - if (attempt >= FLIP_BACK_RUNTIME_ATTEMPTS) throw error - log(JSON.stringify({ event: 'orca_relay_reserve_watch_runtime_retry', cellId: request.cellId, attempt, message: error instanceof Error ? error.message : String(error) })) - await sleep(FLIP_BACK_BACKOFF_MS * 2 ** (attempt - 1)) - } - } -} - -/** - * The flag tool's admitMode=db write at the generation read just now. A generation conflict (some - * other write landed first) is re-read and retried while the switch file still says reserve; a - * file that already says db is done. A cell that still does not answer after the runtime retries - * needs break glass, which only an operator may type. - */ -export async function flipBack(values, dependencies) { - const log = dependencies.log ?? console.log - const fetchImpl = dependencies.fetchImpl ?? fetch - const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) - const base = dbFlipRequest(values) - const record = (result) => - log( - JSON.stringify({ - event: 'orca_relay_reserve_watch_flip_back', - mode: base.mode, - cellId: base.cellId, - written: result.written, - generation: result.generation ?? null, - ...(result.alreadyDb ? { alreadyDb: true } : {}) - }) - ) - for (let attempt = 1; ; attempt += 1) { - const current = await readCurrentObject(fetchImpl, base, dependencies.accessToken) - const stillOn = - values.flip === 'phonePath' - ? (current.object?.flags?.phonePath ?? 'db') === 'memory' - : (current.object?.flags?.admitMode ?? 'db') === 'reserve' - if (!stillOn) { - const result = { written: false, alreadyDb: true, generation: current.generation } - record(result) - return result - } - try { - await readRuntimeRetrying(fetchImpl, base, dependencies.idToken, sleep, log) - } catch (error) { - const inputs = breakGlassInputs(base, current.generation) - log(JSON.stringify({ event: 'orca_relay_reserve_watch_break_glass_required', cellId: base.cellId, inputs })) - throw new Error( - `${base.cellId} does not answer runtime-status (${error instanceof Error ? error.message : error}), so ` + - 'the flag tool cannot flip it back. Run Operate Relay Production Cell Flags with break glass: ' + - Object.entries(inputs).map(([key, value]) => `${key}=${value}`).join(', ') - ) - } - try { - const result = await operateCellFlags( - { ...base, expectedGeneration: current.generation }, - { ...dependencies, fetchImpl, log, sleep } - ) - record(result) - return result - } catch (error) { - if (!error?.generationConflict || attempt >= FLIP_BACK_ATTEMPTS) throw error - log(JSON.stringify({ event: 'orca_relay_reserve_watch_flip_back_conflict', cellId: base.cellId, attempt, message: error.message })) - await sleep(FLIP_BACK_BACKOFF_MS * 2 ** (attempt - 1)) - } - } -} - -export const ALARM_EVENT = 'orca_relay_reserve_watch_alarm' -export const ALARM_LOG_NAME = 'orca-relay-reserve-watch' -export const ALARM_KINDS = ['refused', 'flip-back-failed', 'audit-drops', 'renewal-failures'] - -// GitHub's workflow-command escaping, so a verdict cannot end the annotation or start another. -export const escapeWorkflowCommand = (text) => text.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A') -const escapeWorkflowProperty = (text) => escapeWorkflowCommand(text).replace(/:/g, '%3A').replace(/,/g, '%2C') - -/** - * Both kinds can leave the cell in reserve with nothing watching it. Only write mode pages: a - * dry-run watch never flips back, so its refusal or failure changes nothing for the cell. - */ -export async function raiseAlarm({ mode, cellId, projectId, kind, reason, flip }, { runGcloud, log = console.log }) { - if (!ALARM_KINDS.includes(kind)) throw new Error(`--kind must be one of ${ALARM_KINDS.join(', ')}`) - const message = - kind === 'refused' - ? `${cellId} may be in reserve mode and unwatched: the watch was refused (${reason}). Rerun it, or flip the cell to db with Operate Relay Production Cell Flags.` - : kind === 'audit-drops' - ? `${cellId} is losing resume audit rows (${reason}); phones still connect. Check its database writes, or set phonePath=db.` - : kind === 'renewal-failures' - ? `${cellId} cannot renew phone credentials (${reason}); phones still connect but age toward re-pairing. Check its database writes, or set phonePath=db.` - : flip === 'phonePath' - ? `${cellId} may still have phonePath=memory with no watch: turning it off did not land (${reason}). Set phonePath=default by hand with Operate Relay Production Cell Flags.` - : `${cellId} may still be in reserve mode with no watch: the flip back did not land (${reason}). Flip it to db by hand with Operate Relay Production Cell Flags (set=admitMode=db), or with the break-glass inputs logged above if the cell does not answer.` - log(`::error title=${escapeWorkflowProperty(`Reserve watch ${kind} for ${cellId}`)}::${escapeWorkflowCommand(message)}`) - if (mode !== 'write') return { paged: false } - const entry = JSON.stringify({ event: ALARM_EVENT, cellId, kind, reason }) - try { - // The flag tool's project check: a missing --project-id must not reach gcloud as "undefined". - if (!/^[a-z][a-z0-9-]{4,62}$/.test(projectId ?? '')) throw new Error('--project-id is not a GCP project id') - await runGcloud( - ['logging', 'write', ALARM_LOG_NAME, entry, '--payload-type=json', '--severity=ERROR', '--project', projectId], - { timeoutMs: 30_000 } - ) - } catch (error) { - log(`::error title=Reserve watch page failed::${escapeWorkflowCommand(`the alarm log line was not written, so nothing pages: ${error instanceof Error ? error.message : error}`)}`) - return { paged: false } - } - return { paged: true } -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { - const [command, ...rest] = process.argv.slice(2) - const { values } = parseArgs({ - args: rest, - options: Object.fromEntries( - [ - 'mode', - 'cell-id', - 'cell-origin', - 'project-id', - 'director-origin', - 'director-service', - 'confirmation', - 'duration-minutes', - 'segment-minutes', - 'unverifiable-limit', - 'background-503-per-minute', - 'flipped-at', - 'state-file', - 'github-output', - 'verdict', - 'kind', - 'flip' - ].map((name) => [name, { type: 'string' }]) - ) - }) - const idToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN - const runGcloud = (args, { timeoutMs }) => - execFileAsync('gcloud', args, { maxBuffer: 64 * 1024 * 1024, timeout: timeoutMs, killSignal: 'SIGKILL' }) - try { - if (command === 'alarm') { - await raiseAlarm( - { mode: values.mode, cellId: values['cell-id'] ?? '', projectId: values['project-id'], kind: values.kind, reason: values.verdict ?? '', flip: values.flip }, - { runGcloud } - ) - // The alarm step is red, whatever the page did. - process.exitCode = 1 - } else if (command === 'outcome') { - if (!values['state-file'] || !values['github-output']) throw new Error('--state-file and --github-output are required') - console.log(JSON.stringify({ event: 'orca_relay_reserve_watch_outcome', ...(await recordOutcome(values['state-file'], values['github-output'])) })) - } else if (!idToken) { - throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is required') - } else if (command === 'watch') { - const config = await parseWatchSegment(values) - const state = await watchSegment(config, { - idToken, - // For turning the phone path off mid-watch; without it that write fails and the workflow's flip back retries. - accessToken: process.env.ORCA_RELAY_CONTROL_ACCESS_TOKEN, - audit: { - writtenBy: `reserve-watch/${process.env.GITHUB_TRIGGERING_ACTOR ?? process.env.GITHUB_ACTOR ?? 'unknown'}`, - runId: process.env.GITHUB_RUN_ID ?? 'local', - runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? '0' - }, - readRolloutLease: readProductionRolloutLease, - runGcloud - }) - // A cell that was never reserve is the operator's mistake; one flipped back by hand is not. - if (state.state === 'refused') { - console.log(JSON.stringify({ event: 'orca_relay_reserve_watch_refused', verdict: state.verdict })) - process.exitCode = 1 - } - } else if (command === 'flip-back') { - const accessToken = process.env.ORCA_RELAY_CONTROL_ACCESS_TOKEN - if (!accessToken) throw new Error('ORCA_RELAY_CONTROL_ACCESS_TOKEN is required') - if (!values.verdict) throw new Error('--verdict is required') - await flipBack(values, { - accessToken, - idToken, - audit: { - writtenBy: `reserve-watch/${process.env.GITHUB_TRIGGERING_ACTOR ?? process.env.GITHUB_ACTOR ?? 'unknown'}`, - runId: process.env.GITHUB_RUN_ID ?? 'local', - runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? '0' - } - }) - // A trip is a failed watch even once the cell is safely back on db. - console.log(JSON.stringify({ event: 'orca_relay_reserve_watch_verdict', mode: values.mode, verdict: values.verdict })) - process.exitCode = 1 - } else { - throw new Error('command must be watch, outcome, flip-back or alarm') - } - } catch (error) { - console.log( - JSON.stringify({ - event: 'orca_relay_reserve_watch_failed', - message: error instanceof Error ? error.message : String(error) - }) - ) - process.exitCode = 1 - } -} diff --git a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs b/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs deleted file mode 100644 index c0809a80a5a..00000000000 --- a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs +++ /dev/null @@ -1,1680 +0,0 @@ -import assert from 'node:assert/strict' -import { readFileSync } from 'node:fs' -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { test } from 'node:test' -import { - evaluateWatchSample, - initialWatchState, - measuredBackground, - measuredPhoneBackground, - phoneMinuteOver, - phoneShareLine, - PHONE_RULE_NAMES, - PHONE_WATCH_RULES -} from './relay-reserve-watch-rules.mjs' -import { - ALARM_EVENT, - ALARM_LOG_NAME, - FLIP_BACK_ATTEMPTS, - FLIP_BACK_RUNTIME_ATTEMPTS, - flipBack, - parseWatchArguments, - parseWatchSegment, - raiseAlarm, - readCellPhoneMinutes, - readDirector503Minutes, - readSampleLogs, - readSeatBehindRowSample, - sampleLogQueries, - recordOutcome, - rolloutLeaseRefusal, - watchSegment -} from './watch-relay-reserve-cell.mjs' -import { readRelayWorkflow } from './relay-repository.mjs' - -const CELL = 'production-gce-c26' -const START = Date.parse('2026-10-14T12:00:00Z') -const MINUTE = 60_000 -const LIMIT = { unverifiableLimit: 3 } - -const healthyCell = (overrides = {}) => ({ - ok: true, - admitModeConfigured: 'reserve', - admitModeEffective: 'reserve', - draining: false, - controls: 1_000, - enforcedUnits: 1_100, - hardCap: 2_000, - ...overrides -}) -const minutesBefore = (at, count) => - Array.from({ length: count }, (_, index) => - new Date(Math.floor(at / MINUTE) * MINUTE - (count + 3 - index) * MINUTE).toISOString().slice(0, 16) - ) -// Each minute's 503s spread evenly over five director instances, as a fault every instance shares. -const DIRECTORS = ['d1', 'd2', 'd3', 'd4', 'd5'] -const spread = (perMinute) => ({ - ok: true, - perMinute: Object.fromEntries( - DIRECTORS.map((instance) => [instance, Object.fromEntries(Object.entries(perMinute).map(([minute, count]) => [minute, count / DIRECTORS.length]))]) - ), - directors: Object.fromEntries(Object.keys(perMinute).map((minute) => [minute, DIRECTORS.length])) -}) -const quiet503 = (at, perMinute = 10) => { - const minutes = minutesBefore(at, 3) - const counts = Object.fromEntries(minutes.map((minute) => [minute, perMinute])) - return { ok: true, minutes, perMinute: counts, publishedThrough: minutes.at(-1), unmetered: [], instances: spread(counts) } -} -const sampleAt = (at, overrides = {}) => ({ - at, - cell: healthyCell(), - director503: quiet503(at), - ledger: { ok: true, unwrittenRows: 0, directors: 5 }, - seatBehindRow: { ok: true, reports: [] }, - ...overrides -}) -const freshState = (overrides = {}) => ({ - ...initialWatchState({ cellId: CELL, startedAt: START, durationMinutes: 60, background503PerMinute: 10 }), - ...overrides -}) - -// Runs samples one minute apart; returns every result. -function run(samples, state = freshState()) { - const results = [] - for (const sample of samples) { - const result = evaluateWatchSample(state, sample, LIMIT) - results.push(result) - state = result.state - if (state.state !== 'watching') break - } - return results -} - -test('a healthy cell keeps watching and completes at the deadline', () => { - const results = run(Array.from({ length: 61 }, (_, minute) => sampleAt(START + minute * MINUTE))) - assert.equal(results.length, 61) - assert.ok(results.slice(0, 60).every((result) => result.state.state === 'watching')) - assert.equal(results.at(-1).state.state, 'completed') - const line = JSON.parse(results[0].line) - assert.equal(line.event, 'orca_relay_reserve_watch_sample') - assert.deepEqual(Object.keys(line.rules), ['deadMan', 'overrun', 'director503', 'duplicateSeat', 'hostDrop', 'ledgerDrift']) -}) - -test('503s trip only after three complete minutes over max(1.5x background, background + 20)', () => { - // Background 100: the line is 150. Two minutes over and one under is not a trip. - const state = freshState({ background: { medianPerMinute: 100, source: 'input' } }) - const series = (counts) => { - const minutes = minutesBefore(START, 3) - const perMinute = Object.fromEntries(minutes.map((minute, index) => [minute, counts[index]])) - return { ok: true, minutes, perMinute, publishedThrough: minutes.at(-1), unmetered: [], instances: spread(perMinute) } - } - const twoOver = evaluateWatchSample(state, sampleAt(START, { director503: series([151, 140, 200]) }), LIMIT) - assert.equal(twoOver.state.state, 'watching') - const atLine = evaluateWatchSample(state, sampleAt(START, { director503: series([150, 150, 150]) }), LIMIT) - assert.equal(atLine.state.state, 'watching') - const threeOver = evaluateWatchSample(state, sampleAt(START, { director503: series([151, 160, 170]) }), LIMIT) - assert.equal(threeOver.state.state, 'tripped') - assert.match(threeOver.state.verdict, /^flip back production-gce-c26: director503: /) - // A zero background still needs 20 a minute, not one 503. - const zero = freshState({ background: { medianPerMinute: 0, source: 'input' } }) - assert.equal(evaluateWatchSample(zero, sampleAt(START, { director503: series([5, 5, 5]) }), LIMIT).state.state, 'watching') - assert.equal(evaluateWatchSample(zero, sampleAt(START, { director503: series([21, 21, 21]) }), LIMIT).state.state, 'tripped') - // A published minute with no 503 point had none. - const gap = series([151, 160, 170]) - delete gap.perMinute[gap.minutes[1]] - assert.equal(evaluateWatchSample(state, sampleAt(START, { director503: gap }), LIMIT).state.state, 'watching') -}) - -test('a quiet fleet with no 503 series at all reads zero, not unverifiable', () => { - const minutes = minutesBefore(START, 10) - const quiet = { ok: true, minutes, perMinute: {}, publishedThrough: minutes.at(-1), unmetered: [] } - const [result] = run([sampleAt(START, { director503: quiet })], freshState({ background: { medianPerMinute: 0, source: 'measured' } })) - assert.equal(result.rules.director503.status, 'pass') - assert.deepEqual(Object.values(result.rules.director503.recent), [0, 0, 0]) - assert.equal(result.state.unverifiableStreak, 0) - assert.deepEqual(measuredBackground({ perMinute: {}, minutes, publishedThrough: minutes.at(-1), unmetered: [] }), { medianPerMinute: 0, minutes: 10, source: 'measured' }) -}) - -test('503 minutes past the publishing front are left out, and only no front at all is unverifiable', () => { - const state = freshState({ background: { medianPerMinute: 100, source: 'input' } }) - const minutes = minutesBefore(START, 10) - const judge = (director503) => evaluateWatchSample(state, sampleAt(START, { director503 }), LIMIT) - // The front is two minutes behind: the three minutes before it are judged, the two after are not. - const over = Object.fromEntries(minutes.map((minute) => [minute, 500])) - const behind = judge({ ok: true, minutes, perMinute: over, publishedThrough: minutes[7], unmetered: [], instances: spread(over) }) - assert.equal(behind.rules.director503.status, 'trip') - assert.deepEqual(Object.keys(behind.rules.director503.recent), minutes.slice(5, 8)) - // A sparse 503 series: minutes after its last point but within the front are zero, not late. - const sparse = judge({ ok: true, minutes, perMinute: { [minutes[2]]: 500 }, publishedThrough: minutes.at(-1), unmetered: [] }) - assert.equal(sparse.rules.director503.status, 'pass') - assert.equal(sparse.state.unverifiableStreak, 0) - // No request series at all. - const empty = judge({ ok: true, minutes, perMinute: {}, publishedThrough: null, unmetered: [] }) - assert.equal(empty.rules.director503.status, 'unverifiable') - assert.match(empty.rules.director503.reason, /request series is empty/) - assert.equal(empty.state.unverifiableStreak, 1) - // A front too far behind to leave three published minutes in the read. - const stale = judge({ ok: true, minutes, perMinute: {}, publishedThrough: minutes[1], unmetered: [] }) - assert.equal(stale.rules.director503.status, 'unverifiable') - assert.match(stale.rules.director503.reason, new RegExp(`published only through ${minutes[1]}`)) -}) - -// 10-21 07:55Z: one director instance's database calls stalled, carrying the fleet over its line -// alone and flipping four reserve cells back. Background 10: fleet line 30, instance line 6, broad 200. -test('one stalled director instance does not trip the 503 rule; two instances or a broad fault do', () => { - const state = freshState({ background: { medianPerMinute: 10, source: 'input' } }) - const minutes = minutesBefore(START, 3) - const byMinute = (counts) => Object.fromEntries(minutes.map((minute, index) => [minute, counts[index]])) - const read = (fleet, instances) => ({ - ok: true, - minutes, - perMinute: byMinute(fleet), - publishedThrough: minutes.at(-1), - unmetered: [], - instances: instances.ok === false - ? instances - : { ok: true, perMinute: Object.fromEntries(Object.entries(instances).map(([id, counts]) => [id, byMinute(counts)])), directors: byMinute([5, 5, 5]) } - }) - const judge = (director503) => evaluateWatchSample(state, sampleAt(START, { director503 }), LIMIT) - // The stall as logged, with the next minute's re-dials spread over the other four. - const stalled = judge(read([45, 164, 40], { sick: [54, 155, 5], d2: [1, 4, 12], d3: [0, 1, 9], d4: [1, 0, 13], d5: [0, 4, 3] })) - assert.equal(stalled.state.state, 'watching') - assert.equal(stalled.rules.director503.status, 'pass') - assert.equal(stalled.rules.director503.instancesOver, 0) - // Held over its line for all three minutes, it is still one instance. - const held = judge(read([60, 170, 60], { sick: [57, 165, 57], d2: [3, 5, 3] })) - assert.equal(held.state.state, 'watching') - assert.equal(held.rules.director503.instancesOver, 1) - // Two instances each over 6 for the same three minutes. - const two = judge(read([37, 37, 37], { d1: [16, 16, 16], d2: [16, 16, 16], d3: [5, 5, 5] })) - assert.equal(two.state.state, 'tripped') - assert.match(two.state.verdict, /director503: director 503s above 30\/min for 3 min \(37, 37, 37\), 2 instances each above 6\/min/) - // The instance split is needed only between the lines: over 200 trips with no split at all. - const broad = judge(read([250, 300, 260], { ok: false, error: 'director 503 request log read truncated' })) - assert.equal(broad.state.state, 'tripped') - assert.match(broad.state.verdict, /director 503s above 200\/min for 3 min \(250, 300, 260\)$/) - // Between them, no split is no evidence either way. - const blind = judge(read([150, 150, 150], { ok: false, error: 'director 503 request log read truncated' })) - assert.equal(blind.rules.director503.status, 'unverifiable') - assert.match(blind.rules.director503.reason, /truncated/) - assert.equal(blind.state.unverifiableStreak, 1) - // 503s no director logged, over the whole fleet line by themselves, are no one instance's stall. - const platform = judge(read([37, 37, 37], { unattributed: [31, 31, 31], d2: [6, 6, 6] })) - assert.equal(platform.state.state, 'tripped') - assert.match(platform.state.verdict, /93 of them no director instance logged$/) - // A stalled instance's 30 s lines skew against the fleet's minutes; that remainder is not a second instance. - const skewed = judge(read([60, 170, 60], { sick: [52, 162, 52], unattributed: [8, 8, 8], d2: [0, 0, 0] })) - assert.equal(skewed.state.state, 'watching') - assert.equal(skewed.rules.director503.unattributed, 24) - // The share follows the directors that logged: eight make it 30/8, so an even 45 trips. - const eight = read([45, 45, 45], Object.fromEntries(Array.from({ length: 8 }, (_, index) => [`d${index}`, [5.6, 5.6, 5.6]]))) - eight.instances.directors = byMinute([8, 8, 8]) - assert.equal(judge(eight).state.state, 'tripped') - // A rollout's overlap in one judged minute does not shrink the share: the fewest logging count. - const overlap = read([37, 37, 37], { d1: [5, 5, 5], d2: [5, 5, 5] }) - overlap.instances.directors = byMinute([10, 5, 10]) - assert.equal(judge(overlap).state.state, 'watching') - assert.equal(judge(overlap).rules.director503.instanceAbove, 6) - // No director count for a judged minute is no share, not a pass. - const uncounted = read([37, 37, 37], { d1: [16, 16, 16], d2: [16, 16, 16] }) - delete uncounted.instances.directors[minutes[1]] - assert.equal(judge(uncounted).rules.director503.status, 'unverifiable') - assert.match(judge(uncounted).rules.director503.reason, /no director instance count/) - // One director: the share floors at half the line, so it alone never trips between the lines. - const single = read([60, 60, 60], { d1: [60, 60, 60] }) - single.instances.directors = byMinute([1, 1, 1]) - assert.equal(judge(single).state.state, 'watching') - assert.equal(judge(single).rules.director503.instanceAbove, 15) - // Under the fleet line the split is never read. - const calm = judge(read([20, 20, 20], { ok: false, error: 'director 503 request log read failed' })) - assert.equal(calm.rules.director503.status, 'pass') -}) - -test('per-instance 503s are each director\'s own logged 503s less its re-dials, and the rest is unattributed', async () => { - // Fleet 30 a minute. a logged 20 (10 its own re-dials), b logged 6: 10 nobody accounts for. - const lineOf = (count, own) => ({ - placementRejectionsByReasonDelta: { 'host-rate-limited': own }, - assign503sByCauseDelta: { 'placement-lane': count } - }) - const sources = fakeSources({ per503: () => 30, metrics: () => [lineOf(10, 5), lineOf(3, 0)] }) - const read = await readDirector503Minutes( - { ...fakeReader(), fetch: sources.fetchImpl, runGcloud: sources.runGcloud }, - { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt: START, minutes: 3 } - ) - assert.equal(read.ok, true) - assert.deepEqual(Object.values(read.perMinute), [20, 20, 20]) - assert.deepEqual(Object.keys(read.instances.perMinute).sort(), ['a', 'b', 'unattributed']) - assert.deepEqual(Object.values(read.instances.perMinute.a), [10, 10, 10]) - assert.deepEqual(Object.values(read.instances.perMinute.b), [6, 6, 6]) - assert.deepEqual(Object.values(read.instances.perMinute.unattributed), [4, 4, 4]) - assert.deepEqual(Object.values(read.instances.directors), [2, 2, 2]) -}) - -test('the recorded 10-21 07:55Z director stall reads as one instance and does not trip', async () => { - const recorded = JSON.parse(readFileSync(new URL('../fixtures/relay-reserve-watch/2026-10-21-director-stall.json', import.meta.url), 'utf8')) - const series = (code, perMinute) => ({ - metric: { labels: { response_code: code } }, - points: Object.entries(perMinute).map(([minute, count]) => ({ - interval: { endTime: new Date(Date.parse(`${minute}Z`) + MINUTE).toISOString() }, - value: { int64Value: String(count) } - })) - }) - const ok = Object.fromEntries(Object.keys(recorded.fleet503PerMinute).map((minute) => [minute, 1_000])) - const lines = recorded.directorMetricsLines.map(({ timestamp, instance, ...deltas }) => ({ - timestamp, - labels: { instanceId: instance }, - resource: { type: 'cloud_run_revision' }, - jsonPayload: { event: METRICS_EVENT, ...deltas } - })) - const read = await readDirector503Minutes( - { - ...fakeReader(), - fetch: async () => new Response(JSON.stringify({ timeSeries: [series('200', ok), series('503', recorded.fleet503PerMinute)] })), - runGcloud: async (args) => { - if (args[0] === 'auth') return { stdout: 'token' } - const [, from, to] = args[2].match(/timestamp>="([^"]+)" AND timestamp<"([^"]+)"/) - return { stdout: JSON.stringify(lines.filter(({ timestamp }) => timestamp >= from && timestamp < to)) } - } - }, - { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt: Date.parse('2026-10-21T07:58:00Z'), minutes: 10 } - ) - const judged = evaluateWatchSample( - freshState({ background: { medianPerMinute: 10, source: 'input' } }), - sampleAt(START, { director503: read }), - LIMIT - ) - // The fleet alone was over 30 all three minutes, which is what tripped all four watches. - assert.deepEqual(Object.values(judged.rules.director503.recent), [45, 164, 40]) - assert.equal(judged.rules.director503.status, 'pass') - // Not even the stalled instance held its share of 6 a minute: it fell to 5 in the third minute. - assert.equal(judged.rules.director503.instancesOver, 0) - assert.deepEqual(['07:55', '07:56', '07:57'].map((minute) => read.instances.perMinute.d5[`2026-10-21T${minute}`]), [40, 169, 5]) -}) - -const SEAT_EVENT = 'orca_relay_reserve_ledger_seat_behind_row' -const SUMMARY_EVENT = 'orca_relay_reserve_ledger_summary' -const summaryLine = (at, instanceId = 'a') => ({ - timestamp: new Date(at).toISOString(), - labels: { instanceId }, - jsonPayload: { event: SUMMARY_EVENT, pending: 0, dropped: 0 } -}) - -// A Cloud Monitoring or Cloud Logging reader over fakes, as the watch builds one. -const fakeReader = ({ fetch: fetchImpl = async () => new Response('{}'), runGcloud = async () => ({ stdout: 'token' }) } = {}) => ({ - fetch: fetchImpl, - runGcloud, - retryDelayMs: 0, - readTimeoutMs: 1_000, - now: () => START, - deadlineAt: START + 10_000, - projectId: 'onorca-cloud' -}) - -test('an unparseable timestamp makes the read unverifiable', async () => { - const badPoint = { - timeSeries: [{ points: [{ interval: { endTime: 'not a time' }, value: { int64Value: '3' } }] }] - } - const read503 = await readDirector503Minutes( - fakeReader({ fetch: async () => new Response(JSON.stringify(badPoint)) }), - { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt: START, minutes: 3 } - ) - assert.equal(read503.ok, false) - assert.match(read503.error, /unreadable/) - const seats = await readSeatBehindRowSample( - fakeReader({ runGcloud: async () => ({ stdout: JSON.stringify([summaryLine(START), { timestamp: 'garbage', jsonPayload: { epoch: 1, rowEpoch: 2 } }]) }) }), - { directorService: 'orca-cloud-relay', cellId: CELL, at: START } - ) - assert.equal(seats.ok, false) - assert.match(seats.error, /unparseable timestamp/) -}) - -test('a seat-behind-row key carries the director instance and cell beside the epochs', async () => { - const entry = (instanceId) => ({ timestamp: new Date(START).toISOString(), labels: { instanceId }, jsonPayload: { event: SEAT_EVENT, epoch: 7, rowEpoch: 8 } }) - const read = await readSeatBehindRowSample( - fakeReader({ runGcloud: async () => ({ stdout: JSON.stringify([entry('a'), summaryLine(START), entry('b')]) }) }), - { directorService: 'orca-cloud-relay', cellId: CELL, at: START } - ) - assert.deepEqual(read.reports.map((report) => report.key), [`a/${CELL}/7:8`, `b/${CELL}/7:8`]) -}) - -test('the duplicate read carries a positive control: no fresh ledger summary in it is unverifiable, not zero', async () => { - const filters = [] - const readWith = (entries) => - readSeatBehindRowSample( - fakeReader({ runGcloud: async (args) => (filters.push(args[2]), { stdout: JSON.stringify(entries) }) }), - { directorService: 'orca-cloud-relay', cellId: CELL, at: START } - ) - // One read, one window: the control rides the same filter as the seat lines. - assert.deepEqual(await readWith([summaryLine(START - 30_000)]), { ok: true, reports: [] }) - assert.match(filters[0], new RegExp(`\\(\\(jsonPayload\\.event="${SEAT_EVENT}" AND jsonPayload\\.cellId="${CELL}"\\) OR jsonPayload\\.event="${SUMMARY_EVENT}"\\)`)) - assert.match(filters[0], /resource\.labels\.service_name="orca-cloud-relay"/) - for (const entries of [[], [summaryLine(START - 10 * MINUTE)]]) { - const blind = await readWith(entries) - assert.equal(blind.ok, false) - assert.match(blind.error, /no fresh ledger summary \(positive control\)/) - } - // Blind is a gap in the sample, which counts toward the unverifiable streak. - const [gap] = run([sampleAt(START, { seatBehindRow: await readWith([]) })]) - assert.equal(gap.rules.duplicateSeat.status, 'unverifiable') - assert.equal(gap.state.unverifiableStreak, 1) -}) - -test('the measured background is the median of the metered minutes before the flip, published gaps as zero', () => { - const minutes = ['2026-10-14T11:47', '2026-10-14T11:48', '2026-10-14T11:49'] - const front = minutes[2] - const background = (perMinute, publishedThrough, readMinutes = minutes, unmetered = []) => - measuredBackground({ perMinute, minutes: readMinutes, publishedThrough, unmetered }) - assert.equal(background({ '2026-10-14T11:48': 30 }, front).medianPerMinute, 0) - assert.equal(background({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, front).medianPerMinute, 20) - assert.equal(background({}, '2026-10-14T11:52').medianPerMinute, 0) - // Not published through the last minute yet, or no request series: no background yet. - assert.equal(background({ [minutes[0]]: 10 }, minutes[1]), null) - assert.equal(background({}, null), null) - assert.equal(background({}, front, []), null) - // An unmetered minute is left out, not read as zero; under half metered is no background. - assert.deepEqual(background({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, front, minutes, [minutes[0]]), { medianPerMinute: 25, minutes: 2, source: 'measured' }) - assert.equal(background({}, front, minutes, minutes.slice(1)), null) -}) - -test('any overrun trips at once', () => { - const [result] = run([sampleAt(START, { cell: healthyCell({ enforcedUnits: 2_001 }) })]) - assert.equal(result.state.state, 'tripped') - assert.match(result.state.verdict, /overrun: ledger units 2001 over hard cap 2000/) - const [atCap] = run([sampleAt(START, { cell: healthyCell({ enforcedUnits: 2_000 }) })]) - assert.equal(atCap.state.state, 'watching') -}) - -test('a seat reported behind its row again past the grace trips; inside the grace it does not', () => { - const report = (at) => ({ at, key: '7:8' }) - const within = run([ - sampleAt(START, { seatBehindRow: { ok: true, reports: [report(START - 30_000)] } }), - sampleAt(START + MINUTE, { seatBehindRow: { ok: true, reports: [report(START - 30_000), report(START + 50_000)] } }) - ]) - assert.equal(within.at(-1).state.state, 'watching') - // Carried across samples: the second read no longer holds the first report. - const past = run([ - sampleAt(START, { seatBehindRow: { ok: true, reports: [report(START - 30_000)] } }), - sampleAt(START + 2 * MINUTE, { seatBehindRow: { ok: true, reports: [report(START + 70_000)] } }) - ]) - assert.equal(past.at(-1).state.state, 'tripped') - assert.match(past.at(-1).state.verdict, /duplicateSeat: a seat behind its ledger row \(7:8\)/) - // Another seat's report is not the same seat. - const other = run([ - sampleAt(START, { seatBehindRow: { ok: true, reports: [report(START - 30_000)] } }), - sampleAt(START + 2 * MINUTE, { seatBehindRow: { ok: true, reports: [{ at: START + 70_000, key: '3:4' }] } }) - ]) - assert.equal(other.at(-1).state.state, 'watching') -}) - -test('a host drop over 5% in 5 minutes trips outside a drain only', () => { - const controls = [1_000, 990, 980, 960, 949] - const samples = (draining) => - controls.map((count, minute) => - sampleAt(START + minute * MINUTE, { cell: healthyCell({ controls: count, draining: draining && minute === 2 }) }) - ) - assert.equal(run(samples(false)).at(-1).state.state, 'tripped') - assert.match(run(samples(false)).at(-1).state.verdict, /hostDrop: connected hosts fell 1000 -> 949/) - assert.equal(run(samples(true)).at(-1).state.state, 'watching') - // A drop spread over more than 5 minutes is not one. - const slow = [1_000, 990, 980, 970, 960, 955, 950].map((count, minute) => - sampleAt(START + minute * MINUTE, { cell: healthyCell({ controls: count }) }) - ) - assert.equal(run(slow).at(-1).state.state, 'watching') - // Under the floor on a nearly empty cell: 100 -> 92 is 8%, but only 8 hosts. - const small = [100, 92].map((count, minute) => sampleAt(START + minute * MINUTE, { cell: healthyCell({ controls: count }) })) - assert.equal(run(small).at(-1).state.state, 'watching') -}) - -test('ledger drift over 0.5% trips once it has lasted 2 minutes', () => { - const drift = (rows) => ({ ok: true, unwrittenRows: rows, directors: 5 }) - const brief = run([6, 6, 0, 6].map((rows, minute) => sampleAt(START + minute * MINUTE, { ledger: drift(rows) }))) - assert.equal(brief.at(-1).state.state, 'watching') - const sustained = run([6, 6, 6].map((rows, minute) => sampleAt(START + minute * MINUTE, { ledger: drift(rows) }))) - assert.equal(sustained.length, 3) - assert.equal(sustained.at(-1).state.state, 'tripped') - assert.match(sustained.at(-1).state.verdict, /ledgerDrift: ledger drift 0\.60% for 120 s/) - const atLine = run([5, 5, 5, 5].map((rows, minute) => sampleAt(START + minute * MINUTE, { ledger: drift(rows) }))) - assert.equal(atLine.at(-1).state.state, 'watching') -}) - -test('a tripped dead-man is a trip: Postgres still records reserve until a db write', () => { - const [result] = run([sampleAt(START, { cell: healthyCell({ admitModeEffective: 'db' }) })]) - assert.equal(result.state.state, 'tripped') - assert.match(result.state.verdict, /deadMan: the cell tripped its dead-man/) -}) - -test('unreadable data is never healthy: N samples in a row with any gap trip', () => { - const unreadable = { ok: false, error: 'cell runtime status returned 502' } - const gaps = [ - { cell: unreadable }, - { ledger: { ok: false, error: 'no placing director logged a ledger summary' } }, - { director503: { ok: false, error: 'director 503 metric read failed', perMinute: {}, minutes: [], publishedThrough: null } } - ] - const results = run(gaps.map((gap, minute) => sampleAt(START + minute * MINUTE, gap))) - assert.deepEqual(results.map((result) => result.state.unverifiableStreak), [1, 2, 3]) - assert.equal(results.at(-1).state.state, 'tripped') - assert.match(results.at(-1).state.verdict, /unverifiable: 3 samples in a row could not read director503/) - // One whole sample resets the streak. - const reset = run( - [gaps[0], {}, gaps[0], gaps[0]].map((gap, minute) => sampleAt(START + minute * MINUTE, gap)) - ) - assert.deepEqual(reset.map((result) => result.state.unverifiableStreak), [1, 0, 1, 2]) - assert.equal(reset.at(-1).state.state, 'watching') - // No background yet is a gap too. - const noBackground = freshState({ background: null }) - assert.equal(evaluateWatchSample(noBackground, sampleAt(START), LIMIT).state.unverifiableStreak, 1) - // A measured trip still wins over a gap elsewhere. - const [both] = run([sampleAt(START, { cell: healthyCell({ enforcedUnits: 2_500 }), ledger: gaps[1].ledger })]) - assert.equal(both.state.state, 'tripped') - assert.match(both.state.verdict, /overrun/) -}) - -test('a cell no longer in reserve stops the watch without a write; one never in reserve is refused', () => { - const dbCell = healthyCell({ admitModeConfigured: 'db', admitModeEffective: 'reserve' }) - const later = run([sampleAt(START), sampleAt(START + MINUTE, { cell: dbCell })]) - assert.equal(later.at(-1).state.state, 'stopped') - assert.match(later.at(-1).state.verdict, /is not in reserve mode \(switch file says db\)/) - // Never reserve within the wait: refused, which the workflow does not flip back. - const never = run(Array.from({ length: 12 }, (_, minute) => sampleAt(START + minute * MINUTE, { cell: dbCell }))) - assert.equal(never.at(-1).state.state, 'refused') - assert.equal(never.length, 11) - assert.match(never.at(-1).state.verdict, /did not report reserve mode within 10 min \(switch file says db\)/) -}) - -test('a watch dispatched before the reserve write applied waits for it, then judges as usual', () => { - const dbCell = healthyCell({ admitModeConfigured: 'db', admitModeEffective: 'db' }) - const results = run([ - sampleAt(START, { cell: dbCell, ledger: { ok: false, error: 'not yet' } }), - sampleAt(START + MINUTE, { cell: dbCell }), - sampleAt(START + 2 * MINUTE), - sampleAt(START + 3 * MINUTE, { cell: healthyCell({ enforcedUnits: 2_500 }) }) - ]) - // Waiting samples judge nothing: not a dead-man trip on a db cell, not a gap. - assert.deepEqual(results.slice(0, 2).map((result) => result.state.state), ['watching', 'watching']) - assert.equal(results[0].rules.reserve.status, 'pending') - assert.deepEqual(Object.keys(results[0].rules), ['reserve']) - assert.deepEqual(results.map((result) => result.state.unverifiableStreak), [0, 0, 0, 0]) - assert.equal(results[2].state.reserveSeenAt, START + 2 * MINUTE) - assert.equal(results[2].rules.overrun.status, 'pass') - assert.equal(results[3].state.state, 'tripped') - // Once seen in reserve, a later db reads as a hand flip back, not a refusal. - const after = run([sampleAt(START + 2 * MINUTE), sampleAt(START + 3 * MINUTE, { cell: dbCell })], results[0].state) - assert.equal(after.at(-1).state.state, 'stopped') -}) - -const watchValues = (overrides = {}) => ({ - mode: 'dry-run', - 'cell-id': CELL, - 'cell-origin': 'https://c26.relay.onorca.dev', - 'project-id': 'onorca-cloud', - 'director-origin': 'https://relay.onorca.dev', - 'director-service': 'orca-cloud-relay', - 'duration-minutes': '120', - 'segment-minutes': '55', - 'unverifiable-limit': '3', - 'background-503-per-minute': '', - 'flipped-at': new Date(START - 5 * MINUTE).toISOString(), - 'state-file': '/unused', - ...overrides -}) -const parseAt = (values) => parseWatchArguments(values, START) - -test('watch arguments reuse the flag tool checks, so a write watch needs its confirmation up front', () => { - assert.equal(parseAt(watchValues()).background503PerMinute, null) - assert.equal(parseAt(watchValues()).flippedAt, START - 5 * MINUTE) - assert.equal(parseAt(watchValues({ 'background-503-per-minute': '12.5', 'flipped-at': '' })).background503PerMinute, 12.5) - assert.throws(() => parseAt(watchValues({ mode: 'write' })), /WRITE_RELAY_CELL_FLAGS production-gce-c26/) - assert.doesNotThrow(() => parseAt(watchValues({ mode: 'write', confirmation: `WRITE_RELAY_CELL_FLAGS ${CELL}` }))) - assert.throws(() => parseAt(watchValues({ 'cell-origin': 'https://c27.relay.onorca.dev' })), /names another cell/) - assert.throws(() => parseAt(watchValues({ 'duration-minutes': '301' })), /duration-minutes/) - assert.throws(() => parseAt(watchValues({ 'segment-minutes': '56' })), /segment-minutes/) - assert.throws(() => parseAt(watchValues({ 'background-503-per-minute': '-1' })), /background/) -}) - -test('the 503 background needs flipped-at or an explicit value; flipped-at must be a recent zoned time', () => { - assert.throws(() => parseAt(watchValues({ 'flipped-at': '' })), /--flipped-at .* or --background-503-per-minute is required/) - assert.throws(() => parseAt(watchValues({ 'flipped-at': '2026-10-14T11:55:00' })), /ISO time with a zone/) - assert.throws(() => parseAt(watchValues({ 'flipped-at': '2026-10-14T12:01:00Z' })), /in the future/) - assert.throws(() => parseAt(watchValues({ 'flipped-at': '2026-10-13T11:00:00Z' })), /more than 24 hours/) - assert.equal(parseAt(watchValues({ 'flipped-at': '2026-10-14T07:55:00-04:00' })).flippedAt, START - 5 * MINUTE) -}) - -// A cell, Cloud Monitoring and Cloud Logging, answering from `script` per sample. -const METRICS_EVENT = 'orca_relay_runtime_metrics' -// Each instance's line every 30 s inside the read's timestamp bounds, newest first. -function metricsLines(filter, metrics) { - const [, from, to] = filter.match(/timestamp>="([^"]+)" AND timestamp<"([^"]+)"/) - const lines = [] - for (let at = Math.ceil(Date.parse(from) / 30_000) * 30_000; at < Date.parse(to); at += 30_000) { - metrics(at).forEach((deltas, index) => { - lines.push({ timestamp: new Date(at).toISOString(), labels: { instanceId: FAKE_DIRECTORS[index] }, jsonPayload: deltas }) - }) - } - return lines.reverse() -} - -// The `metrics` lines' instances, in order. -const FAKE_DIRECTORS = ['a', 'b', 'c', 'd', 'e'] - -function fakeSources({ - cell = () => ({}), - // Logged at each sample's minute, so it stays fresh for the positive control. - ledger = (sample) => [{ timestamp: new Date(START + (sample - 1) * MINUTE).toISOString(), labels: { instanceId: 'a' }, jsonPayload: { pending: 0, dropped: 0 } }], - per503 = () => 10, - // Requests of every other code per minute; zero is a minute with no traffic at all. - perOther = () => 1_000, - seatControl = true, - rolloutLease = null, - // Each director instance's runtime-metrics deltas for the 30 s ending at `endedAt`. - metrics = () => [{}], - // The watched cell's own runtime-metrics deltas, when it runs the phone path. - phone = null -} = {}) { - let samples = 0 - const monitoringSpans = [] - const fetchImpl = async (url) => { - const text = String(url) - if (text.endsWith('/v1/admin/runtime-status')) { - samples += 1 - const overrides = cell(samples) - return new Response( - JSON.stringify({ - role: 'cell', - cellId: CELL, - draining: false, - flagsApplied: { generation: 4, flags: { admitMode: 'reserve', readinessLocal: true } }, - admitModeEffective: 'reserve', - connectionCapacity: { hardCap: 2_000 }, - runtime: { controls: 1_000, enforcedConnectionUnits: 1_100 }, - ...overrides - }) - ) - } - if (text.startsWith('https://monitoring.googleapis.com/')) { - const params = new URL(text).searchParams - const span = { startedAt: Date.parse(params.get('interval.startTime')), endedAt: Date.parse(params.get('interval.endTime')) } - monitoringSpans.push(span) - assert.equal(params.get('aggregation.groupByFields'), 'metric.label."response_code"') - assert.doesNotMatch(params.get('filter'), /response_code/) - // Cloud Monitoring returns no point for a minute with no requests of that code. - const seriesOf = (code, count) => { - const points = [] - for (let minute = span.startedAt; minute < span.endedAt; minute += MINUTE) { - if (count(minute) > 0) { - points.push({ interval: { endTime: new Date(minute + MINUTE).toISOString() }, value: { int64Value: String(count(minute)) } }) - } - } - return points.length === 0 ? [] : [{ metric: { labels: { response_code: code } }, points }] - } - return new Response(JSON.stringify({ timeSeries: [...seriesOf('200', perOther), ...seriesOf('503', per503)] })) - } - throw new Error(`unexpected fetch ${text}`) - } - const director = (entries) => entries.map((entry) => ({ ...entry, resource: { type: 'cloud_run_revision' } })) - // One query's lines, as Cloud Logging would answer its filter alone. - const answer = (filter) => { - const summaries = director(ledger(samples).map((entry) => ({ ...entry, jsonPayload: { ...entry.jsonPayload, event: SUMMARY_EVENT } }))) - if (filter.includes(SEAT_EVENT)) return seatControl ? summaries : [] - if (filter.includes(SUMMARY_EVENT)) return summaries - if (filter.includes('gce_instance')) { - return phone ? metricsLines(filter, (at) => [{ ...phone(at), event: METRICS_EVENT, cellId: CELL }]).map((entry) => ({ ...entry, resource: { type: 'gce_instance' } })) : [] - } - if (filter.includes(METRICS_EVENT)) return director(metricsLines(filter, (at) => metrics(at).map((deltas) => ({ ...deltas, event: METRICS_EVENT })))) - return [] - } - const logReads = [] - const runGcloud = async (args) => { - if (args[0] === 'auth') return { stdout: 'token' } - const filter = args[2] - logReads.push({ sample: samples, filter }) - // A sample's combined read is `(A) OR (B) ...`; each part answers as it would alone, deduplicated. - const parts = filter.startsWith('(') ? filter.slice(1, -1).split(') OR (') : [filter] - const seen = new Set() - const entries = parts.flatMap(answer).filter((entry) => { - const key = JSON.stringify(entry) - return !seen.has(key) && seen.add(key) - }) - return { stdout: JSON.stringify(entries) } - } - return { fetchImpl, runGcloud, monitoringSpans, logReads, readRolloutLease: async () => rolloutLease } -} - -test('a segment samples once a minute, carries state to the next segment, and stops on a trip', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - let clock = START - const lines = [] - const config = parseAt( - watchValues({ 'state-file': join(directory, 'state.json'), 'github-output': join(directory, 'out'), 'segment-minutes': '3' }) - ) - const sources = fakeSources({ cell: (sample) => (sample === 5 ? { runtime: { controls: 1_000, enforcedConnectionUnits: 2_001 } } : {}) }) - const dependencies = { - ...sources, - idToken: 'id', - retryDelayMs: 0, - now: () => clock, - sleep: async (ms) => { clock += ms }, - log: (line) => lines.push(JSON.parse(line)) - } - const first = await watchSegment(config, dependencies) - assert.equal(first.state, 'watching') - assert.equal(lines.length, 3) - assert.equal(first.background.source, 'measured') - assert.equal(await readFile(join(directory, 'out'), 'utf8'), 'state=watching\nverdict=\nflip=admitMode\n') - const second = await watchSegment(config, dependencies) - assert.equal(second.state, 'tripped') - assert.equal(second.samples, 5) - assert.deepEqual(lines.map((line) => line.sample), [1, 2, 3, 4, 5]) - assert.equal(lines.at(-1).rules.overrun.status, 'trip') - assert.match(await readFile(join(directory, 'out'), 'utf8'), /state=tripped\nverdict=flip back production-gce-c26: overrun: /) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -test('a sample that cannot read its sources counts as unverifiable rather than crashing the watch', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - let clock = START - const config = parseAt(watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10' })) - const sources = fakeSources({ ledger: () => [] }) - const state = await watchSegment(config, { - ...sources, - idToken: 'id', - retryDelayMs: 0, - now: () => clock, - sleep: async (ms) => { clock += ms }, - log: () => {} - }) - assert.equal(state.state, 'tripped') - assert.equal(state.samples, 3) - // No summary also blinds the duplicate rule, whose positive control it is. - assert.match(state.verdict, /unverifiable: 3 samples in a row could not read duplicateSeat, ledgerDrift/) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -test('a watch that ended without a verdict records failed, which the workflow flips back', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - const output = join(directory, 'out') - assert.equal((await recordOutcome(join(directory, 'missing.json'), output)).state, 'failed') - await writeFile(join(directory, 'watching.json'), JSON.stringify(freshState())) - assert.equal((await recordOutcome(join(directory, 'watching.json'), output)).state, 'failed') - await writeFile(join(directory, 'tripped.json'), JSON.stringify(freshState({ state: 'tripped', verdict: 'flip back x:\ny' }))) - assert.equal((await recordOutcome(join(directory, 'tripped.json'), output)).state, 'tripped') - assert.match(await readFile(output, 'utf8'), /state=tripped\nverdict=flip back x: y\nflip=admitMode\n$/) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -// Google Storage and the director, as far as the flag tool's flip back reaches them. `onUpload` -// returns 'conflict' to play another writer landing first, optionally with its own admitMode. -function fakeFlipBack({ - generation = '7', - onUpload = () => 'ok', - cellAnswers = true, - flags = { admitMode: 'reserve', readinessLocal: true }, - supportedFlags = { - admitMode: { type: 'enum', values: ['db', 'reserve'] }, - readinessLocal: { type: 'boolean' }, - phonePath: { type: 'enum', values: ['db', 'memory'] } - } -} = {}) { - const calls = [] - let uploads = 0 - let object = { generation: Number(generation), flags } - let applied = object - const fetchImpl = async (url, init = {}) => { - const text = String(url) - calls.push({ url: text, method: init.method ?? 'GET', body: init.body }) - if (text.includes('/upload/storage/v1/')) { - uploads += 1 - assert.match(text, new RegExp(`ifGenerationMatch=${object.generation}$`)) - const outcome = onUpload(uploads) - if (outcome !== 'ok') { - object = { generation: object.generation + 1, flags: { ...object.flags, admitMode: outcome.admitMode ?? 'reserve' } } - applied = object - return new Response('', { status: 412 }) - } - object = { generation: object.generation + 1, flags: JSON.parse(init.body.split('\r\n')[7]).flags } - applied = object - return new Response(JSON.stringify({ generation: String(object.generation) })) - } - if (text.includes('?alt=media')) { - return new Response(JSON.stringify({ v: 1, cellId: CELL, flags: object.flags })) - } - if (text.includes('/storage/v1/b/')) return new Response(JSON.stringify({ generation: String(object.generation) })) - if (text.endsWith('/v1/admin/runtime-status')) { - if (!cellAnswers) return new Response('', { status: 502 }) - return new Response( - JSON.stringify({ - role: 'cell', - cellId: CELL, - flagsApplied: applied, - supportedFlags, - admitModeEffective: applied.flags.admitMode, - runtime: { controls: 10 } - }) - ) - } - if (text.endsWith('/v1/admin/cell-admit-mode')) { - const admitMode = JSON.parse(init.body).admitMode - return new Response(JSON.stringify({ v: 1, cellId: CELL, admitMode: admitMode ?? 'reserve', heartbeatFresh: true })) - } - throw new Error(`unexpected fetch ${text}`) - } - return { fetchImpl, calls, uploads: () => uploads } -} - -const flipDependencies = (fetchImpl, lines, sleeps = []) => { - let clock = START - return { - fetchImpl, - accessToken: 'access', - idToken: 'id', - audit: { writtenBy: 'reserve-watch/test', runId: '1', runAttempt: '1' }, - now: () => clock, - sleep: async (ms) => { sleeps.push(ms); clock += ms }, - log: (line) => lines.push(JSON.parse(line)) - } -} - -const flipValues = (overrides = {}) => ({ - mode: 'write', - 'cell-id': CELL, - 'cell-origin': 'https://c26.relay.onorca.dev', - 'project-id': 'onorca-cloud', - 'director-origin': 'https://relay.onorca.dev', - confirmation: `WRITE_RELAY_CELL_FLAGS ${CELL}`, - ...overrides -}) - -test('the flip back is the flag tool write: admitMode=db at the generation just read, then Postgres db', async () => { - const { fetchImpl, calls } = fakeFlipBack() - const lines = [] - const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines)) - assert.equal(result.written, true) - const upload = calls.find((call) => call.url.includes('/upload/storage/v1/')) - assert.match(upload.body, /"flags":\{"admitMode":"db","readinessLocal":true\}/) - assert.ok(calls.some((call) => call.url.endsWith('/v1/admin/cell-admit-mode') && JSON.parse(call.body).admitMode === 'db')) - assert.deepEqual(lines.at(-1), { - event: 'orca_relay_reserve_watch_flip_back', - mode: 'write', - cellId: CELL, - written: true, - generation: '8' - }) -}) - -test('a dry-run flip back plans the write and writes nothing', async () => { - const { fetchImpl, calls } = fakeFlipBack() - const lines = [] - const result = await flipBack(flipValues({ mode: 'dry-run', confirmation: '' }), { - fetchImpl, - accessToken: 'access', - idToken: 'id', - audit: {}, - sleep: async () => {}, - log: (line) => lines.push(JSON.parse(line)) - }) - assert.equal(result.written, false) - assert.ok(!calls.some((call) => call.url.includes('/upload/'))) - assert.ok(!calls.some((call) => call.url.endsWith('/v1/admin/cell-admit-mode') && JSON.parse(call.body).admitMode)) - assert.equal(lines.find((line) => line.event === 'orca_relay_cell_flags_plan').desired.flags.admitMode, 'db') -}) - -test('a write flip back without the typed confirmation is refused before any read', async () => { - const { fetchImpl, calls } = fakeFlipBack() - await assert.rejects( - flipBack(flipValues({ confirmation: '' }), { fetchImpl, accessToken: 'a', idToken: 'i', audit: {}, log: () => {} }), - /WRITE_RELAY_CELL_FLAGS production-gce-c26/ - ) - assert.equal(calls.length, 0) -}) - -test('a generation conflict is re-read and retried while the switch file still says reserve', async () => { - const { fetchImpl, uploads } = fakeFlipBack({ onUpload: (n) => (n < 3 ? { admitMode: 'reserve' } : 'ok') }) - const lines = [] - const sleeps = [] - const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines, sleeps)) - assert.equal(result.written, true) - assert.equal(uploads(), 3) - assert.deepEqual(lines.filter((line) => line.event === 'orca_relay_reserve_watch_flip_back_conflict').map((line) => line.attempt), [1, 2]) - assert.deepEqual(sleeps.slice(0, 1), [2_000]) - assert.equal(lines.at(-1).generation, '10') -}) - -test('a conflict whose writer already flipped the cell to db ends the flip back without a write', async () => { - const { fetchImpl, uploads } = fakeFlipBack({ onUpload: () => ({ admitMode: 'db' }) }) - const lines = [] - const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines)) - assert.equal(uploads(), 1) - assert.equal(result.written, false) - assert.equal(result.alreadyDb, true) - assert.deepEqual(lines.at(-1), { event: 'orca_relay_reserve_watch_flip_back', mode: 'write', cellId: CELL, written: false, generation: '8', alreadyDb: true }) -}) - -test('conflicts past the retry bound fail the flip back', async () => { - const { fetchImpl, uploads } = fakeFlipBack({ onUpload: () => ({ admitMode: 'reserve' }) }) - await assert.rejects(flipBack(flipValues(), flipDependencies(fetchImpl, [])), /object changed since expected-generation/) - assert.equal(uploads(), FLIP_BACK_ATTEMPTS) -}) - -test('an unreachable cell fails with the exact break-glass inputs and writes nothing', async () => { - const { fetchImpl, uploads } = fakeFlipBack({ cellAnswers: false }) - const lines = [] - await assert.rejects( - flipBack(flipValues(), flipDependencies(fetchImpl, lines)), - (error) => { - assert.match(error.message, /production-gce-c26 does not answer runtime-status \(cell runtime status returned 502\)/) - assert.match(error.message, /set=admitMode=default, break-glass=BREAK GLASS production-gce-c26, expected-generation=7/) - return true - } - ) - assert.equal(uploads(), 0) - assert.deepEqual(lines.at(-1).inputs, { - mode: 'write', - 'cell-id': CELL, - set: 'admitMode=default', - 'break-glass': `BREAK GLASS ${CELL}`, - 'expected-generation': '7', - confirmation: `WRITE_RELAY_CELL_FLAGS ${CELL}` - }) -}) - -test('the 503 background is the 10 minutes before flipped-at, read once they are published', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - let clock = START - // Flipped one minute ago: the minutes before it are not published until 3 minutes after it. - const flippedAt = START - MINUTE + 20_000 - const config = parseAt( - watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '3', 'flipped-at': new Date(flippedAt).toISOString() }) - ) - const sources = fakeSources({ per503: (minute) => (minute < START - MINUTE ? 4 : 40) }) - const lines = [] - const state = await watchSegment(config, { - ...sources, - idToken: 'id', - retryDelayMs: 0, - now: () => clock, - sleep: async (ms) => { clock += ms }, - log: (line) => lines.push(JSON.parse(line)) - }) - // Pending until then, and pending is not a gap: the streak stays at zero. - assert.deepEqual(lines.map((line) => line.rules.director503.status), ['pending', 'pending', 'pass']) - assert.match(lines[0].rules.director503.reason, /background before the flip is not published yet/) - assert.deepEqual(lines.map((line) => line.unverifiableStreak), [0, 0, 0]) - const baseline = sources.monitoringSpans.find((span) => span.endedAt === START - MINUTE) - assert.equal(baseline.startedAt, START - 11 * MINUTE) - assert.deepEqual(state.background, { medianPerMinute: 4, minutes: 10, source: 'measured' }) - // An explicit value overrides the measurement. - const explicit = initialWatchState({ cellId: CELL, startedAt: START, durationMinutes: 60, background503PerMinute: 9, flippedAt }) - assert.deepEqual(explicit.background, { medianPerMinute: 9, source: 'input' }) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -test('a quiet fleet with zero director 503s watches on and never trips', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - let clock = START - const config = parseAt( - watchValues({ - 'state-file': join(directory, 'state.json'), - 'segment-minutes': '20', - 'flipped-at': new Date(START - 2 * MINUTE).toISOString() - }) - ) - const lines = [] - const state = await watchSegment(config, { - ...fakeSources({ per503: () => 0 }), - idToken: 'id', - retryDelayMs: 0, - now: () => clock, - sleep: async (ms) => { clock += ms }, - log: (line) => lines.push(JSON.parse(line)) - }) - assert.equal(state.state, 'watching') - assert.equal(lines.length, 20) - assert.deepEqual(state.background, { medianPerMinute: 0, minutes: 10, source: 'measured' }) - assert.ok(lines.slice(1).every((line) => line.rules.director503.status === 'pass')) - assert.ok(lines.every((line) => line.unverifiableStreak === 0)) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -// 10-20 c29: a hand flip back to db mid-watch crashed the segment ("result.pages is not iterable"). -test('a segment that waits for reserve, then sees a hand flip back to db, stops without crashing', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - let clock = START - const config = parseAt( - watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10', 'flipped-at': new Date(START - 2 * MINUTE).toISOString() }) - ) - const db = { flagsApplied: { generation: 5, flags: { admitMode: 'db', readinessLocal: true } }, admitModeEffective: 'db' } - const lines = [] - const state = await watchSegment(config, { - ...fakeSources({ cell: (sample) => (sample === 1 || sample >= 4 ? db : {}) }), - idToken: 'id', - retryDelayMs: 0, - now: () => clock, - sleep: async (ms) => { clock += ms }, - log: (line) => lines.push(JSON.parse(line)) - }) - assert.equal(lines[0].rules.reserve.status, 'pending') - assert.equal(state.state, 'stopped') - assert.match(state.verdict, /is not in reserve mode \(switch file says db\)/) - assert.equal(lines.length, 4) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -// 2026-10-18's false trips: 503s tripled with traffic, nearly all a host re-dialling early. -const rateLimited = (perLine) => ({ placementRejectionsByReasonDelta: { 'host-rate-limited': perLine }, assign503sByCauseDelta: { 'placement-lane': perLine } }) - -async function watchFor(minutes, sources, flippedAt = START - 2 * MINUTE) { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - let clock = START - const config = parseAt( - watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': String(minutes), 'flipped-at': new Date(flippedAt).toISOString() }) - ) - const lines = [] - const state = await watchSegment(config, { - ...sources, - idToken: 'id', - retryDelayMs: 0, - now: () => clock, - sleep: async (ms) => { clock += ms }, - log: (line) => lines.push(JSON.parse(line)) - }) - return { state, lines } - } finally { - await rm(directory, { recursive: true, force: true }) - } -} - -test('a host re-dialling inside its own interval never trips the 503 rule, however much it grows', async () => { - // Background 20/min, all re-dials; then 50/min, 48 of them re-dials. All 503s would trip at 40. - const flippedAt = START - 2 * MINUTE - const { state, lines } = await watchFor(20, fakeSources({ - per503: (minute) => (minute < flippedAt ? 20 : 50), - metrics: (at) => [rateLimited(at <= flippedAt ? 10 : 24)] - }), flippedAt) - assert.equal(state.state, 'watching') - assert.deepEqual(state.background, { medianPerMinute: 0, minutes: 10, source: 'measured' }) - assert.ok(lines.slice(1).every((line) => line.rules.director503.status === 'pass')) - assert.ok(lines.slice(1).every((line) => Object.values(line.rules.director503.recent).every((count) => count <= 2))) -}) - -test('503s the director could not serve trip after three minutes over the line, beside the re-dials', async () => { - const flippedAt = START - 2 * MINUTE - const { state } = await watchFor(10, fakeSources({ - per503: (minute) => (minute < flippedAt ? 20 : 60), - // 10 re-dials a minute on a throughout; the 40 more after the flip are queue waits that timed - // out, 20 a minute on each of two instances. - metrics: (at) => { - const waits = at <= flippedAt ? {} : { stickyRejectionsByReasonDelta: { 'wait-timeout': 10 }, assign503sByCauseDelta: { 'sticky-lane': 10 } } - const a = rateLimited(5) - return [{ ...a, ...waits, assign503sByCauseDelta: { ...a.assign503sByCauseDelta, ...waits.assign503sByCauseDelta } }, waits] - } - }), flippedAt) - assert.equal(state.state, 'tripped') - assert.deepEqual(state.background, { medianPerMinute: 10, minutes: 10, source: 'measured' }) - assert.match(state.verdict, /director503: director 503s above 30\/min for 3 min \(50, 50, 50\), 2 instances each above 15\/min$/) -}) - -test('the combined read hands each query only its own lines', async () => { - const at = START - const endedAt = START - 3 * MINUTE - const queries = sampleLogQueries({ - phoneRequest: { cellId: CELL, endedAt, minutes: 10 }, - director: { directorService: 'orca-cloud-relay', endedAt, minutes: 10 }, - seatRequest: { directorService: 'orca-cloud-relay', cellId: CELL, at } - }) - const director = (ago, jsonPayload) => ({ timestamp: new Date(at - ago).toISOString(), resource: { type: 'cloud_run_revision' }, labels: { instanceId: 'a' }, jsonPayload }) - const entries = { - metrics: director(5 * MINUTE, { event: METRICS_EVENT }), - phone: { timestamp: new Date(at - 5 * MINUTE).toISOString(), resource: { type: 'gce_instance' }, jsonPayload: { event: METRICS_EVENT, cellId: CELL } }, - freshSummary: director(60_000, { event: SUMMARY_EVENT, pending: 0, dropped: 0 }), - staleSummary: director(10 * MINUTE, { event: SUMMARY_EVENT, pending: 0, dropped: 0 }), - ownSeat: director(2 * MINUTE, { event: SEAT_EVENT, cellId: CELL, epoch: 1, rowEpoch: 2 }), - otherSeat: director(2 * MINUTE, { event: SEAT_EVENT, cellId: 'production-gce-c99', epoch: 1, rowEpoch: 2 }) - } - let filter - const logs = await readSampleLogs( - fakeReader({ runGcloud: async (args) => ((filter = args[2]), { stdout: JSON.stringify(Object.values(entries)) }) }), - queries - ) - const names = (read) => read.entries.map((entry) => Object.keys(entries).find((name) => JSON.stringify(entries[name]) === JSON.stringify(entry))) - assert.equal(filter.split(') OR (').length, 4) - assert.deepEqual(names(logs.directorMetrics), ['metrics']) - assert.deepEqual(names(logs.phone), ['phone']) - // The ledger rule wants a summary from the last 150 s; the seat read's positive control, 15 min. - assert.deepEqual(names(logs.ledger), ['freshSummary']) - assert.deepEqual(names(logs.seatBehindRow), ['freshSummary', 'staleSummary', 'ownSeat']) - // Without the phone path its query is not sent. - const unwatched = sampleLogQueries({ phoneRequest: false, director: { directorService: 'orca-cloud-relay', endedAt, minutes: 10 }, seatRequest: { directorService: 'orca-cloud-relay', cellId: CELL, at } }) - const sent = await readSampleLogs(fakeReader({ runGcloud: async (args) => ((filter = args[2]), { stdout: '[]' }) }), unwatched) - assert.equal(filter.split(') OR (').length, 3) - assert.doesNotMatch(filter, /gce_instance/) - assert.equal('phone' in sent, false) - // A full read is truncated for every query; a failed one names its error. - const full = await readSampleLogs(fakeReader({ runGcloud: async () => ({ stdout: JSON.stringify(Array.from({ length: 1_000 }, () => entries.metrics)) }) }), unwatched) - assert.ok(Object.values(full).every((read) => read.failed && read.error === 'sample log read truncated')) -}) - -// 10-21 18:39Z: twelve watches at ~5 Cloud Logging reads a sample hit the project's 60 a minute. -test('a sample makes one Cloud Logging read, and a 429 retried inside its budget stays verifiable', async () => { - const memory = { flagsApplied: { generation: 4, flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } } } - const sources = fakeSources({ cell: () => memory, phone: () => ({ phoneAttachDeadlinesDelta: 0, clientAcceptCompletedDelta: 10 }) }) - // Sample 6's combined read answers 429 once; sample 8's (the 9th read), on all three attempts. - let combined = 0 - let ninth = 0 - const runGcloud = async (args, options) => { - if (args[0] === 'logging' && args[2].startsWith('(')) { - combined += 1 - if (combined === 6) throw new Error('RESOURCE_EXHAUSTED: 429 Quota exceeded for ReadRequestsPerMinutePerProject') - if (combined >= 9 && ninth < 3) { - ninth += 1 - throw new Error('RESOURCE_EXHAUSTED: 429 Quota exceeded for ReadRequestsPerMinutePerProject') - } - } - return sources.runGcloud(args, options) - } - const { state, lines } = await watchFor(12, { ...sources, runGcloud }) - assert.equal(state.state, 'watching') - assert.equal(lines.length, 12) - // One combined read per sample: 12, one retry for sample 6 and two for sample 8. Only the - // one-time backgrounds read alone. - assert.equal(combined, 15) - const reads = sources.logReads.map(({ filter }) => filter) - assert.equal(reads.filter((filter) => filter.startsWith('(')).length, 11) - assert.ok(reads.filter((filter) => !filter.startsWith('(')).length <= 3) - const combinedRead = reads.find((filter) => filter.startsWith('(')) - for (const part of ['gce_instance', METRICS_EVENT, SUMMARY_EVENT, SEAT_EVENT]) assert.ok(combinedRead.includes(part), part) - // The retried 429 left sample 6 whole; sample 8, which never read, is one gap the next heals. - assert.equal(lines[5].unverifiableStreak, 0) - assert.ok(Object.values(lines[5].rules).every((rule) => rule.status !== 'unverifiable')) - assert.equal(lines[7].unverifiableStreak, 1) - assert.match(lines[7].rules.duplicateSeat.reason, /RESOURCE_EXHAUSTED/) - assert.match(lines[7].rules.director503.reason, /director runtime metrics read failed: RESOURCE_EXHAUSTED/) - assert.equal(lines[8].unverifiableStreak, 0) -}) - -test('a minute no director instance logged runtime metrics for is unverifiable, not zero', async () => { - // Two instances, both silent for three minutes in the middle of the read. - const endedAt = START + 2 * MINUTE - const silent = (at) => at > endedAt - 5 * MINUTE && at <= endedAt - 2 * MINUTE - const reader = fakeSources({ per503: () => 50, metrics: (at) => (silent(at) ? [] : [rateLimited(12), rateLimited(12)]) }) - const read = await readDirector503Minutes( - { ...fakeReader(), fetch: reader.fetchImpl, runGcloud: reader.runGcloud }, - { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt, minutes: 10 } - ) - assert.equal(read.ok, true) - assert.deepEqual(read.unmetered, read.minutes.slice(5, 8)) - assert.equal(read.perMinute[read.minutes[0]], 2) - const judged = evaluateWatchSample(freshState({ background: { medianPerMinute: 0, source: 'measured' } }), sampleAt(START, { director503: read }), LIMIT) - assert.equal(judged.rules.director503.status, 'unverifiable') - assert.match(judged.rules.director503.reason, /no director runtime metrics for/) - // No line at all in the read is short, not a quiet fleet. - const unlogged = fakeSources({ per503: () => 50, metrics: () => [] }) - const none = await readDirector503Minutes( - { ...fakeReader(), fetch: unlogged.fetchImpl, runGcloud: unlogged.runGcloud }, - { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt, minutes: 10 } - ) - assert.equal(none.ok, false) - assert.match(none.error, /runtime metrics read failed, truncated, or short/) -}) - -test('no request series at all is unverifiable and trips at the limit', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - let clock = START - const config = parseAt(watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10', 'background-503-per-minute': '0', 'flipped-at': '' })) - const state = await watchSegment(config, { - ...fakeSources({ per503: () => 0, perOther: () => 0 }), - idToken: 'id', - retryDelayMs: 0, - now: () => clock, - sleep: async (ms) => { clock += ms }, - log: () => {} - }) - assert.equal(state.state, 'tripped') - assert.match(state.verdict, /unverifiable: 3 samples in a row could not read director503/) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -test('a later segment judges flipped-at against the watch start, so a long watch does not age it out', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - const stateFile = join(directory, 'state.json') - const values = watchValues({ 'state-file': stateFile, 'flipped-at': new Date(START - 23 * 60 * MINUTE).toISOString() }) - const later = START + 4 * 60 * MINUTE - await writeFile(stateFile, JSON.stringify(freshState())) - assert.equal((await parseWatchSegment(values, later)).flippedAt, START - 23 * 60 * MINUTE) - // The first segment (no state file yet) still refuses it, and records refused. - await rm(stateFile) - await assert.rejects(parseWatchSegment(values, later), /more than 24 hours/) - assert.equal(JSON.parse(await readFile(stateFile, 'utf8')).state, 'refused') - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -test('a corrupt state file fails the segment, never refuses or restarts it, and the outcome reads failed', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - const stateFile = join(directory, 'state.json') - const output = join(directory, 'out') - const values = watchValues({ 'state-file': stateFile }) - for (const corrupt of ['null', '[]', '7', JSON.stringify({ state: 'watching', cellId: 'c7' })]) { - await writeFile(stateFile, corrupt) - await assert.rejects(parseWatchSegment(values, START), /state file/) - assert.equal(await readFile(stateFile, 'utf8'), corrupt) - assert.equal((await recordOutcome(stateFile, output)).state, 'failed') - } - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -test('a slow runtime-status reply is retried before the flip back calls for break glass', async () => { - const { fetchImpl: answering, uploads } = fakeFlipBack() - let runtimeCalls = 0 - // fetchAdminOnceMore makes two tries per read; the first read's two both fail. - const fetchImpl = async (url, init) => { - if (String(url).endsWith('/v1/admin/runtime-status') && (runtimeCalls += 1) <= 2) return new Response('', { status: 503 }) - return answering(url, init) - } - const lines = [] - const sleeps = [] - const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines, sleeps)) - assert.equal(result.written, true) - assert.equal(uploads(), 1) - assert.deepEqual(lines.filter((line) => line.event === 'orca_relay_reserve_watch_runtime_retry').map((line) => line.attempt), [1]) - assert.ok(!lines.some((line) => line.event === 'orca_relay_reserve_watch_break_glass_required')) - assert.ok(sleeps.includes(2_000)) - assert.ok(FLIP_BACK_RUNTIME_ATTEMPTS > 1) -}) - -const watchWorkflow = readRelayWorkflow('operate-relay-production-reserve-watch.yml') -const jobOf = (name) => { - const start = watchWorkflow.indexOf(`\n ${name}:\n`) - assert.notEqual(start, -1, `job ${name} is missing`) - const next = watchWorkflow.slice(start + 1).search(/\n [a-z-]+:\n/) - return watchWorkflow.slice(start, next === -1 ? undefined : start + 1 + next) -} - -test('workflow: neither job takes the flag workflow group, which would let a newer run cancel a pending flip back', () => { - assert.doesNotMatch(watchWorkflow, /group: relay-production-cell-flags/) - assert.doesNotMatch(jobOf('flip-back'), /^\s+concurrency:/m) - assert.doesNotMatch(jobOf('watch'), /^\s+concurrency:/m) -}) - -test('workflow: a watch that failed or was cancelled without a verdict flips back too', () => { - const flip = jobOf('flip-back') - // A cancel re-evaluates each step: every flip-back step must run on always() too. - const steps = flip.slice(flip.indexOf('\n steps:\n')).split(/\n - /).slice(1) - assert.equal(steps.length, 7) - for (const step of steps) assert.match(step, /\n\s+if: \$\{\{ always\(\) \}\}/, step.split('\n')[0]) - assert.match(flip, /if: \$\{\{ always\(\) && vars\.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && needs\.watch\.result != 'skipped'/) - assert.match(flip, /!contains\(fromJSON\('\["completed","stopped","refused"\]'\), needs\.watch\.outputs\.state\)/) - // Every segment checks its ID token's shape, and runs the one shared command. - const segments = watchWorkflow.match(/run: bash -c "\$\{WATCH_SEGMENT\}"/g) ?? [] - assert.equal(segments.length, 6) - assert.match(jobOf('watch'), /process\.env\.ORCA_RELAY_ADMIN_ID_TOKEN/) -}) - -test('a live rollout lease refuses the watch before its first sample; an expired one does not', async () => { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - const stateFile = join(directory, 'state.json') - const output = join(directory, 'out') - const config = parseAt(watchValues({ 'state-file': stateFile, 'github-output': output, 'segment-minutes': '1' })) - const lease = (expiresAt) => ({ record: { expires_at: expiresAt }, holder: 'Deploy Relay Production Same-Cap (run 9)' }) - let clock = START - const dependencies = (rolloutLease) => ({ - ...fakeSources({ rolloutLease }), - idToken: 'id', - retryDelayMs: 0, - now: () => clock, - sleep: async (ms) => { clock += ms }, - log: () => {} - }) - const held = await watchSegment(config, dependencies(lease(START + MINUTE))) - assert.equal(held.state, 'refused') - assert.match(held.verdict, /a rollout holds the Cloud SQL rollout lease until .*Same-Cap \(run 9\)/) - assert.equal(JSON.parse(await readFile(stateFile, 'utf8')).state, 'refused') - assert.match(await readFile(output, 'utf8'), /^state=refused\n/) - await rm(stateFile) - assert.equal((await watchSegment(config, dependencies(lease(START - 1)))).state, 'watching') - // Only the first segment checks: a later one carries on from its state file. - assert.equal((await watchSegment(config, dependencies(lease(clock + MINUTE)))).state, 'watching') - // A lease it cannot read refuses too, as the lease action fails closed. - assert.match(await rolloutLeaseRefusal(async () => { throw new Error('403') }, START), /lease is unreadable \(403\)/) - assert.equal(await rolloutLeaseRefusal(async () => null, START), null) - } finally { - await rm(directory, { recursive: true, force: true }) - } -}) - -const stepOf = (job, name) => { - const text = jobOf(job) - const start = text.indexOf(`- name: ${name}\n`) - assert.notEqual(start, -1, `step ${name} is missing`) - const end = text.indexOf('\n - ', start + 1) - return text.slice(start, end === -1 ? undefined : end) -} -const ALARM_COMMAND = /node dev\/scripts\/watch-relay-reserve-cell\.mjs alarm --kind KIND \\\n\s+--mode "\$\{MODE\}" --cell-id "\$\{CELL_ID\}" --project-id "\$\{GCP_PROJECT_ID\}" --verdict "\$\{VERDICT_VAR\}"/ -const alarmCommand = (kind, verdict) => new RegExp(ALARM_COMMAND.source.replace('KIND', kind).replace('VERDICT_VAR', verdict)) - -test('workflow: a flip back that did not land and a refused watch both raise the alarm', () => { - const alarm = stepOf('flip-back', 'Alarm when the flip back did not land') - assert.match(alarm, /if: \$\{\{ always\(\) \}\}/) - assert.match(alarm, /grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "\$\{RUNNER_TEMP\}\/relay-reserve-flip-back\.jsonl"/) - assert.match(alarm, alarmCommand('flip-back-failed', 'VERDICT')) - const refused = stepOf('watch', 'Alarm on a refused watch') - assert.match(refused, /if: \$\{\{ always\(\) && steps\.outcome\.outputs\.state == 'refused' \}\}/) - assert.match(refused, /WATCH_VERDICT: \$\{\{ steps\.outcome\.outputs\.verdict \}\}/) - assert.match(refused, alarmCommand('refused', 'WATCH_VERDICT')) - // After the outcome step, so it reads the recorded state. - assert.ok(jobOf('watch').indexOf('- name: Alarm on a refused watch') > jobOf('watch').indexOf('- id: outcome')) - // The flip-back line names only the write's own event, never its conflict retries. - assert.match(readFileSync(new URL('./watch-relay-reserve-cell.mjs', import.meta.url), 'utf8'), /event: 'orca_relay_reserve_watch_flip_back',\n/) -}) - -test('an alarm is an escaped error annotation everywhere, and in write mode a paging log line', async () => { - const calls = [] - const lines = [] - const runGcloud = async (args, options) => (calls.push({ args, options }), { stdout: '' }) - const alarm = (overrides) => - raiseAlarm( - { mode: 'write', cellId: CELL, projectId: 'onorca-cloud', kind: 'refused', reason: '50%\r\nlease held', ...overrides }, - { runGcloud, log: (line) => lines.push(line) } - ) - assert.deepEqual(await alarm(), { paged: true }) - assert.equal( - lines[0], - `::error title=Reserve watch refused for ${CELL}::${CELL} may be in reserve mode and unwatched: the watch was refused (50%25%0D%0Alease held). ` + - 'Rerun it, or flip the cell to db with Operate Relay Production Cell Flags.' - ) - assert.deepEqual(calls[0].args, [ - 'logging', 'write', ALARM_LOG_NAME, - JSON.stringify({ event: ALARM_EVENT, cellId: CELL, kind: 'refused', reason: '50%\r\nlease held' }), - '--payload-type=json', '--severity=ERROR', '--project', 'onorca-cloud' - ]) - // A dry run never flips back, so its alarm annotates but does not page. - assert.deepEqual(await alarm({ mode: 'dry-run', kind: 'flip-back-failed' }), { paged: false }) - assert.equal(calls.length, 1) - assert.match(lines[1], /^::error title=Reserve watch flip-back-failed for production-gce-c26::.*did not land/) - // A title property escapes its own separators too. - await alarm({ mode: 'dry-run', cellId: 'a:b,c' }) - assert.match(lines[2], /^::error title=Reserve watch refused for a%3Ab%2Cc::/) - // A failed log write is named, never swallowed silently. - const failed = await raiseAlarm( - { mode: 'write', cellId: CELL, projectId: 'onorca-cloud', kind: 'flip-back-failed', reason: 'x' }, - { runGcloud: async () => { throw new Error('PERMISSION_DENIED') }, log: (line) => lines.push(line) } - ) - assert.deepEqual(failed, { paged: false }) - assert.match(lines.at(-1), /^::error title=Reserve watch page failed::.*nothing pages: PERMISSION_DENIED/) - await assert.rejects(alarm({ kind: 'other' }), /--kind must be one of/) - // A missing project still annotates, then names the page it could not send. - const before = calls.length - assert.deepEqual(await alarm({ projectId: undefined }), { paged: false }) - assert.equal(calls.length, before) - assert.match(lines.at(-2), /^::error title=Reserve watch refused for /) - assert.match(lines.at(-1), /nothing pages: --project-id is not a GCP project id/) -}) - -// Step 6 phone rules. A quiet window, minute by minute: ordinary 4401s from phones whose pairing -// was removed, an occasional failed confirm or half-open control. Synthetic: replaying a recorded -// production window needs read access this suite does not have. -const QUIET_PHONE_MINUTE = (index) => ({ - credentialRefusals: [0, 1, 0, 2, 1][index % 5], - confirmFailures: [0, 0, 1, 0][index % 4], - attachDeadlines: [1, 0, 2, 0, 1, 3][index % 6], - acceptDatabaseErrors: 0, - renewalFailures: 0, - completedAccepts: [10, 12, 8][index % 3] -}) -const phoneRead = (minuteCount, perMinute) => { - const minutes = Array.from({ length: minuteCount }, (_, index) => `2026-10-19T12:${String(index).padStart(2, '0')}`) - const rows = minutes.map((_, index) => perMinute(index)) - const series = Object.fromEntries(Object.keys(rows[0]).map((signal) => [signal, rows.map((row) => row[signal])])) - return { ok: true, minutes, series, unmetered: [] } -} -const memoryCell = (overrides = {}) => healthyCell({ phonePathConfigured: 'memory', resumeAuditDropped: 0, ...overrides }) -const phoneState = (overrides = {}) => - freshState({ - flippedAt: START - 10 * MINUTE, - phoneSince: START - 10 * MINUTE, - phoneBackground: measuredPhoneBackground(phoneRead(10, QUIET_PHONE_MINUTE)), - ...overrides - }) -// The last 10 minutes as of watch minute `offset`, the newest minutes shaped by `spike`. -const phoneSampleAt = (offset, spike = () => ({}), cell = memoryCell()) => - sampleAt(START + offset * MINUTE, { - cell, - phone: phoneRead(10, (index) => ({ ...QUIET_PHONE_MINUTE(offset + index), ...spike(index) })) - }) - -test('phone rules: a quiet hour replays without a trip', () => { - const results = run(Array.from({ length: 60 }, (_, offset) => phoneSampleAt(offset)), phoneState({ deadlineAt: START + 120 * MINUTE })) - assert.equal(results.length, 60) - assert.ok(results.every((result) => result.state.state === 'watching' && result.state.phoneTrip === null)) - assert.deepEqual(Object.keys(results[0].rules).filter((name) => name.startsWith('phone')), PHONE_RULE_NAMES) -}) - -test('phone rules: judged as a share of attempts, over half and over the background', () => { - assert.equal(phoneShareLine(0), 0.5) - assert.equal(phoneShareLine(0.3), 0.55) - assert.equal(phoneShareLine(0.5), 0.6) - // Exactly half is not over; nor is a minute with under 4 attempts. - assert.equal(phoneMinuteOver('credentialRefusals', 9, 9, 0.5), false) - assert.equal(phoneMinuteOver('credentialRefusals', 11, 10, 0.5), true) - assert.equal(phoneMinuteOver('credentialRefusals', 3, 0, 0.5), false) - assert.equal(phoneMinuteOver('credentialRefusals', 4, 0, 0.5), true) - // Each accept makes one confirm: every confirm failing is all of them, not half the attempts. - assert.equal(phoneMinuteOver('confirmFailures', 4, 4, 0.6), true) - assert.equal(phoneMinuteOver('confirmFailures', 2, 4, 0.5), false) - assert.equal(phoneMinuteOver('confirmFailures', 9, 3, 0.5), false) - const background = measuredPhoneBackground(phoneRead(10, () => ({ ...QUIET_PHONE_MINUTE(0), credentialRefusals: 1, completedAccepts: 9 }))) - assert.equal(background.share.credentialRefusals, 0.1) - assert.equal(measuredPhoneBackground({ ...phoneRead(10, QUIET_PHONE_MINUTE), unmetered: ['2026-10-19T12:03'] }), null) -}) - -test('phone rules: each fires on its own spike, and the watch goes on with the reserve rules', () => { - const spikes = { - phoneCredentialRefusals: { credentialRefusals: 25 }, - phoneConfirmFailures: { confirmFailures: 25 }, - phoneAttachDeadlines: { attachDeadlines: 25 }, - phoneAcceptDatabaseErrors: { acceptDatabaseErrors: 25 } - } - for (const [rule, counts] of Object.entries(spikes)) { - const [result] = run([phoneSampleAt(0, (index) => (index >= 7 ? counts : {}))], phoneState()) - assert.equal(result.state.state, 'watching', rule) - assert.match(result.state.phoneTrip, new RegExp(`turn off the phone path on ${CELL}: ${rule}: `), rule) - } - // Two minutes over the line is not three. - const [twoMinutes] = run([phoneSampleAt(0, (index) => (index >= 8 ? { credentialRefusals: 25 } : {}))], phoneState()) - assert.equal(twoMinutes.state.phoneTrip, null) - // Once the watch turned the phone path off, the phone rules stand down. - const [off] = run([phoneSampleAt(0, () => ({ credentialRefusals: 50 }))], phoneState({ phoneOffAt: START })) - assert.deepEqual(Object.keys(off.rules).filter((name) => name.startsWith('phone')), []) -}) - -test('phone rules: judged only while the switch file says memory; reserve rules still flip the cell back', () => { - const [off] = run([sampleAt(START, { cell: healthyCell({ phonePathConfigured: 'db' }) })], freshState()) - assert.deepEqual(Object.keys(off.rules).filter((name) => name.startsWith('phone')), []) - const [overrun] = run([phoneSampleAt(0, () => ({}), memoryCell({ enforcedUnits: 2_001 }))], phoneState()) - assert.equal(overrun.state.state, 'tripped') - assert.equal(overrun.state.flip, 'admitMode') -}) - -test('phone rules: missing phone data turns off only the phone path, and needs a flipped-at background', () => { - const gap = (offset) => ({ ...phoneSampleAt(offset), phone: { ...phoneSampleAt(offset).phone, unmetered: ['2026-10-19T12:09'] } }) - const results = run([gap(0), gap(1), gap(2)], phoneState()) - assert.equal(results.at(-1).state.state, 'watching') - assert.equal(results.at(-1).state.unverifiableStreak, 0) - assert.match(results.at(-1).state.phoneTrip, /unverifiable: 3 samples/) - const [noBackground] = run([phoneSampleAt(0)], freshState({ flippedAt: null })) - assert.match(noBackground.rules.phoneCredentialRefusals.reason, /--flipped-at/) - // A gap before the judged 3 minutes changes nothing. - const early = { ...phoneSampleAt(0), phone: { ...phoneSampleAt(0).phone, unmetered: ['2026-10-19T12:02'] } } - const [earlyGap] = run([early], phoneState()) - assert.equal(earlyGap.rules.phoneCredentialRefusals.status, 'pass') -}) - -test('phone rules: memory set partway through a watch measures from then, not from the reserve flip', () => { - const state = freshState({ flippedAt: START - 60 * MINUTE }) - const [before, after] = run([sampleAt(START), phoneSampleAt(1)], state) - assert.equal(before.state.phoneSince, null) - assert.equal(after.state.phoneSince, START) - assert.equal(after.rules.phoneCredentialRefusals.status, 'pending') -}) - -test('phone rules: audit drops and failing renewals page once when sustained, and flip nothing', () => { - const dropping = (offset, dropped) => phoneSampleAt(offset, () => ({}), memoryCell({ resumeAuditDropped: dropped })) - // One drop, then none: no page. - const single = run([dropping(0, 0), dropping(1, 1), ...Array.from({ length: 8 }, (_, index) => dropping(index + 2, 1))], phoneState()) - assert.ok(single.every((result) => result.pages.length === 0)) - const rising = run(Array.from({ length: 9 }, (_, offset) => dropping(offset, offset * 3)), phoneState()) - assert.deepEqual(rising.map((result) => result.pages.join()), ['', '', '', '', '', '', 'audit-drops', '', '']) - const renewing = run(Array.from({ length: 7 }, (_, offset) => phoneSampleAt(offset, () => ({ renewalFailures: 1 }))), phoneState()) - assert.deepEqual(renewing.map((result) => result.pages.join()).filter(Boolean), ['renewal-failures']) - assert.ok([...rising, ...renewing].every((result) => result.state.state === 'watching' && result.state.phoneTrip === null)) -}) - -test('the cell phone read sums each minute from the cell runtime metrics lines, and refuses an old image', async () => { - const endedAt = Date.parse('2026-10-19T12:03:00Z') - const line = (at, payload) => ({ - timestamp: new Date(at).toISOString(), - jsonPayload: { - phoneRejectionsByCodeDelta: {}, - resumeConfirmFailuresByCodeDelta: {}, - phoneAttachDeadlinesDelta: 0, - phoneAcceptDatabaseErrorsDelta: 0, - resumeRenewalFailuresDelta: 0, - clientAcceptCompletedDelta: 0, - ...payload - } - }) - let filter - const read = await readCellPhoneMinutes( - fakeReader({ - runGcloud: async (args) => { - filter = args[2] - return { - stdout: JSON.stringify([ - line(endedAt - 150_000, { phoneRejectionsByCodeDelta: { 4401: 2, 4404: 9 } }), - line(endedAt - 120_000, { resumeConfirmFailuresByCodeDelta: { 'reject-revoked': 1, error: 2 } }), - line(endedAt - 90_000, { phoneAttachDeadlinesDelta: 3, phoneAcceptDatabaseErrorsDelta: 1, resumeRenewalFailuresDelta: 4, clientAcceptCompletedDelta: 7 }) - ]) - } - } - }), - { cellId: CELL, endedAt, minutes: 3 } - ) - assert.match(filter, /resource\.type="gce_instance"/) - assert.match(filter, new RegExp(`jsonPayload\\.cellId="${CELL}"`)) - assert.deepEqual(read.series, { - credentialRefusals: [2, 0, 0], - confirmFailures: [3, 0, 0], - attachDeadlines: [0, 3, 0], - acceptDatabaseErrors: [0, 1, 0], - renewalFailures: [0, 4, 0], - completedAccepts: [0, 7, 0] - }) - assert.deepEqual(read.unmetered, ['2026-10-19T12:02']) - const old = await readCellPhoneMinutes( - fakeReader({ runGcloud: async () => ({ stdout: JSON.stringify([{ timestamp: new Date(endedAt - 1).toISOString(), jsonPayload: {} }]) }) }), - { cellId: CELL, endedAt, minutes: 3 } - ) - assert.match(old.error, /no phone counters/) - assert.equal(old.permanent, true) -}) - -// The watch over a memory cell whose phone metrics come from `phoneMetrics(at)`. -async function phoneWatch(phoneMetrics, minutes) { - const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) - try { - let clock = START - const lines = [] - const config = parseAt(watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': String(minutes) })) - const sources = fakeSources({ - cell: () => ({ flagsApplied: { generation: 4, flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } } }), - phone: phoneMetrics - }) - const state = await watchSegment(config, { - ...sources, - idToken: 'id', - accessToken: 'access', - audit: {}, - retryDelayMs: 0, - now: () => clock, - sleep: async (ms) => { clock += ms }, - log: (line) => lines.push(line) - }) - return { state, lines } - } finally { - await rm(directory, { recursive: true, force: true }) - } -} -const phoneCounters = (overrides = {}) => ({ - phoneRejectionsByCodeDelta: {}, - resumeConfirmFailuresByCodeDelta: {}, - phoneAttachDeadlinesDelta: 0, - phoneAcceptDatabaseErrorsDelta: 0, - resumeRenewalFailuresDelta: 0, - clientAcceptCompletedDelta: 5, - ...overrides -}) - -test('a phone trip whose phonePath=db write fails keeps watching, retries each sample, and pages after 3 failures', async () => { - // No bucket here, so every write fails. - const { state, lines } = await phoneWatch((at) => phoneCounters(at > START ? { phoneAttachDeadlinesDelta: 20 } : {}), 15) - assert.equal(state.state, 'watching') - assert.equal(state.phoneOffAt, null) - assert.match(state.phoneTrip, /phoneAttachDeadlines: /) - const failures = lines.filter((line) => line.includes('orca_relay_reserve_watch_phone_path_off_failed')) - assert.ok(failures.length > 3, `${failures.length} failed writes`) - assert.equal(state.phoneOffFailures, failures.length) - const alarms = lines.filter((line) => line.includes('may still have phonePath=memory')) - assert.equal(alarms.length, 1) -}) - -test('a phone trip still unwritten when the watch ends leaves it to the flip back', () => { - const [last] = run([phoneSampleAt(0)], phoneState({ deadlineAt: START, phoneTrip: 'turn off the phone path on x: phoneAttachDeadlines: y' })) - assert.equal(last.state.state, 'tripped') - assert.equal(last.state.flip, 'phonePath') - assert.match(last.state.verdict, /phoneAttachDeadlines: y; phonePath=db did not land before the watch ended/) -}) - -test('a phone background broken by a restart or roll refuses the watch instead of turning phones off for certain', async () => { - const { state } = await phoneWatch((at) => (at < START - 8 * MINUTE ? {} : phoneCounters()), 5) - assert.equal(state.state, 'refused') - assert.match(state.verdict, /phone background.*incomplete.*later --flipped-at/) -}) - -test('a phone-rule flip back removes phonePath, and does nothing once the file no longer says memory', async () => { - const { fetchImpl, calls } = fakeFlipBack({ flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } }) - const result = await flipBack(flipValues({ flip: 'phonePath' }), flipDependencies(fetchImpl, [])) - assert.equal(result.written, true) - const upload = calls.find((call) => call.url.includes('/upload/storage/v1/')) - assert.match(upload.body, /"flags":\{"admitMode":"reserve","readinessLocal":true\}/) - assert.ok(!calls.some((call) => call.url.endsWith('/v1/admin/cell-admit-mode') && JSON.parse(call.body).admitMode)) - const done = await flipBack(flipValues({ flip: 'phonePath' }), flipDependencies(fakeFlipBack().fetchImpl, [])) - assert.equal(done.alreadyDb, true) -}) - -test('both flip backs land on a cell rolled back to an image without the phonePath switch', async () => { - const flags = { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } - for (const flip of ['admitMode', 'phonePath']) { - const { fetchImpl, calls } = fakeFlipBack({ flags, supportedFlags: { admitMode: { type: 'enum', values: ['db', 'reserve'] }, readinessLocal: { type: 'boolean' } } }) - const result = await flipBack(flipValues({ flip }), flipDependencies(fetchImpl, [])) - assert.equal(result.written, true, flip) - const upload = calls.find((call) => call.url.includes('/upload/storage/v1/')) - assert.doesNotMatch(upload.body, /phonePath/, flip) - } -}) - -test('workflow: the flip back writes what the watch says tripped, and the whole cell when it says nothing', () => { - const workflow = watchWorkflow - assert.match(workflow, /flip: \$\{\{ steps\.outcome\.outputs\.flip \}\}/) - assert.match(workflow, /FLIP: \$\{\{ needs\.watch\.outputs\.flip \|\| 'admitMode' \}\}/) - assert.match(workflow, /--flip "\$\{FLIP\}"/) - // A failed phone flip back pages with the phone wording. - assert.match(workflow, /--verdict "\$\{VERDICT\}" --flip "\$\{FLIP\}"\n/) - assert.match(workflow, /ORCA_RELAY_CONTROL_ACCESS_TOKEN="\$\(gcloud auth print-access-token\)"\n\s+echo "::add-mask::/) -}) - - -test('a failed phone flip back and the phone pages say what is still on', async () => { - const lines = [] - const runGcloud = async () => ({ stdout: '' }) - await raiseAlarm({ mode: 'dry-run', cellId: CELL, projectId: 'onorca-cloud', kind: 'flip-back-failed', reason: 'x', flip: 'phonePath' }, { runGcloud, log: (line) => lines.push(line) }) - await raiseAlarm({ mode: 'dry-run', cellId: CELL, projectId: 'onorca-cloud', kind: 'renewal-failures', reason: 'y' }, { runGcloud, log: (line) => lines.push(line) }) - assert.match(lines[0], /may still have phonePath=memory/) - assert.doesNotMatch(lines[0], /reserve mode/) - assert.match(lines[1], /cannot renew phone credentials/) -}) - -// Recorded, not synthetic: 26 h of c7, c28 and c29 runtime metrics (read-only, 10-18/19). The phone -// counters are not on those images yet, so auth failures (host and phone 4401s, an over-count) -// stand in for the refused or failed attempts, against completed accepts. They cannot stand in for -// confirm failures: c28's 04:08Z minutes (9/9, 9/9, 11/10) read as confirms are a total outage. -const PHONE_RULE_SIGNAL_NAMES = ['credentialRefusals', 'confirmFailures', 'attachDeadlines', 'acceptDatabaseErrors'] -const recordedPhoneMinutes = JSON.parse(readFileSync(new URL('./relay-phone-watch-recorded-quiet-minutes.json', import.meta.url), 'utf8')).cells -// The minute index a recorded watch of `signal` from `start` trips at, or null; `minute(i)` may rewrite a minute. -function recordedTrip({ authFailuresPerMinute: failures, completedAcceptsPerMinute: accepts }, signal, start, watchMinutes, minute = (i) => [failures[i], accepts[i]]) { - const { readMinutes, consecutiveMinutes } = PHONE_WATCH_RULES - const refused = failures.slice(start - readMinutes, start) - const background = measuredPhoneBackground({ - ok: true, - minutes: refused, - unmetered: [], - series: { ...Object.fromEntries(PHONE_RULE_SIGNAL_NAMES.map((name) => [name, refused])), completedAccepts: accepts.slice(start - readMinutes, start) } - }) - const line = phoneShareLine(background.share[signal]) - let run = 0 - for (let index = start; index < start + watchMinutes; index += 1) { - run = phoneMinuteOver(signal, ...minute(index), line) ? run + 1 : 0 - if (run >= consecutiveMinutes) return index - } - return null -} - -test('phone rules: 26 recorded hours of c7, c28 and c29 trip no attempt rule in any 60- or 300-minute watch', () => { - assert.deepEqual(Object.keys(recordedPhoneMinutes), ['production-gce-c7', 'production-gce-c28', 'production-gce-c29']) - for (const [cell, minutes] of Object.entries(recordedPhoneMinutes)) { - for (const watchMinutes of [60, 300]) { - for (let start = 10; start + watchMinutes <= minutes.authFailuresPerMinute.length; start += 5) { - assert.equal(recordedTrip(minutes, 'credentialRefusals', start, watchMinutes), null, `${cell}: a ${watchMinutes}-min watch from minute ${start} trips`) - } - } - } -}) - -// From minute 30 of each recorded hour, a broken path; it must trip by its third minute (sooner -// when the minute before was already over) wherever those minutes carry 4 attempts each. -const BROKEN_PATHS = { - 'two refusals per accept': ['credentialRefusals', (failed, accepted) => [Math.max(2 * accepted, 4), accepted]], - 'every accept errors on the database, phones give up after one try': ['acceptDatabaseErrors', (failed, accepted) => [accepted, 0]], - // Only trips are tested: the auth-failure stand-in cannot measure confirm false trips. - 'every confirm fails': ['confirmFailures', (failed, accepted) => [accepted, accepted]] -} -test('phone rules: the same recorded hours trip on each broken path wherever its minutes are judged', () => { - for (const [name, [signal, breakMinute]] of Object.entries(BROKEN_PATHS)) { - for (const [cell, minutes] of Object.entries(recordedPhoneMinutes)) { - const { authFailuresPerMinute: failures, completedAcceptsPerMinute: accepts } = minutes - let judged = 0 - for (let start = 10; start + 60 <= failures.length; start += 5) { - const broken = (index) => (index >= start + 30 ? breakMinute(failures[index], accepts[index]) : [failures[index], accepts[index]]) - const judgedMinutes = [0, 1, 2].every((offset) => { - const [count, accepted] = broken(start + 30 + offset) - return (signal === 'confirmFailures' ? accepted : count + accepted) >= PHONE_WATCH_RULES.minAttemptsPerMinute - }) - if (!judgedMinutes) continue - judged += 1 - const at = recordedTrip(minutes, signal, start, 60, broken) - // Confirms may trip earlier: the stand-in reads as failing confirms on c28 (see above). - const earliest = signal === 'confirmFailures' ? start : start + 30 - assert.ok(at !== null && at >= earliest && at <= start + 32, `${cell} ${name}: from minute ${start + 30} trips at ${at}`) - } - // Busy cells are judged most of the day; low-volume c7 rarely reaches 4 attempts a minute. - if (cell !== 'production-gce-c7') assert.ok(judged > 150, `${cell} ${name}: ${judged} judged`) - } - } -}) diff --git a/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs b/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs deleted file mode 100644 index cdf3b51c565..00000000000 --- a/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs +++ /dev/null @@ -1,155 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' - -import { - hasTerraformRoot, - renderAttributeConditions -} from './render-workload-identity-conditions.mjs' - -// GCP rejects an attribute_condition longer than this. -const ATTRIBUTE_CONDITION_LIMIT = 4096 - -const EXPECTED_CONDITIONS = { - staging: { - relay: { - github_staging_relay_capacity: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-prove-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-recover-relay-staging-c4-image.yml@refs/heads/main')", - github_staging_relay_deploy: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-staging-cell-flags.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-power-relay-staging.yml@refs/heads/main')", - github_relay_asia_topology: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-asia-topology.yml@refs/heads/main'", - github_relay_asia_proof: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-prove-relay-asia-staging.yml@refs/heads/main'", - }, - // The relay root creates this provider only in production, so staging has exactly one - // definition and it lives here. - apps: { - github: - "assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-auth-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/load-skill-finalization-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/recover-skill-object-staging.yml@refs/heads/main')", - }, - }, - production: { - relay: { - github_push: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main'", - - github: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-cell-flags.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-reserve-watch.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))", - github_monitor: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production-job.yml@refs/heads/main'", - github_fence: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main'", - github_production_relay_capacity: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main'))", - github_relay_asia_topology: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-asia-topology.yml@refs/heads/main'", - }, - apps: { - github_production_app_deploy: - "assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-auth-production.yml@refs/heads/main')", - }, - }, -} - -// The one repository each root trusts, with the workflow-ref head it contributes. The relay root -// moved to the public repository, where the workflow files carry the `cloud-` prefix; the apps -// root still deploys from the private one. -const ROOT_REPOSITORIES = { - relay: { - claims: - "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420'", - workflowHead: 'stablyai/orca/.github/workflows/cloud-' - }, - apps: { - claims: - "assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420'", - workflowHead: 'stablyai/orca-cloud/.github/workflows/' - } -} - -// [root, provider, condition] for every provider the environment creates, across all roots. -async function flatten(environment) { - const rendered = await renderAttributeConditions(environment) - return Object.entries(rendered).flatMap(([root, providers]) => - Object.entries(providers).map(([provider, condition]) => [root, provider, condition]) - ) -} - -// Only roots whose directory ships can be rendered; the apps root stays in the private -// repository, so its expectations sit above unused until that directory is present. -const expectedRoots = (environment) => - Object.fromEntries( - Object.entries(EXPECTED_CONDITIONS[environment]).filter(([root]) => hasTerraformRoot(root)) - ) - -for (const environment of Object.keys(EXPECTED_CONDITIONS)) { - const roots = expectedRoots(environment) - test(`${environment} renders the exact reviewed attribute conditions`, async () => { - const rendered = await renderAttributeConditions(environment) - assert.deepEqual(Object.keys(rendered).sort(), Object.keys(roots).sort()) - for (const [root, providers] of Object.entries(roots)) { - assert.deepEqual(Object.keys(rendered[root]).sort(), Object.keys(providers).sort(), root) - for (const [provider, condition] of Object.entries(providers)) { - assert.equal(rendered[root][provider], condition, `${environment} ${root} ${provider}`) - } - } - }) - - test(`${environment} attribute conditions stay under the GCP length limit`, async () => { - for (const [root, provider, condition] of await flatten(environment)) { - assert.ok( - condition.length < ATTRIBUTE_CONDITION_LIMIT, - `${environment} ${root} ${provider} is ${condition.length} chars` - ) - } - }) - - test(`${environment} pins repository, branch, and environment on every provider`, async () => { - for (const [root, provider, condition] of await flatten(environment)) { - for (const pin of [ - ROOT_REPOSITORIES[root].claims, - "assertion.ref == 'refs/heads/main'", - `assertion.environment == '${environment}'` - ]) { - assert.ok(condition.includes(pin), `${environment} ${root} ${provider} is missing ${pin}`) - } - assert.ok( - condition.includes('assertion.workflow_ref ==') || - condition.includes('assertion.job_workflow_ref =='), - `${environment} ${root} ${provider} names no workflow` - ) - } - }) - - // A prefix or suffix match would turn each allowlist into a namespace grant. - test(`${environment} attribute conditions compare workflows only by equality`, async () => { - for (const [root, provider, condition] of await flatten(environment)) { - assert.doesNotMatch( - condition, - /startsWith|endsWith|matches|in \[/, - `${environment} ${root} ${provider}` - ) - } - }) -} - -// Why: the cutover left one arm per relay provider. A leftover `stablyai/orca-cloud` claim or -// workflow ref would keep trusting a repository whose relay workflows are retired, and an unprefixed -// ref would name a file the public repository does not have. -for (const environment of Object.keys(EXPECTED_CONDITIONS)) { - test(`${environment} admits only the public repository through every relay provider`, async () => { - const { claims, workflowHead } = ROOT_REPOSITORIES.relay - const rendered = await renderAttributeConditions(environment) - for (const [provider, condition] of Object.entries(rendered.relay)) { - assert.ok(condition.startsWith(`${claims} && `), `${provider} does not lead with the claims`) - assert.doesNotMatch(condition, /stablyai\/orca-cloud|1273841466/, `${provider} keeps an old arm`) - const refs = [...condition.matchAll(/(?:job_)?workflow_ref == '([^']+)'/g)].map( - (match) => match[1] - ) - assert.ok(refs.length > 0, `${provider} names no workflow`) - for (const ref of refs) { - assert.ok(ref.startsWith(workflowHead), `${provider} names a stray ref ${ref}`) - } - } - }) -} diff --git a/cloud/docs/orca-relay-capacity-testing.md b/cloud/docs/orca-relay-capacity-testing.md deleted file mode 100644 index 73c0e5c6e5f..00000000000 --- a/cloud/docs/orca-relay-capacity-testing.md +++ /dev/null @@ -1,259 +0,0 @@ -# Orca Relay capacity testing - -This harness covers two different launch gates. The deterministic model proves that phase spreading produces the required aggregate heartbeat and auth-refresh rates without a synchronized cliff. The control harness opens real WebSockets, completes the host-key challenge, answers heartbeats, refreshes authorization, and reconnects with full jitter. - -Neither mode sends phone payloads or terminal content. Reports contain aggregate counts only. Access tokens and signing keys must be supplied through the documented secret paths and are never printed by the harness. - -## Deterministic 4k/10k model - -Run: - -```sh -pnpm load:relay:model -``` - -The default profiles are 4,000 and 10,000 standing controls over 15 modeled minutes. The command fails unless: - -- 15-second heartbeats produce approximately 267 and 667 pings per second; -- uniformly distributed 180–240-second token refreshes produce approximately 19 and 48 exchanges per second; -- one-second heartbeat and refresh bins remain inside the reviewed burst bounds. - -This is a schedule gate, not evidence that a cell can hold those connections. - -## Real control load - -Use a relay-scoped token source in one of two ways: - -1. Set `ORCA_RELAY_LOAD_ACCESS_TOKEN` and pass `--auth-origin`. Every control and refresh then uses the real auth-plane relay-token exchange. -2. Pass `--signing-key-file` containing the environment's auth signing key. This operator-only mode isolates relay capacity from auth capacity. Prefer memory-backed process substitution directly with `node`; `pnpm` may close that file descriptor. - -Never put either credential in command-line arguments, URLs, shell history, or reports. - -For staging, keep the signing key process-local and out of the filesystem: - -```sh -node dev/scripts/load-relay-controls.mjs \ - --director-origin https://relay-staging.onorca.dev \ - --auth-origin https://auth-staging.onorca.dev \ - --signing-key-file <(gcloud secrets versions access latest \ - --secret=orca-cloud-auth-signing-key \ - --project=onorca-cloud-staging) \ - --controls 840 \ - --ramp-seconds 210 \ - --duration-seconds 900 -``` - -Against a local or legacy combined service only: - -```sh -pnpm load:relay:controls -- \ - --target-origin http://127.0.0.1:8080 \ - --auth-origin http://127.0.0.1:8081 \ - --signing-key-file "$KEY_FILE" \ - --controls 800 \ - --ramp-seconds 210 \ - --duration-seconds 900 -``` - -Against the stable director and stamped cells: - -```sh -ORCA_RELAY_LOAD_ACCESS_TOKEN="$ACCESS_TOKEN" pnpm load:relay:controls -- \ - --director-origin https://relay-staging.onorca.dev \ - --auth-origin https://auth-staging.onorca.dev \ - --controls 800 \ - --ramp-seconds 210 \ - --duration-seconds 900 -``` - -The director form performs a real assignment for every generated relayHostId and follows the returned cell URL/epoch. Stamped GCE cells require this durable assignment; `--target-origin` cannot bypass it. Fresh identities must not exceed `hardCap - controlRebindReserve - unobservedBound` for the target cell. At the reviewed 1,000/60 policy, that placement ceiling is 840 even though the cell can hold 900 already-assigned ordinary controls. - -## Sharding - -Run one process per shard when the client machine becomes the bottleneck. Every shard receives a disjoint host/user index space: - -```sh -pnpm load:relay:controls -- \ - --director-origin https://relay-staging.onorca.dev \ - --auth-origin https://auth-staging.onorca.dev \ - --controls 1000 \ - --shard-count 4 \ - --shard-index 0 -``` - -Start shard indexes `0..3` with the same count and timing. `--controls` is per shard. Compare the combined client reports with Cloud Monitoring rather than summing only successful connection messages. - -## Required observations - -Capture these before and throughout a launch-gate run: - -- active controls, total connections, pending and active splices; -- auth successes/failures and refresh arrival rate; -- reconnect arrival distribution and close codes; -- SQL query failures and maximum interval latency; -- process queued bytes, heap, and event-loop p99; -- GCE instance CPU/memory, LB request/backend latency and 5xx metrics, plus Cloud Run director request/concurrency metrics; -- Cloud SQL CPU, connections, locks, and failover state. - -The real harness fails when fewer than 95% of requested controls become active, fewer than 95% remain active after ramp-up, or any steady-state connection, excess ramp retry, excess unexpected close, protocol, refresh, or socket error occurs. Public-load tests may set explicit small ramp-retry and close budgets; both default to zero and remain visible in the aggregate report. Use `--ramp-start-delay-ms` to desynchronize independent client shards. Failure reasons are reported only as bounded aggregate categories. `--allow-partial` exists only for diagnosing a known capacity boundary; a run using it cannot satisfy a launch gate. - -For hard-capped candidates, separately verify director placement stops at -`hardCap - controlRebindReserve - unobservedBound` and target ordinary socket -admission stops at `hardCap - controlRebindReserve`. Then overlap up to 100 -same-host replacement sockets that present valid authorization, assignment, -generation, and resume data and receive an encrypted host challenge, without -admitting unrelated controls into that reserve. The boundary mode proves -pre-activation socket headroom; activation and generation replacement remain -separate protocol tests. Above 100 concurrent replacements, verify the deployed -clients use the recorded bounded retry path. - -Use two independent staging proofs. The temporary 1,000/0 policy permits 900 -fresh assignments and exposes the exact physical boundary: - -```sh -node dev/scripts/load-relay-controls.mjs \ - --director-origin https://relay-staging.onorca.dev \ - --auth-origin https://auth-staging.onorca.dev \ - --signing-key-file <(gcloud secrets versions access latest \ - --secret=orca-cloud-auth-signing-key \ - --project=onorca-cloud-staging) \ - --controls 900 \ - --rebind-probes 100 \ - --rebind-hold-ms 4000 \ - --ramp-seconds 210 \ - --duration-seconds 900 -``` - -The command must keep all 100 replacements open for the complete hold, require -the next physical socket to receive HTTP 503, and pass the 15-minute soak. - -Then apply the final 1,000/60 policy and start a separate 840-control process. -During its explicit delay, rerun the reviewed 1,000/60 transition so the no-op -cell plan performs one exact C3 restart. The boundary checks begin only after -all 840 controls recover: - -```sh -CAPACITY_SA="$(gh variable get STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT --env staging)" -ORCA_RELAY_ADMIN_ID_TOKEN="$(gcloud auth print-identity-token \ - --impersonate-service-account="${CAPACITY_SA}" \ - --project=onorca-cloud-staging \ - --include-email \ - --audiences=https://relay-staging.onorca.dev/v1/admin/drain)" \ -node dev/scripts/load-relay-controls.mjs \ - --director-origin https://relay-staging.onorca.dev \ - --auth-origin https://auth-staging.onorca.dev \ - --signing-key-file <(gcloud secrets versions access latest \ - --secret=orca-cloud-auth-signing-key \ - --project=onorca-cloud-staging) \ - --controls 840 \ - --placement-overflow-probes 1 \ - --capacity-cell-id staging-gce-c3 \ - --capacity-hard-cap 1000 \ - --capacity-unobserved-bound 60 \ - --rebind-probes 100 \ - --allow-planned-transition-retries \ - --skip-rebind-overflow-check \ - --rebind-delay-seconds 1800 \ - --rebind-hold-ms 4000 \ - --ramp-seconds 210 \ - --duration-seconds 900 -``` - -The inline environment assignment keeps the one-hour admin token process-local; do -not export, print, or persist it. Before probing, the harness requires two advancing director -heartbeats after recovery with exact 840-connection telemetry, no pending connection -reservations, and the reviewed 1,000/60 policy. This run requires the 841st fresh -placement to receive the capacity-specific HTTP 503 response, then requires a newer -exact director heartbeat while all 840 ordinary controls remain unchanged. This proves at -least 100 replacement sockets remain available, and completes the normal soak. -The explicit planned-transition flag permits only connection retries before the -recovery gate. Expected drain closes and transition retries remain separately -counted; any missing recovery or steady-state error fails the proof. Never -persist generated host keys or resume credentials. - -## Cap transition and rollback order - -Changing the reviewed cap is a fail-closed operation. The capacity workflow -first builds and prunes a capacity-protocol-2 director rollback pair. It drains -C3, validates the saved Terraform plan, updates the director inventory, and -requires the old cell heartbeat to become stale. It then validates and applies -only C3's instance-template replacement and MIG pointer. A fresh heartbeat must -report the exact cap and unobserved bound before C3 returns to general -placement. Ordinary director deploys do not prune historical revisions. - -The staging proof uses reversible admission states. C3 moves to -`migration-only` and drains before replacement. After its fresh heartbeat, C3 -becomes the sole general cell while C2 moves temporarily to `migration-only`, -so fresh load identities can only land on C3. The restore mode first makes C2 -general as a safe fallback, verifies that C3 is healthy and non-draining, then -restores the reviewed C2/C3 general set. It never uses irreversible -`existing-only` for temporary isolation. - -The cell rollout is the forward point of no return. To restore 600, keep the -new director code active, configure 600 there first, then roll the empty cell -back to 600 and require its fresh matching heartbeat. Only after that may an -older director image be considered. Never shift director traffic to a -pre-protocol-2 image while any cell is configured above 600. - -## Soak profiles - -For the accelerated three-cycle gate, use a test deployment whose lease/rotation intervals are explicitly shortened together and run at both modeled 4k and 10k aggregate levels. Keep enough shards/cells that no client or cell exceeds its reviewed ceiling. Record listener, timer, socket, queued-byte, heap, SQL, and event-loop bounds at every cycle. - -For each real-time load-balancer canary, use exact GCE cell hosts through the public external HTTPS LB with the production 86,400-second backend timeout: - -- run a low-scale socket past the actual 86,400-second cap to prove the observed LB termination and configured-director recovery path; -- run a separate production-jitter canary spanning at least three proactive rotations before that cap; -- force at least three fixed-one GCE instance terminations and record recovery through strictly newer director assignments. - -The control harness is one input to those canaries. The full canary must also run real phone/data pairs and verify earlier-leg deadline handling, close/1006/502/503/504 recovery, subscription replay, deterministic rejection of ordinary in-flight RPCs, and idempotent credential reconciliation. These long-running canaries are public-launch gates; modest served load remains sufficient for implementation iteration. - -## Legacy recovery-wave gate - -After an isolated local or staging exercise, evaluate its aggregate report: - -```sh -pnpm load:relay:recovery-gate -- --report "$AGGREGATE_REPORT" -``` - -The evaluator has no HTTP or GCP client and rejects production project IDs, -production origins, unknown fields, malformed values, and reports larger than -1 MiB. It exits nonzero unless the report proves all numeric gates: - -- 760–840 draining desktops under 10,450–11,550 background requests/minute, - including the observed assignment-heavy mix and 8,500–10,500 assignment - `503` responses/minute; -- two targets, each capped at and observed below 600 connections, with every - desktop recovered; the report must separate physical sockets, in-flight - upgrades, pending host-data units, and their enforced sum; -- concurrent boundary probes proving every accepted phone can consume its - reserved host-data leg, control rebind remains available, established - sockets stay open, and failed upgrades release capacity exactly once; -- a 2-vCPU database, pool size 3, two total public slots, and one - resolve-priority slot; -- the production one-to-two-instance director range, Cloud Run concurrency 80, - and an observed two-instance peak during the wave; -- a separate old/new-revision rollout-overlap result that reaches four - processes and eight public operations while preserving the same resolve, - readiness, pool-wait, and database-CPU gates; -- zero migration expirations, migration aborts, retry exhaustion, readiness - failures, and non-public maintenance failures; -- one key-proven target registration per desktop, at least ten minutes on the - oldest migration lease at drain, and full-wave registration within five - minutes; -- at least 90% of baseline assignment throughput, at least 95% eligible - resolve success, and below 1% resolve overload; -- pool-wait p95 below 500 ms, pool-wait max below 5 seconds, database CPU p95 - below 70%, and database CPU max below 85%; and -- recovery within 14 minutes, one minute before the migration lease expires. - -The report must come from a controlled production-shaped run that joins client -counts with relay metrics and Cloud SQL monitoring. The gate validates evidence; -it does not generate the reconnect wave or prove that supplied observations are -authentic. Never use a hand-authored passing report as launch evidence. - -The rollout-overlap fields may come from a separate isolated sub-run. A -steady-state two-instance wave cannot substitute for the four-process overlap -created while old and new Cloud Run revisions coexist. The old side must run -the current production-equivalent shared gate with zero resolve-priority -slots; the new side must run the candidate two-public-slot scheduler with one -resolve-priority slot. diff --git a/cloud/docs/orca-relay-operations.md b/cloud/docs/orca-relay-operations.md deleted file mode 100644 index 7f1c0b3360b..00000000000 --- a/cloud/docs/orca-relay-operations.md +++ /dev/null @@ -1,692 +0,0 @@ -# Orca Relay operations runbook - -This runbook applies to the stable Cloud Run director and the production-shaped GCE cells in both environments. It does not authorize a full Terraform apply: staging and production contain unrelated drift, so inspect a saved targeted plan and its destroy count before every apply. - -## PostgreSQL statement statistics - -Relay schema startup exposes `pg_stat_statements` when the server already preloads -that collector and the schema identity can install its extension. Servers without -the collector or the required privileges continue normally. Installation does not -change preload settings, reset collected counters, or require a database restart; -concurrent startups yield to one installer. An existing extension is left in place. - -For SQL incidents, inspect bounded aggregates of `calls`, `total_exec_time`, -`shared_blks_read`, `shared_blks_dirtied`, and `wal_bytes`, scoped to the relay -database and identified query IDs. Compare counter deltas over the same interval -as fleet runtime metrics; retain the statistics reset timestamp. Do not export -query text, identities, credentials, or invoke `pg_stat_statements_reset()` during -an investigation. Treat an unavailable view as missing evidence, not zero work. - -The relay is automatically active for entitled signed-in desktops. There is no rollout flag, cohort, or user toggle. The emergency product kill switch is the auth plane refusing relay-token exchange; use cell drains only to move or terminate existing data-plane work. - -## Safety rules - -- Never put relay JWTs, access tokens, invite/resume credentials, or signing keys in URLs, shell history, logs, or reports. -- Mint the admin identity token with the exact configured audience, which is the stable director origin plus `/v1/admin/drain`, even when calling another admin path. -- A drain response contains only `recovery: resolve-director`. Never provide a recovery URL from a cell. -- Start the target revision and commit its assignment before asking the source control to drain. Keep the source revision alive while the desktop registers the verified target; existing source splices, installs, and confirmations stay origin-owned until their leases settle or grace ends. -- Treat `4401`, `4404`, and `4429` as endpoint-scoped. `4409`, `4503`, transport failure, `1006`, `503`, and `504` recover through the configured director. -- Resume recovery uses bounded `POST /v1/resolve`; an unexpired invite may use only the director compatibility WebSocket. -- Public assignment and resume recovery share two public database slots. A bounded - resolve waiter takes the next slot ahead of new assignments, leaving the third - director-pool connection for non-public work. -- Stop if a plan or command includes an unrelated service, database, bucket, destroy, or a `REPLACE_*` value. - -Set environment-specific values without printing the resulting token: - -```sh -export PROJECT_ID=onorca-cloud-staging -export REGION=us-central1 -export DIRECTOR_ORIGIN=https://relay-staging.onorca.dev -export DEPLOY_SERVICE_ACCOUNT=orca-cloud-staging-gha-deploy@onorca-cloud-staging.iam.gserviceaccount.com -export ADMIN_AUDIENCE="${DIRECTOR_ORIGIN}/v1/admin/drain" -ADMIN_TOKEN="$(gcloud auth print-identity-token \ - --impersonate-service-account="${DEPLOY_SERVICE_ACCOUNT}" \ - --audiences="${ADMIN_AUDIENCE}")" -``` - -Unset `ADMIN_TOKEN` when the operation finishes. - -## Preflight and stop conditions - -Before any rebalance, evacuation, deploy, or game day: - -1. Confirm `/health` on the stable director and every native target service URL. -2. Confirm the target cell is enabled, has a fresh heartbeat, and has reservation headroom for source units plus its migration lease. -3. Check active controls, splices, pending splices, queued bytes, auth failures, reconnects, SQL failures/latency, heap, and event-loop delay. -4. Confirm Cloud SQL is healthy and the auth JWKS endpoint is serving the expected current and rotation keys. -5. Start a timestamped operator log containing only resource names, epochs, aggregate counts, and response codes. - -Stop and roll back or leave the source intact if the target cannot register, source-owned operations do not drain, SQL errors rise, queue/heap alerts fire, or reconnect arrivals form a cliff. - -## Staging sleep and wake - -The `Power Relay Staging` workflow may stop staging outside internal test windows; it never targets -the production project. Its nightly 09:00 UTC sleep is guarded and fails safely when any cell -reports an active lease, request unit, migration, or observed connection. A successful sleep -disables cell admission, proves quiescence again, makes the active auth/director Cloud Run revisions -scale-to-zero compatible, scales every staging MIG to zero, and finally stops Cloud SQL. -After selector generation 1, scheduled sleep fails closed because waking would -require prohibited re-enablement. Selector-era wake restores every retained -cell process and verifies admission without rewriting it. - -Before staging testing, dispatch `wake` with the default `configured` selection. It starts Cloud -SQL, c1, and c2, waits for health, readiness, and authenticated heartbeats, and restores the -Terraform-declared admission cells; disabled c3 stays off. Before any staging Terraform apply or -candidate workflow, wake `all` cells. The local apply command deliberately rejects a stopped or -partially awake staging topology. - -Use `status` for a read-only view of SQL activation, MIG target sizes, and the minimum-instance -setting of the active Cloud Run revisions. If a sleep fails after admission was disabled, the -workflow attempts to restore previously enabled cells and leaves SQL and MIGs running. If status -shows SQL stopped while a MIG is nonzero, do not retry sleep: wake staging and inspect the partial -state first. - -## Legacy staging tagged-revision deployment - -The blue/green script remains for historical protocol fixtures, but live staging now uses GCE cells. This procedure is staging-only and must not be used to replace a production GCE cell. For each stamped cell the script: - -1. Adds a drain tag to the exact current 100%-traffic revision and queries its actual tag URL. -2. Registers the candidate cell as disabled, deploys its tagged revision with no traffic and the tag as its cryptographically bound public origin, queries that tag URL, and passes `/health`. -3. Clones the old image into a no-traffic previous-tag keeper with the old cell ID and its tag as the bound public origin. This is the safe return target for recently dormant assignments; the exact original revision remains the source of live controls. -4. Atomically records the keeper URL while disabling new source assignments, enables the candidate, starts bounded aggregate evacuation batches, and drains the exact original revision so desktops resolve the committed target. -5. Waits for every migration lease to report a key-proven target registration, shifts service traffic only after the verified count matches, then completes migrations only as source activity reaches zero. - -The workflow obtains Google identity tokens in memory and emits aggregate counts only. It retains drain/previous tags and disabled source-cell rows so live origin-owned work can finish and recently dormant assignments remain routable until normal dormant-TTL reassignment. Do not delete old tags while any assignment still resolves to their cell IDs. - -If the workflow stops before source disable, leave the disabled no-traffic candidate in place for inspection. If it stops after migrations begin, do not edit epochs or reservations and do not blindly re-enable the source. Follow the rollback rules below; a migration that never registered a target automatically returns to the source after its lease expires with another newer epoch. - -## Production GCE candidate deployment - -Production publishes an immutable relay image first, then declares a distinct disabled candidate cell in a reviewed targeted Terraform change. The candidate must have a new cell ID, exact hostname, route, backend service, fixed-one `RECREATE` MIG, and durable incarnation. Never replace the backend behind an existing cell origin. - -Run `Deploy Relay Production Candidate` in `preflight` mode before any mutation. The workflow verifies exact-host TLS, `/health`, dependency-backed `/ready`, a fresh authenticated heartbeat, the runtime service account, served digest, fixed-one topology, private-only networking, and survivor request-unit headroom. All production cells remain disabled until an explicit go-live decision. - -After launch, `execute` additionally requires the exact `EVACUATE` confirmation. It enables the proven candidate, disables new source assignment, and performs bounded target-first evacuation. Preserve the source route, backend, and MIG until every source-owned splice, install, confirmation, assignment, reservation, and migration lease is drained. Remove those resources only in a later reviewed Terraform change. - -## Production multi-target evacuation - -Use `Deploy Relay Production Multi-Target` when one candidate cannot hold the -source below the reviewed connection ceiling. Targets are sorted by cell ID, -and active assignments are apportioned deterministically before any mutation. -The workflow rejects a plan when projected or observed target connections -reach 600, request-unit reservations do not fit, or target topology and served -digests do not match Terraform. Current targets expose counts through their -authenticated runtime status. A legacy source without that field must have a -runtime-metrics log no older than 90 seconds; missing or stale telemetry is a -hard stop. - -Preflight separately proves that every source assignment can reserve one target -control. It subtracts enforced units and pending reservations from each target's -normal-admission pause instead of using the physical hard cap. Missing, stale, or -internally inconsistent connection-capacity telemetry is a hard stop. - -Only new-image cells explicitly configured with -`ORCA_RELAY_CELL_CONNECTION_HARD_CAP=600` and -`ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=` use the hard -cap. Configure the same values in the director's cell inventory. Existing -cells without both values retain their current 900-unit admission behavior. -Never add the limit to an old-image cell. - -For a limited cell, authenticated status and heartbeat report physical -connections, in-flight upgrades, pending host-data units, and their enforced -sum. The director admits one new control only while: - -```text -enforced connection units - + durable pending-control leases - + configured unobserved-arrival bound - < 600 - 100 control-rebind reserve -``` - -Missing, stale, wrong-incarnation, mismatched-cap, or internally inconsistent -telemetry makes that cell ineligible. The unobserved bound is the isolated -test's maximum arrivals over one heartbeat plus reaction latency and maximum -pre-auth/in-flight units, with 20% safety margin. Record p99 separately as an -SLO, not the safety bound. - -The target reserves two units for each accepted phone: the phone socket and -its future host-data socket. The second leg transfers that reservation instead -of competing for capacity. Exactly 100 units are exclusive to same-host -control rotation/rebind overlap; first controls, phones, and unreserved data -sockets stop at the target's exact 500-unit ordinary limit and cannot borrow -them. The director stops placement earlier at -`500 - unobserved-arrival bound`. Authenticated runtime status publishes both -thresholds so rollout automation can gate their exact values. At the hard ceiling the -target returns HTTP 503 for new work and leaves established sockets open. A -503 causes clients to consult the configured director, but a healthy assignment -normally resolves to the same cell; it is backpressure, not an automatic -migration. Rebind rejection within the tested 100-concurrent replacement -bound, or a new-phone rejection rate above the load-proven threshold, stops -rollout; excess simultaneous rebinds use the tested bounded retry path. - -Run `preflight` first with every target admission-disabled. `execute` requires -`EVACUATE_MULTI`, disables the source, enables targets serially, publishes each -target's fixed quota serially, and then rechecks all targets. It refuses -`/drain` unless the oldest active migration has at least ten minutes remaining. - -Any `/drain` attempt is rollback-unsafe because a lost response cannot prove -the old source did not accept it. Before that attempt, the workflow may restore -source admission only when no target registered. After it, keep the source -disabled and use `recover-forward`; never restore its old assignment epochs. -If the durable attempt is still exactly `prepared`, recover-forward may acquire -the original send permit once and send its original trace and grace. A -`send-may-have-started` attempt without an application receipt remains -ambiguous and must freeze; it is never resent. -For immutable legacy c3, the director durably records the exact cell -incarnation before the workflow sends `{v:1,graceMs:120000}`. A lost response -is never retried before the grace deadline plus 30 seconds. If authenticated -legacy aggregate counts still show active controls/splices then, -`recover-forward` may durably record and send at most one -`{v:1,graceMs:0}` request. It does not add unsupported fields to c3. -Before recover-forward, run the 15-minute production monitor with its explicit -`recover-forward` migration policy and exact existing-only source cell. That -signed policy permits only already registered migrations from that source whose -target control is inactive, which the recovery drain is intended to reconnect. -Recovery registers a conservative capacity snapshot and catches up newly active -source assignments for at most five passes; it still requires exact zero before drain. -Ordinary execute evidence remains strict, and -blocked or expired/unregistered migrations, inactive target runtimes, selector -drift, stale telemetry, and all capacity and database thresholds still stop -both the gate and immediate live recheck. - -If the old source has zero controls, splices, pending splices, activity leases, -and reserved units but closing transports prevent completion, run -`fence-source` with `FENCE_SOURCE`. It additionally requires every migration to -have a key-proven active target and zero source activity. The workflow resizes -the fixed-one source MIG to zero, proves no instance remains and its heartbeat -is stale, records a short-lived exact-incarnation fence attestation, then -completes through the shared strict database guard. A new heartbeat invalidates -the attestation. -If the resize response or workflow is interrupted, rerun the same fence mode; -an already-zero source resumes only after the retained topology and database -guards pass. - -Terraform intentionally ignores only operational MIG `target_size` drift, so a -later apply cannot recreate a fenced source. All other MIG topology remains -managed and preflighted. Do not resize a relay MIG directly: the guarded -workflow is the only supported zero-size path. - -## Dormant return and dormant rebalance - -A normal assignment may move only after every activity count is zero and the bounded dormant TTL has elapsed. A returning desktop with a stale epoch resolves/assigns through the director and accepts only the newer authenticated epoch. - -For an explicitly selected dormant assignment: - -```sh -curl --fail-with-body --request POST "${DIRECTOR_ORIGIN}/v1/admin/rebalance-dormant" \ - --header "Authorization: Bearer ${ADMIN_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "{\"v\":1,\"userId\":\"${USER_ID}\",\"relayHostId\":\"${RELAY_HOST_ID}\",\"targetCellId\":\"${TARGET_CELL_ID}\"}" -``` - -`assignment_active` is a stop result, not permission to clear counters. Investigate leases and wait; do not manually rewrite assignment rows. - -Validate that source reservations fall, target reservations rise by exactly one pending control, and the next returning desktop registers the returned epoch on the target. - -## Cell admission selector - -The director has three durable admission states: - -- `existing-only` preserves current assignments and sticky resolution but receives no ordinary, dormant, or dead-cell placement. -- `migration-only` receives only explicitly published evacuation assignments. -- `general` receives ordinary placement and may also receive explicit evacuation assignments. - -Before the selector boundary, the legacy `enabled` admin field remains compatible: -`false` means `existing-only` and `true` means `general`. Use the explicit -`state` field to stage migration-only targets. Once selector generation 1 is -committed, direct cell-state and cell-config admission changes fail closed. -Every later change must use the selector CAS. - -Deploy the selector-compatible director before cutover. Blue/green deployment -creates a separate `selector-rollback` revision at minimum instances zero, -promotes a second compatible revision, and retains older revisions through the -stabilization window. With a floor above one, promotion first moves 20% of -traffic and waits (up to six minutes) until every floor instance has served a -request, because Cloud Run routes new floor instances only ~100 s after they -start; a timeout returns all traffic to the previous serving revision. Each -revision carries its own instance maximum. A service-level cap would be split by -traffic share (a 20% candidate under a cap of 5 gets one instance), so the deploy -resets any service cap below two full revisions to Cloud Run's default first. - -If the deploy job is cancelled or loses its runner during that wait, the 20/80 -split stays in place, and both the next deploy and the rehome workflow refuse a -director without one 100% revision. Restore it with -`gcloud run services update-traffic orca-cloud-relay --project onorca-cloud --region us-central1 --to-revisions==100` -(the revision that held 100% before the deploy). A timed-out deploy restores -that itself but removes every tag, including `selector-rollback`, so the rehome -workflow's rollback-digest check refuses (`rollback tag missing`) until the next -successful director deploy. Cutover removes every older revision and refuses to -proceed unless only the compatible active and rollback revisions remain. - -First inspect the current generation and exact membership: - -```sh -curl --fail-with-body --request POST \ - "${DIRECTOR_ORIGIN}/v1/admin/admission-selector/status" \ - --header "Authorization: Bearer ${ADMIN_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data '{"v":1}' -``` - -Apply one exact, complete partition of every configured cell: - -```sh -curl --fail-with-body --request POST \ - "${DIRECTOR_ORIGIN}/v1/admin/admission-selector/apply" \ - --header "Authorization: Bearer ${ADMIN_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data @selector-request.json -``` - -`selector-request.json` must contain `v:1`, a unique 8–128 character -`attemptId`, the inspected `expectedGeneration`, and `membership` arrays named -`existingOnly`, `migrationOnly`, and `general`. A cell must appear exactly -once. For generation zero it must also contain `expectedMembershipSha256`: the -lowercase SHA-256 of the inspected membership's canonical UTF-8 JSON, with keys -in `existingOnly`, `migrationOnly`, `general` order and each array sorted. Hash -the current inspected membership, not the desired membership. Prefer the -audited `cutover-admission` operation, which derives this fingerprint directly -from its inspection. The transaction updates the compatibility boolean and all -tri-state membership atomically. - -If the apply response is lost or ambiguous, do not create a new attempt. -Inspect status with the same `attemptId`. Continue only when it reports either -`committed` with the exact intended generation and membership or `unchanged` -with the exact previous generation and membership. `diverged` is a freeze and -review result. Reapplying the identical attempt is idempotent. - -After the boundary is active, register new empty migration targets only through -`POST /v1/admin/admission-selector/add-migration-cells`. The request contains -one durable attempt ID, the exact current generation, and the complete new cell -configs, including canonical origins and reviewed connection limits. The -operation requires every cell and origin to be new, inserts inventory, -admission, and limit rows, extends migration-only membership, and advances the -selector exactly once in one transaction. Replay the same request after an -ambiguous response; changing its config or reusing its attempt ID fails closed. -One cell may be registered alone; evacuation and supersession still require -their reviewed multi-cell target sets. - -Use `retire-migration-cell` to stop exactly one migration-only cell from -receiving new migrations before fencing it. Supply only that cell as the target, -an exact durable selector attempt ID, and `RETIRE_MIGRATION_CELL`. The mode -requires the cell to be migration-only and applies one generation-bound CAS -that moves it to existing-only while preserving every other membership. - -For production, publish and blue/green deploy the selector-version-2 director -first. Add the disabled fixed-one Terraform cells, review a targeted plan with -zero destroys and replacements, and apply only their templates, MIGs, backend -services, and exact URL-map routes. Then run `Deploy Relay Production -Multi-Target` in `add-migration-cells` mode with `ADD_MIGRATION_CELLS`, a stable -attempt ID, and the reviewed unobserved bound. Wait for exact TLS, health, -readiness, digest, topology, and heartbeat evidence before including the new -generation in a preflight or monitoring gate. - -After generation 1, an `existing-only` cell can never return to -`migration-only` or `general`. A proven migration-only cell may transition to -general as additive capacity through a later CAS. Compatible director -restarts verify this boundary and do not restore legacy admission. - -Production cutover uses `Deploy Relay Production Multi-Target` in -`cutover-admission` mode with `CUTOVER_SELECTOR`. Supply the complete proven -general pool and migration-only target set. Every other Terraform cell becomes -existing-only in the single CAS. A lost response is inspected by attempt ID; -only the exact committed result is accepted. - -Also supply the exact `unobserved-connection-bound` produced by the passing -incident load gate. Before pruning old director revisions, the workflow proves -every proposed general or migration-only cell is a healthy fixed-one Terraform -deployment serving its pinned digest with a fresh heartbeat. Each must expose -the integrated runtime connection-capacity contract: hard cap 600, control -rebind reserve 100, the supplied unobserved bound, and the corresponding normal -admission pause. Cutover also requires fewer than 45 pre-auth connections and -enough pause headroom for current enforced units plus the director's durable -pending control reservations. - -The cutover workflow depends on the hard-cap release exposing -`connectionCapacity` from cell runtime status and -`pendingControlReservations` from director cell status. Missing or mismatched -evidence is a hard stop; do not weaken the gate to deploy the selector branch -alone. - -After cutover, candidate and multi-target workflows require existing-only -sources and migration-only targets. Pre-drain failure preserves that selector -membership and never restores legacy general admission. - -## Hot-cell rebalance - -1. Stop sending new assignments to a demonstrably unhealthy cell without changing already-issued client URLs: - -```sh -curl --fail-with-body --request POST "${DIRECTOR_ORIGIN}/v1/admin/cell-state" \ - --header "Authorization: Bearer ${ADMIN_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "{\"v\":1,\"cellId\":\"${SOURCE_CELL_ID}\",\"enabled\":false}" -``` - -The disabled state survives director restart. Before selector generation 1, -explicitly re-enable the cell through the same endpoint only after health and -capacity checks pass. After generation 1, use a reviewed selector CAS and -never re-enable a legacy existing-only cell. -2. Move fully dormant assignments first with `rebalance-dormant`. -3. Recompute source/target request-unit headroom. One control costs one unit; one splice costs two; invite/install/confirmation/migration leases also reserve their contract units. -4. Move active assignments one at a time or in a bounded batch using the target-first evacuation below. -5. Pause whenever target total connections approach 800, queued bytes exceed 48 MiB, or any runtime/SQL alert fires. - -Do not infer required cells from DAU or phones. Use measured standing controls, splice/reservation distribution, startup/login bursts, and explicit safety margin. - -## Target-first active evacuation - -Start a durable migration on the director: - -```sh -curl --fail-with-body --request POST "${DIRECTOR_ORIGIN}/v1/admin/evacuate" \ - --header "Authorization: Bearer ${ADMIN_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "{\"v\":1,\"userId\":\"${USER_ID}\",\"relayHostId\":\"${RELAY_HOST_ID}\",\"targetCellId\":\"${TARGET_CELL_ID}\"}" -``` - -Record the returned `sourceCellId`, `targetCellId`, and strictly newer `assignmentEpoch`. The transaction reserves the target before publishing the epoch. - -Ask the source control to resolve the committed assignment while the source revision remains alive: - -```sh -curl --fail-with-body --request POST "${SOURCE_NATIVE_OR_TAG_ORIGIN}/v1/admin/drain" \ - --header "Authorization: Bearer ${ADMIN_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data '{"v":1,"graceMs":120000}' -``` - -Wait for the desktop to register a key-proven target control. Do not proceed on the basis of a socket open alone. Confirm the migration's target registration and that source-owned splices/install/confirmation work remains on the source control. - -Complete each migration only after source activity reaches zero: - -```sh -curl --fail-with-body --request POST "${DIRECTOR_ORIGIN}/v1/admin/migration-complete" \ - --header "Authorization: Bearer ${ADMIN_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "{\"v\":1,\"userId\":\"${USER_ID}\",\"relayHostId\":\"${RELAY_HOST_ID}\",\"assignmentEpoch\":${ASSIGNMENT_EPOCH}}" -``` - -`migration_target_not_registered`, `migration_source_still_active`, and `migration_target_not_active` are safety stops. Do not bypass them. - -### Dead-source completion - -Use only `Deploy Relay Production Multi-Target` in `fence-source` mode. The -workflow is the authority that proves the source MIG is durably zero, no -instance remains, retained topology matches Terraform, and the exact -incarnation heartbeat is stale before creating the short-lived database -attestation. There is no per-assignment dead-source operator endpoint. - -The aggregate operation is idempotent. A missing or expired attestation, new -source heartbeat, source activity, stale target heartbeat, inactive target -control, changed admission, request-unit drift, or aggregate reservation -mismatch is a stop result. - -### Registered-target supersession - -Use `Deploy Relay Production Multi-Target` in `supersede-target` mode with -`SUPERSEDE_TARGET`. The workflow requires the original source to remain -disabled, proves the failed target's retained topology, disabled admission, -MIG desired size zero, zero instances, and stale exact-incarnation heartbeat, -then records a short-lived fence attestation. It proves replacement health and -conservative connection headroom before enabling the replacement and invoking -the aggregate database operation. Rerun the same mode after a lost resize or -workflow response; it resumes from durable GCE and database state. - -Target supersession runs only through the IAM-authenticated private broker. The -requester cannot access Terraform state, Compute mutations, or director -mutation routes. The broker permits one configured cell triple, binds its -gitless checkout to the immutable image commit, and holds a durable GCS lease -through the exact-plan operation. It retains that lease after a failure so only -the same request can resume before expiry. This repair does not consume or -weaken the normal 15-minute source-recovery gate; run that unchanged gate only -after failed-target contention is gone. - -The transaction removes only the proven-dead target's leases, preserves -original-source activity, reserves replacement capacity, publishes exactly -`assignmentEpoch + 1`, and retires the old migration atomically. A repeated -identical request returns the same successor. A different concurrent -replacement, live current target, unavailable replacement, unexpected lease -topology, or accounting mismatch fails closed. - -## Dead cell - -Do not call an unreachable cell or trust it to supply a target. For each affected assignment: - -1. Verify another cell has capacity. -2. Start the director evacuation to publish a newer target epoch. -3. Let desktop `1006`/transport recovery resolve the configured director and register the target. -4. Phones resolve through the director; unexpired invites use the director compatibility route. -5. Complete only after expired source activity leases release and the target control is active. - -If the dead cell returns, leave its stale epoch fenced. Never decrement an epoch or restore an old assignment row. - -## Global admission or memory pressure - -At connection headroom, queue, heap, or event-loop alerts: - -1. Preserve established controls/splices and reject new pre-auth work through the existing admission reserves. -2. Identify whether pressure is source-local, cell-wide, auth/JWKS, SQL, or a synchronized reconnect event using aggregate metrics only. -3. Move dormant work first. Evacuate active work only when a target has measured request and memory headroom. -4. If every cell is unsafe, make the auth plane refuse new relay-token exchanges, then issue authenticated drains with full-jitter client recovery. This is the kill switch; do not add a flag. - -Never raise runtime admission or queued-byte limits during an incident without a load result proving memory headroom. - -## Auth outage and host token keys - -Cells and directors verify host relay tokens against auth's JWKS. When a refetch fails they keep verifying against the last set auth served for up to 30 minutes (`RELAY_JWKS_MAX_STALE_MS`), so a still-valid token is not refused while auth is down. A key the served set never had is always refused, and the first successful fetch replaces the set, so a rotation takes effect as soon as a process can reach auth. - -Relay tokens last 5 minutes, so an auth outage longer than about 6 minutes still disconnects every desktop: neither side can refresh. The stale set only keeps the tokens minted before the outage working until they expire. - -Emergency purge, after rotating a leaked signing key while some processes cannot reach auth: restart them (cells by the reviewed roll, the director by a blue/green deploy). A restarted process holds no set and refuses every host token until its first fetch from auth succeeds, so it fails closed. Do this only for a key compromise; for an ordinary outage, waiting is correct. - -## Drain and SIGTERM - -Planned drain uses the authenticated admin endpoint and a reviewed grace that permits target-first registration and origin-owned operations to settle. Phones and desktops always recover through the configured director. - -SIGTERM is a degraded path. It may advertise zero grace, rejects new work, closes phone/data pairs together with `4503`, and cannot guarantee target registration first. Game-day this separately from planned drain. - -After either path, verify close/`1006`/`504` observations, bounded reconnect arrival, subscription replay, deterministic ordinary in-flight rejection, and idempotent install/confirmation reconciliation. - -## Director Cloud SQL transport (proxy sidecar) - -`relay_director_cloud_sql_proxy_sidecar` moves the director off Cloud Run's built-in `/cloudsql` -socket, whose `connect()` blocks the event loop for the whole upstream dial (10 s per attempt while -Cloud SQL starts or fails over), onto a Cloud SQL Auth Proxy sidecar on `127.0.0.1:5432`. Staging -runs it since 2026-10-11 (serving `orca-cloud-relay-staging-00177-qur`); production keeps the socket -until it is enabled through this sequence. - -1. In production, only after a reviewed `production.tfvars` PR sets the variable (and - `relay_event_loop_stall_alert`) and a staging Cloud SQL stop/start drill on the sidecar has passed. - Plan with exactly these targets and review it: `google_cloud_run_v2_service.relay`, the three - `relay_director_database_url` resources (secret, version, accessor), - `google_logging_metric.relay_event_loop_stall` and - `google_monitoring_alert_policy.relay_event_loop_stall`. Expect `5 to add, 1 to change`. The - director change also carries the template drift already on `main` (`ORCA_RELAY_IMAGE_DIGEST` - dropped, which the deploy restores; reordered or default-valued environment entries); stop on - anything else, such as concurrency, timeout or image. -2. Apply the saved plan; in production, only while holding the production Cloud SQL rollout lease - (`terraform/state/cloud-sql-rollout/production.lock` in `onorca-cloud-terraform-state`, the - `cloud-sql-rollout-lease` action). **The metric must exist before its alert policy**: a new log-based metric - takes minutes to become visible, and the first apply can fail on the policy with `404 Cannot - find metric(s)`. Re-plan and apply the policy alone a few minutes later, or apply the metric in - an earlier run. Staging hit this on 2026-10-11. -3. The apply creates a 0% revision (the service declares no traffic block). Cloud Run does **not** - start it: it reads `Ready` with reason `Retired` and logs nothing, so it proves the template - parses, not that the sidecar boots. Record its name; it is `status.latestCreatedRevisionName`. -4. Dispatch the director deploy workflow with the **serving** image digest, - `shape-baseline-revision=` and `confirmation=ADOPT_RELAY_DIRECTOR_TERRAFORM_SHAPE`. - The deploy compares its candidates with that revision's shape instead of the serving one, and - refuses an image change, an identity bootstrap, a topology change or pruning in the same run. - The first real sidecar boot is the deploy's `selector-rollback` candidate, health-checked before - any traffic moves; the proxy logs `Listening on 127.0.0.1:5432` and its startup and liveness - probes pass. -5. Verify the serving revision has `relay` then `cloud-sql-proxy`, `DATABASE_URL` from - `orca-cloud-relay-director-database-url`, `ORCA_RELAY_IMAGE_DIGEST` restored, the warm floor - kept, and no `run.googleapis.com/cloudsql-instances` annotation. - -Rollback: the pre-sidecar revision survives (shape adoption refuses pruning) and keeps its own -`/cloudsql` connection and socket secret, so `gcloud run services update-traffic ---to-revisions==100` restores it. No workflow owns that move; in production it -needs approval and the rollout lease. The next deploy then refuses until a reviewed apply with the -variable `false` and a deploy that adopts that apply's revision. - -## Rollback - -Before target registration, an expired migration automatically releases target reservations and returns the assignment to the source with another strictly newer epoch. Verify: - -- the migration is marked aborted; -- target pending-control and migration leases are gone; -- source reservation is restored exactly once; -- resolved epoch is newer than both the original and failed target epochs. - -Once a target control is registered, do not force the pre-registration rollback. Keep the source control and operations alive, repair the target, or perform a new target-first evacuation to a healthy cell. Never edit epochs or reservation counters manually. - -After a deployment traffic shift, preserve the old revision/tag until metrics and live reconnect checks pass. If the new revision is unhealthy, shift traffic back only while old controls are still valid, then issue a strictly newer director migration rather than reusing a prior epoch. - -## Regional rehoming - -Idle regional correction requires both source and target cells to advertise -`regionalRehomeProtocol >= 3`. PR #20105 introduced this capability version with -the idle handoff implementation. With that runtime, both rehome trust environment -settings must be configured to advertise 3; otherwise the cell advertises 0. -An older trusted runtime can advertise 1: configuring trust alone does not upgrade -its implementation. The separate `connectionCapacityProtocol: 2` health field does -not establish regional-correction readiness. Verify the live runtime version and -image, not only instance-template configuration, before rollout or enablement. -Incompatible cells are excluded from correction selection; enabling the cohort -cannot override this check. Director and cell deployments are separate operations. - -The idle-rehome commit runs on the source cell, so from an Asia cell every -statement is a cross-region round trip to the database. It takes no fleet-wide -lock. It locks the global rehome control and worker rows NOWAIT, then the host's -own rows, and reads the cell inventory, runtime, capability and safety tables -unlocked. Its only cell lock is the target cell row, taken NOWAIT by the last -statement before COMMIT. That statement re-checks that the target is enabled, in -general admission and has request capacity. The target row is held for about one -round trip, roughly 175 ms from Asia, and no other cell row is held at all. If -the target changed admission, filled up, or is locked by another writer such as -an admission change, the commit rolls back whole and answers `deferred` with -reason `candidate-ineligible`. The cell logs -`orca_relay_idle_rehome_target_deferred` with the cause, and the director moves -to its next candidate. Runtime metrics report the hold as -`rehomeTargetRowHoldMsMax` and `rehomeTargetRowHolds`. The same hold also feeds -`cellInventoryHoldMsMax`, and `cellInventoryHoldMaxSite` names the lock that -produced that max. An Asia-sourced commit still holds the global rehome control -row for its whole length, roughly 6.5 s at 175 ms per statement. An operator -pause through `applyRegionalRehomeControl` waits 1 s for that row, 3 attempts, -so it can fail during one commit: retry a pause that fails once, and do not -treat that as a fault. - -Each hold site also reports its own p99: `inventoryHoldMsP99`, -`rehomeTargetRowHoldMsP99` and `isolatedReplacementHoldMsP99`, the last being -the regional target rows a drain return locks; a site with no holds in the -interval reports 0, so read these filtered to values above 0. Directors also -report lane slot times (`stickyServiceMsP50/P99`, `drainReturnServiceMsP50/P95`, -omitted when the lane served nothing; a sticky slot held only for a drain-return -or unverified host's verification read counts too, so its p50 moves with that -share), every `/v1/assign` 503 by cause in -`assign503sByCauseDelta`, and `assignNonDrain503sDelta`, which leaves out the -scheduled `drain-return-deferred` answers a fast drain produces by design. -Every 5 s each director samples `pg_stat_activity` for relay backends waiting -on a lock, keyed by waiter role, table and holder role -(`dbLockWaitersByKeyDelta`). The holder is the root of the wait chain, and the -table is the first relay table the waiting statement names. The sample runs -on the director's own 3-connection pool, so when every connection is waiting on -a lock it queues behind them and undercounts director waiters; a dedicated -sampler connection would fix that. Summed waiters divided by `dbLockWaitSamplesDelta` -is the mean number waiting, i.e. lock-wait seconds per second; directors and -cells share one database user, so Query Insights cannot make this split. -Reconciliation logs `orca_relay_reservation_drift` for each cell whose -`reserved_requests` it corrected. - -`host-cooldown-ms` is the minimum gap between two rehomes of one host. It bounds the damage from -a desktop whose region probe flips: without it the host would be dragged back across the ocean on -every flip, since the preference age never expires while the host keeps reconnecting. - -Rehome is currently inflow-only into Asia. The director only picks source cells in its own region, -which is the database's region (US): hosts move from US cells to Asia cells, and a host already on -an Asia cell stays there. Asia cells remain valid targets. The source cell runs the rehome commit, -and an Asia source pays a cross-ocean round trip per statement while holding row locks every cell -needs, which stalled the fleet's database. The preview reports those hosts as -`source-outside-director-region`, and the poll summary line reports the skipped Asia cells as -`skippedOffRegionSourceCells`. This is a temporary stopgap: it is removed once the rehome commit -no longer holds those locks across round trips. Deploy or remove it only while rehome is paused. - -## Game-day matrix - -Run and record each scenario in staging before launch: - -- kill a cell instance mid-session and observe configured-director recovery; -- send zero-grace SIGTERM and compare it with authenticated drain; -- hold an invite install and a resume confirmation across drain; -- wedge a slow receiver until the bounded queue closes only that splice; -- delay mobile Blob conversion and confirm text/binary counter order; -- rotate JWKS while controls refresh; -- make auth unavailable through expiry plus 60-second existing-splice-only grace, then recover with distributed jitter; -- fail SQL during install/confirm and verify only `not-found` or the one committed result is externally visible; -- return a dormant host, overload a cell, kill a cell, evacuate active work, and exercise pre-registration rollback. - -The served black-box relay suite validates the protocol/state transitions used by these procedures. The physical-device and real-GFE canaries remain separate launch gates; unit/black-box success cannot replace them. - -## Optional measured region correction (deployment gated) - -New optimization claims require both the durable regional-rehome control and -`ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT` (integer 0–100, default **0**). -Turning either gate off stops new optional moves; ordinary migration cleanup and -recovery continue. Legacy preferred-region hints do not certify a correction. Both -cells must advertise regional protocol3 and the authenticated desktop control must -advertise idle-regional-rehome-v1. The source must have no actual client sockets or -pending admission/control work; a live control socket alone does not prevent a move. - -The monitor/deploy identity can read **GET `/v1/admin/regional-rehome-preview`**. -It returns full-population eligibility/exclusion counts, open-migration capacity, -process-safety gating and aggregate migration outcomes; it never claims a -host or changes the failure budget. This is advisory, with separately read state: -concurrent assignments, capacity changes, rate pauses and control changes can make -the next claim differ. Inspect the durable control separately before enabling. -Do not treat an unavailable/failed preview as zero eligible hosts. - -`orca_relay_region_correction_outcomes` reports attempts by source/target, -registration/completion/abort state, oldest open age and target -reservation units every five minutes. `orca_relay_region_comparison` samples a -stable 10% of accepted reports (including unchanged hosts), keyed by host digest, -assignment epoch and decision generation. Existing control RTT and client-accept -logs include assignment epoch, control generation and drain mode; join those for -matched before/after and unchanged-cohort comparisons. Client accept latency is -connection setup, not application command round trip. No application-latency -improvement has been demonstrated by probe differences alone. - -Quiet live connections count as work and defer optional correction indefinitely. -A returning client may race with the short admission gate and retry normally. No -optimization timer may close an established client. Investigate failed registration, -ambiguous authority, stuck reservations and reconnect/failure rates against agreed -limits. A database outage can keep the source fenced until locked reconciliation -establishes its authority; timeout alone is not permission to reopen admissions. - -All directors must run the reviewed idle worker before enabling. Record the tested -immutable source and rollback revisions, then verify the ordinary migration recovery -path before rollout. There is no retained-source table or renewal protocol. Deploying -supporting cells/desktops and enabling a cohort require separate rollout authorization -and explicit numerical stop criteria; this change enables neither. - -### Setting the correction cohort during a reviewed director rollout - -The existing **Deploy Relay Production Director** workflow accepts -`region-correction-cohort-percent`: `preserve` (default) or an integer0–100. -It carries the cohort onto both candidate and compatible rollback revisions and -verifies the environment before promotion. If the predecessor has no setting, -`preserve` stamps zero. An explicit change requires the exact disabled durable -rehome generation; configuring a nonzero cohort does not itself enable the sweep. -The usual image, identity, health and traffic checks remain in force. No workflow -was dispatched as part of implementation. - -Terraform reads the cohort from the same traffic-serving revision used to preserve -regional placement. A later apply therefore preserves a workflow-set cohort, -including explicit zero; only an absent service/setting bootstraps to0. Malformed -or ambiguous live settings fail the plan instead of silently resetting the cohort. -The audited director workflow owns subsequent changes. -Before the first nonzero cohort, verify compatible protocol2 cells, updated -cleanup workers, preview eligibility, both serving/rollback images and the -explicitly approved observation/stop criteria. diff --git a/cloud/docs/push-database-cutover.md b/cloud/docs/push-database-cutover.md deleted file mode 100644 index e2b0be3970d..00000000000 --- a/cloud/docs/push-database-cutover.md +++ /dev/null @@ -1,112 +0,0 @@ -# Dedicated push database operations - -Push attaches only to its dedicated PostgreSQL 17 instance: regional HA, 2 vCPU, -7.5 GiB RAM, 50 GiB SSD with automatic growth, seven retained backups and seven-day -point-in-time recovery. Cloud SQL and Terraform deletion protections remain enabled. -The Cloud SQL connector uses the dedicated URL secret pinned to its managed version. -There is no shared-storage fallback or provision/activate switch. - -## Existing-resource cleanup: operator prerequisite - -This is a plan/runbook, not authorization to apply or delete resources. Preserve the -shared Orca instance, dedicated push instance, all dedicated data and identities, and -unrelated resources. The dedicated resource addresses remain unchanged: - -- `google_sql_database_instance.push_dedicated[0]` -- `google_sql_database.push_dedicated[0]` -- `random_password.push_dedicated_database[0]` -- `google_sql_user.push_dedicated[0]` -- `google_secret_manager_secret.push_dedicated_database_url[0]` -- `google_secret_manager_secret_version.push_dedicated_database_url[0]` -- `google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor[0]` - -The relay state may still own these six obsolete shared-store resources, whose -configuration is removed. An untargeted plan would propose deleting them; do not apply it: - -- `google_sql_database.push[0]` -- `google_sql_user.push[0]` -- `random_password.push_database[0]` -- `google_secret_manager_secret.push_database_url[0]` -- `google_secret_manager_secret_version.push_database_url[0]` -- `google_secret_manager_secret_iam_member.push_database_url_runtime_accessor[0]` - -1. Use the production backend in `infra/terraform/README.md`. Inspect state addresses and - the live service attachment, pinned secret reference and revision resources without - printing credentials. Require the dedicated attachment and no old shared-store consumers; - source connection drain needs an authorized operator's read-only observation. -2. Have the shared database owner adopt the six legacy resources in an explicitly owned - archival configuration before retiring their relay-state ownership. Retain the former - database's `prevent_destroy` protection and secret versions; do not disable protection, - drop databases, rotate passwords or introduce a second runtime attachment. A reviewed - exact-address state transfer must preserve remote IDs and secret material in approved - Terraform storage, with no credential exports to local files or terminal output. -3. Require the owner's import/ownership plan to preserve existing resources and then an - empty plan for those addresses. Only after adoption is proven may the operator remove - precisely the six former addresses from relay state under backend locking. Do not - automate this via `removed` blocks, broad `state rm`, force, or an untargeted apply. -4. Review a fresh relay plan. Reject every delete or replace affecting either SQL instance, - dedicated databases/users/secrets, or unrelated resources. Target only the intended push - service and lease IAM grant for rollout; review their dependency closure too. Existing - unrelated drift must be handled by its owner, outside this cleanup. - -No data transfer, dedicated database reset, or phone re-registration is part of this cleanup. - -## Schema prerequisite for existing internal test databases - -New schemas omit `push_hosts` and the unused `host_public_key` and `transcript` columns -on `push_challenges`. Authentication still verifies the encrypted transcript and consumes -its challenge digest once; sessions and device ownership are unchanged. No compatibility -migration for unpublished builds runs at application startup. - -Before deploying onto an older internal schema, an operator must arrange a separately -reviewed schema-preparation job through the approved database execution path. Its entire -scope is dropping `push_hosts` (including its index) and those two unused challenge columns; -preserve challenge digest/expiry/consumption fields and every session, device and delivery -table. Verify that the old NOT NULL columns are absent before admitting the new image. -Do not hand-edit production SQL or reset the dedicated database to satisfy this prerequisite. -Until that job is reviewed and executed, the new image is not ready for an existing schema. - -## Deployment serialization transition - -Finish all old push workflow runs before changing the workflow's lock namespace. An old -shared-lock push run and a new push-lock run do not exclude each other. Hold off new push -dispatches while preparing the following exact changes: - -1. Review the relay-root plan for - `google_storage_bucket_iam_member.github_push_rollout_lease[0]`. It grants only - `roles/storage.objectAdmin` on - `projects/_/buckets/onorca-cloud-terraform-state/objects/terraform/state/push-rollout/production.lock` - to the dedicated push deploy account. The lease action uses object GET/upload/delete, - so no bucket-wide listing or Terraform-state access is needed. -2. After approval, apply only the reviewed IAM/dependency plan. Verify the exact condition - and principal independently. If foundation still grants push membership in the old - `cloud_sql_rollout_lease_members`, its owner removes only that push member; keep Relay's - existing members and permissions. Do not mutate foundation through the relay root. -3. Publish the reviewed workflow on main with `production-push-rollout`, cancellation - disabled, and the existing lease action pointed at the dedicated object. The durable - lease covers admission, candidate validation, activation, traffic changes and recovery. - A stale/conflicting lease stops the run; it is never stolen or force-deleted. -4. Deploy the reviewed image through `cloud-push-deploy.yml`. Preserve candidate readiness, - runtime-provider validation, exact digest/configuration checks, and explicit activation. - Verify the public origin and real notification delivery/dismissal afterward. - -## Recovery and capacity - -Activation starts schema writes and workers before HTTP promotion. Traffic rollback cannot -undo queue or schema changes. Cloud Run cannot delete its latest revision, so failed -activation creates a known-good successor, verifies it, promotes it, then retires rejected -and previous revisions. When partial activation leaves three resources, retire non-latest -inert validation before creating recovery. Failed retirement stops automation. Admission -requires one serving revision resource; retire historical leftovers under the push lease. -Keep the dedicated attachment for application rollback and retain an immutable compatible -image. An image requiring the removed challenge columns needs separate schema review. - -The two-instance ceiling and two-connection pool draw four configured connections, twelve -across three simultaneous revision resources. Terraform caps instances × pool × 3 at 64 -for serving, validation/rejected and active/recovery pools. Push does not draw from Relay's -shared connection budget. Source connections must have drained before treating that old -allocation as free. Increase capacity only after measuring deployed contention. - -Cloud SQL resizing can interrupt connections despite HA. Durable accepted events remain in -SQL; workers retry within each event's original five-minute deadline. Schedule resizes and -verify reconnection, queue recovery, readiness and real delivery afterward. diff --git a/cloud/docs/push-gateway.md b/cloud/docs/push-gateway.md deleted file mode 100644 index 77aba744963..00000000000 --- a/cloud/docs/push-gateway.md +++ /dev/null @@ -1,405 +0,0 @@ -# Orca mobile push gateway - -`orca-cloud-push` is a public Cloud Run service in `onorca-cloud` that turns a desktop -notification into an APNs or FCM push for a paired phone. The desktop registers each phone's -native token with it and calls `POST /v1/send` after the socket fan-out it already does; the -phone treats APNs/FCM as the sole ordinary OS-banner path. The notification socket is retained only -for live dismissal and reconnect tray reconciliation; it does not create or recover banners. Desktop -notification categories remain authoritative. The service is the only place the Apple -`.p8` signing key is readable, which is the reason it exists as a service at all. - -The request schemas live in `packages/push-contract/src/`. This document covers Terraform -ownership, deployment, credential rotation, and recovery. - -**There is no staging push gateway.** That is a decision, not an omission. `push_gateway_enabled` -is false in `environments/staging.tfvars` and true in `environments/production.tfvars`, and every -resource in `infra/terraform/push-gateway.tf` is behind it. A staging gateway would be a tfvars -edit plus a second set of Apple credentials. - -## Shape - -| Setting | Value | Where | -| ----------------- | ------------------------------------------------------ | -------------------------------------------- | -| Cloud Run service | `orca-cloud-push` | `push_cloud_run_service_name` | -| Region | `us-central1` | `region` | -| Instances | min 1, max 2 | `push_min_instances`, `push_max_instances` | -| Database pool | 6 per instance | `push_database_pool_max` | -| Concurrency | 80 | `push_concurrency` | -| Ingress | all | `INGRESS_TRAFFIC_ALL` | -| Invoker | IAM disabled | `invoker_iam_disabled = true` on the service | -| Runtime identity | `orca-cloud-push@onorca-cloud.iam.gserviceaccount.com` | `google_service_account.push_runtime` | -| Database | `orca_push` on dedicated HA PostgreSQL 17 | `google_sql_database.push_dedicated` | -| Hostname | `push.onorca.dev` | `push_base_url` | - -The minimum of one instance is deliberate and did not move when the ceiling came down to two. A -cold start delays a notification past the point where it is worth showing, so the floor is what -keeps a notification prompt. The -ceiling is a different question, answered below. - -Push uses its approved dedicated two-vCPU HA database. Two instances with a six-connection -pool draw twelve connections; three simultaneous revision resources draw thirty-six. Tagged -candidates can run outside the service-wide cap, so Terraform bounds instances × pool × 3 -at 64 connections, leaving dedicated capacity for maintenance and operators. Increase pool -sizes only after measuring contention. The shared Relay budget excludes push entirely. - -Authentication is the host proof in `POST /v1/host/challenge`, not Cloud Run IAM, so the service -opts out of invoker IAM with `invoker_iam_disabled = true`, exactly as the relay director does. -The project's domain-restricted-sharing policy refuses an `allUsers` invoker binding, so that is -the only way to reach an open service here. - -## Environment - -Set on the container by Terraform: - -| Variable | Source | -| ----------------------------- | -------------------------------------------------------- | -| `PORT` | Cloud Run, container port 8080 | -| `ORCA_PUSH_PUBLIC_URL` | `push_base_url` | -| `ORCA_PUSH_FCM_PROJECT_ID` | `project_id` (required for standalone runtime) | -| `ORCA_PUSH_DATABASE_URL` | Secret `orca-cloud-push-dedicated-database-url`, pinned version | -| `ORCA_PUSH_DATABASE_POOL_MAX` | `push_database_pool_max`, 6 per instance | -| `ORCA_PUSH_APNS_KEY` | Secret `orca-cloud-push-apns-key`, version `latest` | -| `ORCA_PUSH_APNS_KEY_ID` | Secret `orca-cloud-push-apns-key-id`, version `latest` | -| `ORCA_PUSH_APPLE_TEAM_ID` | Secret `orca-cloud-push-apple-team-id`, version `latest` | - -`ORCA_PUSH_APNS_TOPIC` is left to its application default (`com.stably.orca.mobile`). Add it here -only when it has to differ from the code default, so that a code-side change stays visible rather -than silently overridden. - -Terraform owns the three Apple secret **names, labels, and replication, and never a version.** -The `.p8` is issued by the Apple developer portal, so a Terraform-managed version would put the -private key in state and would fight the rotation below. The database URL secret is different: -Terraform generates that password, so it owns that version, exactly as `relay-database.tf` does. -That puts the generated password and the full database URL in the state bucket, which the shared -deploy identity can read; the Apple key never appears there. The three Apple secrets and the -`orca_push` database carry `prevent_destroy`, so disabling the gateway fails the plan instead -of deleting the only copy of the signing key or every live device token. - -## Importing what already exists - -The runtime account, the three Apple secrets, and their accessor bindings were created out of -band alongside the Apple credentials. They are declared so a plan is clean, and imported once. -Run these from `cloud/` after `pnpm infra:init --env production`, review the resulting plan, and -expect the imported resources to show no changes. - -```sh -terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ - 'google_service_account.push_runtime[0]' \ - projects/onorca-cloud/serviceAccounts/orca-cloud-push@onorca-cloud.iam.gserviceaccount.com - -terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ - 'google_project_iam_member.push_runtime_fcm_admin[0]' \ - 'onorca-cloud roles/firebasecloudmessaging.admin serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com' - -terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ - 'google_project_iam_member.push_runtime_service_usage_consumer[0]' \ - 'onorca-cloud roles/serviceusage.serviceUsageConsumer serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com' - -terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ - 'google_secret_manager_secret.push_provider["orca-cloud-push-apns-key"]' \ - projects/onorca-cloud/secrets/orca-cloud-push-apns-key - -terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ - 'google_secret_manager_secret.push_provider["orca-cloud-push-apns-key-id"]' \ - projects/onorca-cloud/secrets/orca-cloud-push-apns-key-id - -terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ - 'google_secret_manager_secret.push_provider["orca-cloud-push-apple-team-id"]' \ - projects/onorca-cloud/secrets/orca-cloud-push-apple-team-id - -terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ - 'google_secret_manager_secret_iam_member.push_provider_runtime_accessor["orca-cloud-push-apns-key"]' \ - 'projects/onorca-cloud/secrets/orca-cloud-push-apns-key roles/secretmanager.secretAccessor serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com' - -terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ - 'google_secret_manager_secret_iam_member.push_provider_runtime_accessor["orca-cloud-push-apns-key-id"]' \ - 'projects/onorca-cloud/secrets/orca-cloud-push-apns-key-id roles/secretmanager.secretAccessor serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com' - -terraform -chdir=infra/terraform import -var-file=environments/production.tfvars \ - 'google_secret_manager_secret_iam_member.push_provider_runtime_accessor["orca-cloud-push-apple-team-id"]' \ - 'projects/onorca-cloud/secrets/orca-cloud-push-apple-team-id roles/secretmanager.secretAccessor serviceAccount:orca-cloud-push@onorca-cloud.iam.gserviceaccount.com' -``` - -The push resources already exist in production. Preserve their addresses, dedicated database -and identities; review the [database cleanup runbook](./push-database-cutover.md) before applying -changes. This root has unrelated standing drift, so an untargeted apply is never automatic. - -Two things this root does **not** declare, because the carve assigns them elsewhere. Neither -affects whether this root's plan is clean, since an undeclared resource is invisible to it. - -- `firebase.googleapis.com` and `fcm.googleapis.com` are project service enablement, which is - `google_project_service.required` in the foundation root. They are already enabled; add them - to the foundation root's list so a foundation plan stays clean. -- The Firebase attachment on `onorca-cloud` is project-level and belongs with foundation for the - same reason. It exists already. - -## Deploying - -`Deploy Push Gateway Production` (`.github/workflows/cloud-push-deploy.yml`) is the only -supported path. Like every `cloud-*` workflow it does nothing until `ORCA_CLOUD_OPERATIONS_ENABLED` -is `true`, it runs only on `main`, and it needs the confirmation string `DEPLOY_PUSH_GATEWAY`. - -It authenticates as the dedicated `orca-cloud-gha-push` identity through -`PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER` and -`PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT`. `push-deploy-identity.tf` restricts Workload Identity -to this exact dispatch workflow on main in the production environment. Its distinct principal -attribute cannot assume the shared Relay deploy identity. - -The account can write images to Artifact Registry, deploy the push service, impersonate only -the push runtime account, and manage exactly `terraform/state/push-rollout/production.lock` -in the production state bucket. The relay root owns that conditional lease grant. It grants -no Terraform-state object access. Publish `github_push_workload_identity_provider` and -`github_push_deploy_service_account` as the production-environment variables above. - -The workflow uses the `production-push-rollout` concurrency group with cancellation disabled -and the existing durable lease action on the push-specific object. Push and Relay deploy -independently; two push deploys cannot race traffic changes. Finish every old shared-lock push -run before enabling the new workflow and lease grant. See the cleanup runbook for the bounded -IAM transition and removal of any obsolete foundation-owned push membership. - -The run builds the reviewed `source_sha` while the workflow stays on `main`. Buildx returns -its own pushed digest (no mutable-tag lookup); every subsequent check and deployment uses that -same digest. Before any production boot, a network-isolated container checks that the image -recognizes `ORCA_PUSH_MODE=validation` and rejects invalid modes. Older images that lack this -capability are refused before they can connect to production. - -Under the production push rollout lease, it records the serving rollback revision and -asserts Terraform-owned scaling. It deploys a tagged, zero-traffic validation revision: - -- Validation opens PostgreSQL with `default_transaction_read_only=on` and skips schema setup. -- No delivery worker or challenge, session, or delivery pruner starts. -- Only `/health` and `/ready` are available; all application routes return 503. -- `/health` attests `mode: validation`; `/ready` checks database connectivity only. It does not - prove schema compatibility, provider delivery, or active-worker readiness. Container probes - can still use `/health` without treating an inert process as unhealthy. - -The build explicitly targets `linux/amd64` with provenance disabled so build metadata records a -single manifest digest, rather than an OCI index that Cloud Run resolves to a different digest. -The workflow verifies the exact image and scaling, probes readiness and mode, and checks the -runtime identity with a validate-only FCM request. Cloud Run rejects deletion of the latest -created revision even when it has no tag or traffic. Activation therefore creates a successor -before removing the validation tag and deleting validation. The dedicated 64-connection budget -reserves three simultaneous revision pools: serving, validation/rejected, -and active/recovery successor (36 configured pool connections at the current two-by-six shape). -Revision deletion is not proof of physical SQL session drain; verify termination and SQL sessions -in controlled rollout acceptance. There is no shutdown sleep used as a drain gate. - -**Activation deliberately starts production effects.** The distinct active revision uses the exact -validated digest with the validation override removed. Schema setup runs on its existing -one-connection untimed pool, followed by workers and pruners, before HTTP promotion. The workflow -checks digest, full runtime spec and secret-reference shape, scaling, readiness and active mode, -then moves HTTP traffic and checks the public origin. Those checks commit the new serving revision; -subsequent retirement failures do not trigger rollback to a possibly deleted previous revision. -The previous consumer is retired and all tags are cleared. Retain the previous immutable image -from the summary: later recovery redeploys that digest, because the previous revision is deleted. - -Before any candidate creation, the workflow requires exactly one revision resource, the sole HTTP -serving revision. Existing historical revisions or leftovers from interrupted runs require explicit -operator review and cleanup under the lease first; the workflow does not blindly delete them. -This gate and retirement after every successful rollout prevent repeated runs accumulating workers. -Terraform still owns configuration and scaling; removing validation mode adds no ignored field. - -On failure before public checks pass, any attempted traffic shift is first rolled back and verified. -If partial activation created a successor, recovery retires non-latest validation first; deletion -failure stops recovery before a fourth resource can be created. Recovery then deploys the captured -known-good digest as a tagged, zero-traffic successor with normal mode. It verifies template shape, -secret references and scaling, probes tagged readiness and active mode, promotes the recovery -revision, verifies traffic and public health, and only then deletes rejected and previous revisions. -The latest recovery revision remains serving. Known-good recovery schema and workers can execute -before promotion; neither recovery nor traffic rollback undoes schema changes or sent notifications. - -Partial creates record deterministic names before mutation. Failed recovery or deletion requires -operator cleanup under the lease; the next automated run refuses leftover resources. A canceled -runner can require the same intervention. Traffic restoration alone does not stop queue consumers. - -Manual recovery must preserve the three-resource bound and keep the successor serving: - -```sh -# Hold the rollout lease; inspect latest, traffic, tags and existing revisions first. -# If three resources remain after partial activation, retire non-latest inert validation first. -# Restore previous traffic if its revision still exists and a failed candidate took traffic. -gcloud run deploy orca-cloud-push \ - --project onorca-cloud --region us-central1 --image \ - --remove-env-vars ORCA_PUSH_MODE --no-traffic \ - --tag --revision-suffix -# Verify exact digest, template spec/secret references/scaling, tagged /ready and active /health. -gcloud run services update-traffic orca-cloud-push \ - --project onorca-cloud --region us-central1 --to-revisions =100 -# Verify traffic and public /ready and /health before retiring old consumers. -gcloud run services update-traffic orca-cloud-push \ - --project onorca-cloud --region us-central1 --clear-tags -gcloud run revisions delete \ - --project onorca-cloud --region us-central1 -# Repeat only for reviewed obsolete revisions; retain the latest serving recovery revision. -``` - -Never merely remove validation mode while the template still holds a rejected image. Terraform -owns environment configuration but ignores the image, so that would activate rejected code. -Remove a tag only if it remains present. Verify the recovery revision is serving, the template is safe, -and obsolete revision deletion and connection drain completed; -already accepted provider sends cannot be undone. Activation-time schema changes must be additive -and compatible with the rollback image: rollback does not reverse migrations or queue mutations. -The inert phase intentionally cannot validate a new schema by applying it to production. Review -migrations and validate them against isolated PostgreSQL before dispatch. No actual Cloud Run -rollout, provider delivery or physical-device acceptance is implied by local contract tests. - -### Incompatible queue rollout prerequisite - -The queue stores one notification object per delivery. Before deploying a revision that changes this -format, stop every older push gateway revision and clear only unpublished push delivery fixtures from -the push database. This is an unpublished feature, so do not preserve or migrate queued fixtures; no -production mutation is implied by this prerequisite. - -### Why the FCM probe impersonates the runtime account - -A gateway that boots and answers `/ready` can still be unable to send: the FCM grant lives on -the runtime service account, not on anything the readiness check touches. The probe therefore -mints an access token for `orca-cloud-push@onorca-cloud.iam.gserviceaccount.com` and posts -`validate_only: true` with a token that cannot exist. `validate_only` stops Google before any -delivery, and a healthy credential answers `INVALID_ARGUMENT` because the device token is -garbage. `PERMISSION_DENIED`, `401`, and `403` are the failures the step exists to catch, and -they fail the run immediately, before traffic moves. Those four answers are the only conclusive -ones: a `429`, a `5xx`, or a transport failure says nothing about the credential, so the send is -retried up to five times at five-second intervals rather than read as either verdict. Probing as the deploy identity instead would prove -something true about the wrong account. - -## Rotating the APNs key - -Apple keys do not expire, so this is for a suspected compromise or a routine rotation. Order -matters: the new key must be serving before the old one is revoked, or every iOS push fails in -the window between. - -1. In the Apple developer portal, create a **new** APNs authentication key. Download the `.p8` - once; Apple will not show it again. Note the new key ID. A team may hold two APNs keys at a - time, which is what makes this overlap possible. -2. Add a version to each changed secret, without printing the value: - - ```sh - gcloud secrets versions add orca-cloud-push-apns-key \ - --project onorca-cloud --data-file /path/to/AuthKey_NEW.p8 - printf '%s' '' | gcloud secrets versions add orca-cloud-push-apns-key-id \ - --project onorca-cloud --data-file=- - ``` - - The team ID does not change, so `orca-cloud-push-apple-team-id` is untouched. - -3. Dispatch `Deploy Push Gateway Production`. The container reads `latest` at start, so only a - new revision picks the key up; there is no in-place reload. -4. Verify from a real device that an iOS notification still arrives. The workflow's FCM probe - covers Android only, and APNs has no validate-only equivalent. -5. Only then revoke the old key in the Apple portal, and disable the superseded secret versions: - - ```sh - gcloud secrets versions disable \ - --project onorca-cloud --secret orca-cloud-push-apns-key - ``` - - Disable rather than destroy, so a rollback to the previous revision still works. Destroy - after the next clean deploy. - -Delete the downloaded `.p8` from disk when you are done. It is the whole credential. - -## Dead tokens - -A push token stops working when the app is uninstalled, when the user restores to a new device, -or when iOS reissues it. Both providers report this, and the shapes differ: - -- APNs: HTTP 410, or 400 with `BadDeviceToken` or `Unregistered`. - `DeviceTokenNotForTopic` is a provider configuration error and leaves the registration live. - Check the APNs topic and environment; future notifications can resume after correction without - phone re-registration. The failed notification is not retried for this non-transient error. -- FCM: `UNREGISTERED`, or `INVALID_ARGUMENT` whose message names the token. - -The gateway marks the registration `dead_at` and returns `status: "dead"` for it, and the -desktop drops the registration when it sees that. Nothing here retries a dead token. A phone -that comes back re-registers the same host/device pair, retaining its `registrationId` and -clearing `dead_at`. The per-minute `delivery_dead` counter measures delivery outcomes, not -currently dead registrations. A spike across many hosts warrants checking credentials and topics. - -## Quotas - -Two independent limits, both enforced in the gateway and both returning HTTP 200 with -`status: "rate_limited"` per result rather than failing the request: - -| Limit | Scope | -| --------------------------------------------- | --------------------------------------- | -| 300 logical alerts per rolling 15 minutes | per `hostFingerprint` | -| 300 logical dismissals per rolling 15 minutes | per `hostFingerprint`, separate budget | -| 20 `registrationIds` | per request, hard cap, HTTP 400 over it | - -Fanout to several phones counts one logical event; there is no per-phone daily allowance. -Unauthenticated handshakes and invalid bearer attempts have separate 30/minute IP buckets. -Authenticated routes use a 600/minute host bucket and a shared 6,000/minute client-IP bucket -per instance. The IP budget cannot be reset by generating another host key. It is shared by -clients behind one NAT and is an abuse safeguard, not a global provider-spending cap. Auth database lookup concurrency -and waiting work are bounded independently of HTTP concurrency. - -`push_events` backs quota accounting. `push_event_recipients` deduplicates fanout and -`push_delivery_batches` retains its historical name and persists individual pending deliveries, -worker leases and retries. A delivery row is deleted when it is sent, dead, dismissed or expired, so -the table holds only live work. Event and recipient identity metadata is retained for 24 hours. -Payloads expire within five minutes. Minute-level cleanup deletes in bounded batches, so a backlog -drains over successive runs instead of in one long statement. FCM project-level provider quotas -remain independent of host limits. - -A worker claims one device's oldest due delivery with a row lock that other claimers skip, and a -non-blocking per-device lock keeps at most one delivery per phone in flight. Each claim also takes -the previous revision's global claim lock in shared mode, so during a deploy overlap an old worker's -claim waits for new leases to commit instead of re-leasing them. That shared lock can be removed one -release after every worker runs this revision. The claim scan only -reads rows due within the notification TTL, so an unpruned backlog does not slow it. Boot adds one -queue index, a partial index of pending rows per device for the head check; it indexes no lease -column, so lease and renew writes stay heap-only updates. Claim, finish and cleanup traffic may -hold at most one fewer connection than the pool size, so request authentication always has a -connection. Lease renewals skip that cap so they never queue behind claims. - -Logging is aggregate counters only. Never log a token, a title, a body, or a full fingerprint; -the first four characters of a fingerprint are the most that may appear. - -## DNS: one hand-managed record - -The Cloud Run domain mapping is created here, and Google issues and renews the certificate. The -`onorca.dev` zone is not in this root: it is a Cloudflare zone whose Terraform-managed records -live in the apps root in `stablyai/orca-cloud`, and whose relay and auth records are managed by -hand. The push record follows the relay's precedent and was created by hand on 2026-09-04: - -```text -push.onorca.dev. CNAME ghs.googlehosted.com. (DNS only, not proxied) -``` - -`terraform -chdir=infra/terraform output push_dns_record` prints the same three fields. If the -record is ever lost, recreate it exactly like that; Cloudflare proxying blocks certificate -issuance and breaks Cloud Run host routing. - -### Recovery and delivery guarantees - -Candidate tags and deterministic revision names are recorded before deployment. Promotion intent is -recorded before changing traffic, so a failed verification or ambiguous mutation result still triggers -rollback. A known-good successor must exist before the rejected latest revision can be deleted. -After verified recovery promotion and public checks, rejected and previous consumers are retired; -the recovery revision remains serving. Failed cleanup blocks subsequent rollout admission. -The summary runs even if candidate discovery or traffic verification fails. - -Push uses the relay's schema-startup retry implementation through `@orca-cloud/postgres-schema`. -Session replacement is serialized per host and a unique host index upgrades older databases by -retaining their newest session. Cloud Verify runs push concurrency tests against PostgreSQL. - -Accepted sends commit quota and pending work together before returning `queued`. Workers resume -unfinished deliveries after restarts without relying on desktop retries. The durable queue and -expiring leases coordinate replicas. All provider attempts retain the original five-minute deadline -and respect provider backoff; no retry extends alert life. Silent dismissal messages have their own -quota and cancel matching unsent alerts. Mobile OS delivery/execution is not guaranteed. - -Shutdown stops admission and new claims; unfinished leases remain recoverable. Provider acceptance -and SQL completion cannot be atomic, so repeated transport delivery remains possible after a crash. -Stable per-event replacement identities reduce duplicates without promising exactly-once visible -delivery. FCM notification messages are inherently collapsible while offline and support only a -small number of concurrent collapse keys per device, so excess pending messages may be discarded and -every offline alert is not guaranteed to appear. Socket reconnect reconciles dismissals against the -current native tray; it has no stored replay watermark and never recovers a missed OS banner. - -### Dedicated database operations - -Push has one dedicated database attachment, with stable Terraform addresses and deletion -protection. There is no switch to shared storage. Follow the [database operations runbook](./push-database-cutover.md) -for deployment prerequisites, legacy resource ownership, capacity and recovery. diff --git a/cloud/docs/relay-database-failure-diagnostics.md b/cloud/docs/relay-database-failure-diagnostics.md deleted file mode 100644 index 1c356f95fae..00000000000 --- a/cloud/docs/relay-database-failure-diagnostics.md +++ /dev/null @@ -1,43 +0,0 @@ -# Relay database failure phases - -`orca_relay_postgres_query_failed` separates failure to acquire a pooled connection -(`phase=acquire`) from failure after acquisition (`phase=execute`). It covers -`PostgresDatabase.query`, including the single-statement control-renewal CTE. -Statements inside explicit transactions use a different query path and are not -covered. These events are diagnostic evidence, not a replacement for total SQL -failure counters. - -The event contains only an allowlisted error code, a connection-timeout boolean, -a transient boolean, the operation category (`control-renewal` or `other`), total -elapsed milliseconds, and pool total/idle/waiting counts at failure. Total elapsed -time includes acquisition. An acquisition timeout can mean either waiting in the -queue or establishing a new connection; `connectionTimeout` covers both, and the -pool counts separate them: a queue wait has waiters, a dial does not. -Unknown error codes stay `unknown`. - -`transient` is the classification the request routes act on, not a second -opinion: true means retryable, false means terminal. It is not a count of HTTP -responses. Every caller of `PostgresDatabase.query` emits this event, including -background sweeps, startup reconciliation, and admin routes that map a failure -to 409, and none of those produces a 503 or a 500. Counting `transient=false` -therefore over-counts user-facing hard failures; narrow by operation, or join -against the route's own rejection logs, before reading it that way. A pool that -cannot hand out a client carries no error code at all, so `code` stays `unknown` -for that whole class and only these two booleans separate it from a genuine -fault such as a rejected password. - -Query text, parameters, error messages, and identifiers are never emitted. -Successful queries emit no additional event. - -Use structured GCE logs with `jsonPayload.event="orca_relay_postgres_query_failed"`. -Compare counts by phase, operation, and code with the same cell's renewal outcomes -and pool pressure, and with independent PostgreSQL wait samples. Establishing the -failure phase does not by itself establish why the pool backed up. - -For production observation, use an immutable image through the same-cap workflow -on one cell, with fresh monitor evidence and the exact predecessor digest. Verify -the serving digest and health, then inspect these events during a naturally -occurring failure. Do not deliberately induce a production database failure. -Rollback uses the same workflow and predecessor image; no schema or database -configuration changes are involved. Do not change rehome limits, timeouts, pool -sizes, or renewal scheduling merely to collect this evidence. diff --git a/cloud/docs/relay-improvement-checklist-2026-09.md b/cloud/docs/relay-improvement-checklist-2026-09.md deleted file mode 100644 index 8d86afd4699..00000000000 --- a/cloud/docs/relay-improvement-checklist-2026-09.md +++ /dev/null @@ -1,192 +0,0 @@ -# Relay improvement: implementation checklist, lanes, and disruption - -Companion to [`relay-improvement-roadmap-2026-09.md`](./relay-improvement-roadmap-2026-09.md) (item numbers -match). This file answers three questions per item: what are the concrete steps, what can run in parallel, -and will a user notice. - -## Status as of 2026-09-06 16:30Z - -Three buckets. "Merged" means the code is on `main` and nothing in production has changed yet. "Deployed" means users are already getting it. "Awaiting owner" means I will not touch production without a go. - -**Deployed to production** -- Roll 2 relay image `4916ed67` (stablyai/orca #18959 + #18722 + #18720 flag unset): director since 2026-09-06 01:02Z, all 19 general cells by 16:29Z. Control lease 6 h ± 30 min, accept abandonment, per-cell inventory locks, pool `statement_timeout`. Record: findings doc, "Roll 2" section. -- Auth instance cap 20 + dead-family audit fix (orca-cloud #474) as revision `orca-cloud-auth-00031-tox`. -- Dynamic NAT ports in both regions (stablyai/orca #18693). Zero drops and zero proxy dial errors since. -- Nine alert policies with log metrics: 4 auth (#475), 3 relay Cloud SQL/NAT (#18693), 1 cell process-exit (#18717), all on the relay Slack channel. - -**Merged, not yet live** -- Cells dial Cloud SQL with `--private-ip` when configured (#18720). Deployed in Roll 2 with the flag unset; inert until 2.1 applies. -- Phone shows a clear "sign in on the desktop again" state when the desktop is signed out (#18698). - -**Merged, ships with the next auth deploy** -- Refresh rotation grace window (orca-cloud #478). Startup adds one nullable column (brief exclusive lock on `refresh_tokens`). -- Pruning job code (orca-cloud #476) is in the image; the job itself is Terraform-disabled until 1.2. - -**Merged, ships with the next desktop release** -- Never replay a refresh token after a timeout; ±10 % jitter on relay lease renewal (#18719). -- Renderer learns when a cloud session is revoked (#18694). - -**Merged, not applied** -- Incident dashboard (#18717) blocked behind the runtime-metric label drift (5.x first item). -- Monitor probe fix (#18723) is live in the workflow; the same-cap roll gate has not yet produced a green dry-run since. - -**Awaiting owner go (production mutations)** -1. Roll 1 cell image roll (1.1): dry-run gate, then c8 canary, then batches. -2. Auth deploy carrying #478 (3.1): quiet minute for the column add. -3. orca-cloud #477 private IP (2.1): merge arms an instance restart and a one-way door. Recommendation: hold. -4. Runtime-metric `region` label drift (5.x): intentional replacement of 21 metrics, or drop the label. -5. Enable pruning (1.2): first budget 20k rows; needs a Terraform apply. -6. Paging channel for auth alerts (5.2): needs the destination from you. - -**Open code follow-ups (no gate, nobody assigned)** -- Monitor summary Markdown does not render `tolerated: true` continuity events (added by #18798); the state artifact has them, the checkpoint table does not. -- Relay container boot races the `cloud-sql-proxy` sidecar: c13's fresh container exited twice (`applyPostgresSchema` connection timeout, 2 s each) before the proxy was listening. Make schema apply wait for the proxy or order the containers. -- `cloud-deploy-relay-production-capacity-job.yml` (~line 416) has the same wave-0 single-shot preflight carve-out that #18778 removes from the same-cap job; its single-evidence path never retries freshness-only failures. -- `cloud/package.json` `test` names every dev-script test file explicitly; an unregistered `*.test.mjs` is silently never run in CI (found by #18769). Needs a glob or a ratchet that fails on an unlisted test file. -- Same-cap job's verify step uses bare `curl --fail-with-body` against the just-rolled cell; one 503 at the LB warm-up edge failed c8 canary #2 (run 33935407461) after the transition verifier had already passed. Needs a bounded retry, same rule as #18723/#18740. -- `verify-mutation` in `cloud-deploy-relay-production.yml`, the multi-target workflow, and the capacity workflow still binds to an exact commit; same exposure #18754 fixed for the same-cap and rehome paths. -- `incident-live-preflight-cli.ts` reports only `source/code` (`active-probe/threshold_max`) with no signal name or observed value, so a failed mutation preflight (c27 recovery #3, run 33986948522) cannot be attributed to an endpoint without an out-of-band probe. Print the signal and observed/threshold pair. Related: the 2 000 ms `endpointLatencyMs` bar is shared by US and Asia cells while Asia /health round trips from a US runner sit at 0.7–1.3 s idle; consider a per-region bar or the p50 of the gate window instead of one shot. Gates #44 and #45 (2026-09-05) both froze on `cell.production-gce-c27.latency_ms` at 2.6–2.7 s with c28 showing the identical tail under operator probes; the bar is now blocking Asia rolls. **Fix: stablyai/orca #18877** (per-region `cellEndpointLatencyMs`, us-central1 2 000 / asia-east2 4 000, plus signal/observed/threshold in preflight messages). Residual: `probeEndpointHealth` in `resource-inventory.ts` still uses the flat 2 000 bar to decide whether to retry after the 10 s readiness-cache wait, so a healthy Asia cell over 2 s costs one extra probe per sample (latency, not verdict); thread the region bar into the retry decision. -- The root oxlint config ignores `cloud/**`, so `check:code-quality:changed` never inspects relay-ops or the cloud dev scripts; typecheck + vitest is the only gate there. -- Monitor bars that froze on non-health today: `directorInstancesMin: 5` with `latest-sum` (one-minute instance recycle), `endpointLatencyMs: 2000` on a US-runner probe to asia-east2, `cloudDataMaxAgeMs: 180000` vs Cloud Monitoring publish lag up to 255 s. Recalibrate with a week of data. -- `parsed()` in `resource-inventory.ts` still returns null on a 200 with a malformed MIG body; a second path to `runtime_power_unknown`. -- Deploy script strips `ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` on every release (3.1 first item). -- `assignOnce` placement lock still global (4.1 remainder). -- Region preference (4.2), retries-bar recalibration after a week of Roll 2 data (4.4), pruner `stopReason` alert (1.5). -- Full apps-root apply for 4 unrelated drifts (1.4), from a host with the 1Password account. - -## Uplift ranking (reliability gained per unit of effort) - -| Rank | Item | Why it ranks here | -|---|---|---| -| 1 | 1.1 cell image roll | Removes the only crash mode we have seen in production. 22 of 23 cells still have it. One afternoon. | -| 2 | 3.1 refresh rotation grace window | Turns the entire "slow auth → mass sign-out" class into a slowdown. One day. | -| 3 | 4.1 inventory lock contention | The floor under every 503 and slow phone accept, every day, not just incidents. One week. | -| — | 2.2 relay/auth database split | **Deferred 2026-09-04** to ~2026-11-01. Biggest structural fix, but the concrete cause is fixed and alerts now page; see roadmap 2.2 for re-open triggers. | -| 4 | 1.2 + 1.3 pruning and reclaim | Defuses the 63 M-row time bomb. Low effort, mostly waiting. | -| 5 | 5.1 + 5.2 crash alert, page a human | Cheapest detection uplift; today's incident ran 4 h unpaged. | -| 6 | 2.1 private IP | Durable version of a fix that already landed (dynamic NAT ports). Do it on the existing instance. | -| 7 | 4.3 + 3.2 desktop hardening | Small, ride the normal desktop release. | -| 8 | 4.2, 4.4, 5.4, 1.4, 1.5 | Housekeeping and quality-of-life. | - -## The shared bottleneck: cell rolls - -Every change to what runs on a cell (image, proxy flag, env, relay code) needs a same-cap roll: drain → -recreate → verify, one wave at a time, gated by the 15-minute monitor, about an afternoon. Each wave forces -the desktops on that cell to re-dial (c7 canary: 807 controls re-dialed in ~10 s) and phones on those -desktops reconnect on their normal retry. Users see a few seconds of "reconnecting" per wave. - -So batch. Two rolls, not five: - -- **Roll 1 (now):** current image only (1.1). Do not wait for anything else. -- **Roll 2 (week 2–3):** proxy `--private-ip` (2.1) + relay pool `statement_timeout` (2.3) + lock-contention - fix (4.1), all in one image/template. Prerequisite: 2.1's peering and private IP exist first. - -## Lanes (independent; different people can own them) - -``` -Lane A data plane 1.1 roll ──────────────────► Roll 2 (2.1 flag + 2.3 + 4.1) ──► 4.4 recalibrate -Lane B auth/DB 1.2 enable pruning ──(10 d)──► 1.3 reclaim 3.1 grace window (any time) -Lane C network 2.1 peering + private IP ─────┐ (feeds Roll 2) (2.2 DB split deferred) -Lane D desktop 3.2 no same-token retry, 4.3 lease jitter (any release; wire-compatible) -Lane E observability 1.5, 5.1, 5.2, 5.4 (Terraform only, any time) -Lane F director 4.2 region preference (Cloud Run deploy, any time) -Misc 1.4 full apps-root apply (any time; see its check) -``` - -Hard dependencies: Roll 2 waits on 2.1's network work; 1.3 waits on 1.2 finishing. Everything else is -independent. (2.2 deferred; if revived, do it after 2.1 so the new instance is private from day one.) - -## Disruption summary - -| Item | User-visible? | What they see | Mitigation | -|---|---|---|---| -| 1.1 / Roll 2 | **Yes, transient** | Per wave, desktops on that cell reconnect within seconds; phones follow on retry. | Waves gated by the monitor; run in the US night. Already rehearsed on c7. | -| 1.2 pruning | No | Background deletes, 5k rows per batch. | Small first budget; watch `stopReason` and Cloud SQL write throughput. Stop the scheduler if checkpoint alerts fire. | -| 1.3 reclaim | **Depends on tool** | `VACUUM FULL` takes an exclusive lock on `refresh_tokens`: sign-in and refresh block for its duration (minutes to tens of minutes on 16 GB). `pg_repack` holds only brief locks. | Use `pg_repack`. If VACUUM FULL, announce a maintenance window. | -| 1.4 full apps apply | Should be none, **verify** | Terraform will create a new auth revision (env added). Traffic is pinned to `00031-tox` by name, so the new revision should receive 0 %. | Confirm in the plan that no `traffic` change appears. If it does, stop: the Terraform image variable is not the serving image. | -| 1.5, 5.x alerts | No | | | -| 2.1 private IP | **Yes, certain** | Google: "Configuring an existing Cloud SQL instance to use private IP causes the instance to restart, resulting in downtime." No in-place path, HA does not avoid it. Expect 1–2 min DB unavailability: sign-in fails, relay renewals retry. **One-way door**: private IP cannot be disabled and the VPC link cannot be removed once set. The proxy flag change rides Roll 2. | Off-peak; only after Roll 1 (old image dies on a 2 min DB blip). Owner decision required before the foundation apply. | -| 2.2 DB split (deferred) | **Yes, scheduled** | Relay unavailable for the cutover (drain all cells → copy relay tables → flip `DATABASE_URL` → restart). Minutes if rehearsed. Desktops and phones reconnect automatically after. | Rehearse on staging; do it in the US night; announce. | -| 2.3 statement timeout | No beyond Roll 2 | | | -| 3.1 grace window | No | Auth deploys are no-traffic candidate → smoke → promote. | Security trade-off: a stolen token replayed inside the window is served once instead of revoking. 60 s is the usual choice. | -| 3.2, 4.3 desktop | No | Normal app update. | | -| 4.1 lock fix | No beyond Roll 2 | | Verify against real Postgres on 55440 with concurrent probes before shipping. | -| 4.2 region preference | **Minor, Asia users** | Phones that start being placed in Asia reconnect once to a nearer cell. | Roll out behind the existing region-preference flag. | -| 4.4 | No | | | - -## Checklists - -### 1.1 Cell image roll (Roll 1) -- [x] Confirm fleet is quiet: 15-min monitor dry-run passes. #19 green 23:07:53Z (run 33927238469). Canary then failed the evidence provenance check because main moved during the gate; re-gating with a same-commit chain. -- [x] Confirm director is on 519f4914 and c7 on 85bf6799 (confirmed 2026-09-04 via instance-template census; 20 serving cells still on `5aedbca5`) (`verify` mode of the same-cap workflow). -- [x] Dispatch `cloud-deploy-relay-production-same-cap` waves per the plan in the findings doc; one wave, verify, next. Done 2026-09-05 01:14Z–22:27Z: c8 canary, US batches c9–c10, c13–c16, c19–c26 at protocol 1, then Asia c27 (recovered via `mode=rollback` re-entry after gate freezes on the flat latency bar, fixed by #18877), c28, c29 as single-cell canaries at protocol 0. -- [x] After each wave: the transition verifier passed at migration-only and again at general on every cell (assignments carried, heartbeat fresh, hard cap 3 000); no `container die` fleet-wide across the whole roll. The 4408/1006 burst per wave was not measured separately; the verifier's assignment count before and after each restart is the recovery evidence recorded. -- [x] Record image census in the findings doc. 2026-09-05 22:27Z: all 19 general cells on `519f4914` except c7 on `85bf6799`; existing-only c1–c6, c11, c12 and migration-only c17, c18 untouched on their older images by design. Selector at gen 148. - -### 1.2 Enable pruning -- [x] `auth_token_pruner_image` = digest of `orca-cloud-auth-00031-tox` (`343a0915…`; it contains the entrypoint). orca-cloud #479 merged. -- [x] `auth_token_pruner_enabled = true`, `auth_token_pruner_max_rows_per_run = 20000` for the first day (orca-cloud #479). -- [x] Targeted plan asserted 9 create / 0 change / 0 destroy. Applied 2026-09-05 02:06Z. -- [x] Trigger one run by hand; read the summary event. 02:18Z: `time-budget`, 73 batches, 365k scanned, 1 040 deleted (1 021 revoked, 19 expired), no errors. Scan-bound. -- [ ] Raise the budget to the default 200k after a clean day; watch Cloud SQL write MB/s and the checkpoint alert. -- [ ] 1.5: log metric + policy on `stopReason != complete`. - -### 1.3 Reclaim -- [ ] Wait for steady-state runs deleting ~0 rows. -- [ ] `pg_repack -t refresh_tokens` off-peak (needs the extension; check `pg_available_extensions`). Not `VACUUM FULL` without a window. -- [ ] Confirm table + index size and `disk/utilization` dropped. - -### 1.4 Full apps-root apply -- [ ] Run from CI or a host with the 1Password account (local plan fails on the Cloudflare data source). -- [ ] Plan shows exactly the four known drifts and **no traffic change** on `google_cloud_run_v2_service.auth`. -- [ ] Apply; confirm `status.traffic` still pins `00031-tox` at 100 %. - -### 2.1 Private IP (PRs open: orca-cloud #477 foundation, stablyai/orca #18720 relay flag) -- [ ] **Owner decision**: the foundation apply restarts the instance and is irreversible on Google's side. Merging #477 arms the next foundation apply; hold the merge until the window is chosen. -- [ ] Director is out of scope: it uses the Cloud Run built-in connector (managed Google path, not the relay VPC NAT), so it consumed none of the exhausted ports; moving it needs Direct VPC egress + a separate DSN secret. Own PR if ever wanted. -- [ ] Step 7 (`ipv4_enabled=false`) is blocked until humans have IAP/bastion access and the director is moved; it breaks both today. -- [ ] Allocate a `/24` private services range on the relay VPC; `google_service_networking_connection`. -- [ ] Add `ip_configuration.private_network` to `google_sql_database_instance.auth` (foundation root). Plan must show update, not replace. -- [ ] Apply off-peak; expect a possible restart. Watch auth 5xx alert and relay `sqlFailures`. -- [ ] Cell template: proxy args add `--private-ip` (code merged #18720; flag not set). Director: Direct VPC egress or connector, then the same flag. Both ride Roll 2. -- [ ] After Roll 2: NAT `port_usage` for relay gateways drops to ~0; then consider `ipv4_enabled = false` (removes the public IP; breaks the local `cloud-sql-proxy --token` workflow unless it also goes private). - -### 2.2 Database split (deferred to ~2026-11-01; checklist kept for when it is revived) -- [ ] New `google_sql_database_instance.relay` (private IP from day one, its own size and flags). Staging first. -- [ ] Relay schema applies cleanly to an empty instance (it does at startup). -- [ ] Rehearsal on staging: drain → `pg_dump` relay tables → restore → flip `relay_database_url` secret → restart director + cells → phones/desktops reconnect. Time it. -- [ ] Production: announce a window; same steps; verify `orca_relay_runtime_metrics` controls recover to pre-cutover count. -- [ ] Update `production-cloud-sql-app-consumers` budget test and both alert policies' `database_id`. - -### 2.3 Relay pool statement timeout (deployed in Roll 2, 2026-09-06) -- [x] `statement_timeout` on the relay `pg.Pool` (5 s, env-configurable; schema pool untimed; `57014` retryable), below the control-renewal deadline; DDL on an untimed connection (same pattern as auth #476). -- [x] Postgres test on 55440: a held lock fails the query fast and the bounded retry takes over. -- [x] Deployed fleet-wide in Roll 2 (`4916ed67`), 2026-09-06. - -### 3.1 Refresh rotation grace window (orca-cloud #478 merged 2026-09-04; deploy pending owner go) -- [ ] Fix the deploy-script env strip for `ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` (pre-existing; found by #478). -- [x] `rotateRefreshToken`: if `rotated_at` within 60 s and not revoked, return the existing successor (idempotent), no revoke, no audit. -- [x] Outside the window or a third presentation: unchanged (revoke + audit). -- [x] Tests: replay inside window returns same successor; outside revokes; concurrent double-present yields one successor. -- [x] Deploy via `deploy-auth-production` (candidate → smoke → promote). Deployed 2026-09-04 23:15Z as `orca-cloud-auth-00035-gos`, cap 20 kept, 0 5xx; `successor_material` column present; sealed successors being written. (candidate → smoke → promote). - -### 3.2 / 4.3 Desktop (merged stablyai/orca #18719; ships next desktop release; relay side of 4.3 deployed in Roll 2) -- [x] 3.2: on refresh timeout, re-read stored session before retrying; do not re-send a token already rotated locally. -- [x] 4.3: ±10 % jitter on control lease renewal; unit test on the distribution; wire-compatible (server accepts early renewals already). -- [x] 4.3 relay side: control lease 55 min → 6 h ± 30 min (#18959), deployed in Roll 2, 2026-09-06. - -### 4.1 Lock contention (partial: stablyai/orca #18722 deployed in Roll 2, 2026-09-06) -- [x] Replace the global `FOR UPDATE` over `relay_cells` with per-cell row locks; counters delta-only. Remaining: `assignOnce` placement lock is still global (optimistic snapshot follow-up). with per-cell row locks or `pg_advisory_xact_lock(cell)`; counters delta-only. -- [x] Postgres tests on 55440 with concurrent probes (in #18722). Staging load run still owed; `postgres_retries` per hour drops in staging load run. -- [x] Shipped in Roll 2 (2026-09-06). Director retries first 6 h on the new image: 13 vs 85 on the predecessor's prior 6 h. -- [ ] 4.4: recalibrate the retries bar from a week of data (after 2026-09-13). - -### 4.2 Region preference -- [ ] Director: honor requested region when the preferred region has headroom, else sticky. Behind the existing flag. -- [ ] Measure with `orca_relay_runtime_metrics` region counters before/after. - -### 5.x Observability -- [x] **Relay-root runtime-metric drift**: resolved by dropping the `region` label to match live state (stablyai/orca #18734). Applied 2026-09-04 23:11Z: 8 never-applied `control_*` renewal metrics + the incident dashboard created, 0 destroyed, 21 live metrics untouched. -- [x] 5.1 `container die` log metric per cell (`relay_cell_process_exit`, applied 2026-09-04 via #18717), > 3 / 15 min, relay channel. -- [ ] 5.2 Add a paging channel (**needs owner input**: destination) to `auth_alert_notification_channels` for refresh rejections + latency. -- [x] 5.4 One dashboard (applied 2026-09-04 23:11Z): `orca_relay_cloud_sql_wal_checkpoint`, NAT drops, `orca_auth_refresh_401`, summed `controls`. diff --git a/cloud/docs/relay-improvement-roadmap-2026-09.md b/cloud/docs/relay-improvement-roadmap-2026-09.md deleted file mode 100644 index 64f33c69a70..00000000000 --- a/cloud/docs/relay-improvement-roadmap-2026-09.md +++ /dev/null @@ -1,67 +0,0 @@ -# Relay improvement roadmap (written 2026-09-04, after the auth/relay outage) - -Owner-facing list of what is left to make the relay more robust, in priority order. Evidence and history -for every item is in [`relay-reconnect-2026-09-findings.md`](./relay-reconnect-2026-09-findings.md) -(Findings 1–13). Everything already landed on 2026-09-04 is listed at the end so this file is complete on -its own. - -## 1. Finish what 2026-09-04 started (this week) - -| # | Item | Why | How | Size | -|---|---|---|---|---| -| 1.1 | **Roll all 23 cells onto the current relay image** | Every cell still runs the image that exits the whole process on a Postgres connect timeout (Finding 6). The fixed image runs only on the director and c7. Any future DB stall repeats the 200-crashes-in-48h pattern. | `cloud-deploy-relay-production-same-cap` waves, gated by the 15-min monitor. Roll inputs and canary results are in the findings doc ("Roll inputs", "Canary blast radius"). | one afternoon | -| 1.2 | **Enable the refresh_tokens pruning job** (orca-cloud #476, merged, off) | `refresh_tokens` is 63 M rows / 26 GB and grows forever; its size is what turned a slow disk into a sign-out storm (Finding 13). | Build an auth image from main (the 21:04Z deploy already contains the entrypoint: `orca-cloud-auth-00031-tox`, digest `343a0915…`), set `auth_token_pruner_enabled = true` and the image digest in `infra/terraform-apps/environments/production.tfvars`, apply targeted. First run with a small `auth_token_pruner_max_deleted_rows`. Watch the run summary's `stopReason`, not the exit code. ~48 M rows drain in ~10 days at 200k/hour. | 1 hour + 10 days of watching | -| 1.3 | **Reclaim the disk after pruning** | Deletes leave dead tuples; the 16 GB table does not shrink on its own. | `pg_repack` (or `VACUUM FULL` in a maintenance window; it takes an exclusive lock) on `refresh_tokens` off-peak, after 1.2 finishes. | 1 evening | -| 1.4 | **Full Terraform apply of the orca-cloud apps root** | The production plan carries four drifts from other merged work: `ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` env on the auth service (#476), a skill-share log exclusion filter change, skill pressure threshold 16→8, an artifacts bucket lifecycle rule. Locally it also fails on the 1Password Cloudflare data source. | Run from CI or a machine with the 1Password account; review the four drifts as ordinary changes. | 30 min | -| 1.5 | **Alert on the pruning job** | A run that only ever times out exits 0 and reads as green. | Log metric on the job's summary event where `stopReason != "complete"`, policy on the relay channel. | 1 hour | - -## 2. Remove the shared fate between auth and relay (2.1 and 2.3 this quarter; 2.2 deferred) - -| # | Item | Why | How | Size | -|---|---|---|---|---| -| 2.1 | **Private IP for Cloud SQL, `--private-ip` on the cell proxies** (do this on the existing shared instance; do not wait for 2.2) | Cells reach the database's public IP through Cloud NAT. Dynamic port allocation (landed) raised the ceiling from 64 to 4096 ports per VM, but the NAT is still in the path and its logs are still the only place port exhaustion shows up (Finding 11). | Add a private IP to `orca-cloud-auth-db` (foundation root, orca-cloud), peer the relay VPC, switch the proxy flag in the cell template, roll. | 1–2 days | -| 2.2 | **Split the relay database from the auth database** — *DEFERRED 2026-09-04 (owner decision): revisit ~2026-11-01 once pruning is done and there is a month of alert history* | One Cloud SQL instance serves `orca_auth`, `orca_relay`, `orca_push`, `orca_skills`. The auth table's growth stalled the relay for a day (Findings 10, 13). Deferral rationale: the concrete cause is fixed (disk 250 GB, WAL 16 GB, index, pruning), 2.3 + 1.1 turn a future stall into retries, and the checkpoint/disk/headroom alerts now page. Re-open if the checkpoint-loop or connection-headroom alert fires, or a large new auth-side table is planned. | New instance for `orca_relay`; migrate with a short relay drain. Relay state is small so the cutover is minutes. | 1–2 weeks incl. rehearsal on staging | -| 2.3 | **Statement timeouts on the relay pool** (the auth pool got one in #476) | A relay query stuck behind a checkpoint fsync should fail fast and let the bounded retry take over rather than hold a pool slot for seconds. | `statement_timeout` on the relay `pg.Pool` in `cloud/apps/relay`, tuned under the lease renewal deadline. | half a day | - -## 3. Make the desktop refresh path forgiving (next 2 weeks) - -| # | Item | Why | How | Size | -|---|---|---|---|---| -| 3.1 | **Refresh-token rotation grace window** | The server revokes the whole family the first time a just-rotated token is presented again. On 2026-09-04 that turned a 30 s server slowdown into 21,605 sign-outs. A short window (e.g. 60 s) where the immediately-previous token is still accepted, returning the same new token, is standard practice. | In `apps/auth/src/tokens/refresh-tokens.ts`: accept `rotated_at` within the window, return the successor instead of revoking. Keep true reuse (outside the window, or a third presentation) as revocation. | 1 day incl. tests | -| 3.2 | **Do not retry `/refresh` with the same token on timeout** | Desktop's 30 s `CLOUD_REQUEST_TIMEOUT_MS` expiring is treated like a network error and retried with a token the server may already have rotated. | In `src/main/orca-profiles/profile-cloud-session-refresh.ts`: on timeout, re-read the stored session first, and prefer a longer single attempt for the refresh call specifically. | half a day | -| 3.3 | **Un-revoke is impossible; make sign-out recovery obvious instead** | Server-side un-revoke does not help because the desktop deletes its local token on the 401. Landed: desktop notices immediately (#18694) and the phone says "desktop signed out" (#18698). | Nothing more unless we want a re-auth deep link from the phone to the desktop. | — | - -## 4. Chronic relay issues already characterised - -| # | Item | Why | How | Size | -|---|---|---|---|---| -| 4.1 | **Cell-inventory lock contention** (partial: PR #18722 narrowed the remaining non-placement sites; `assignOnce` placement lock is the follow-up) | `postgres_retries` is a global `FOR UPDATE` over the 23-row `relay_cells` table with a 1 s `lock_timeout`; it is the floor under every 503 and every slow phone accept (Findings 2, 5; memory `relay-cell-inventory-lock-contention`). | Per-cell row locks or an advisory lock keyed by cell; move capacity counters to delta writes. Verify against real Postgres on 55440. | 1 week | -| 4.2 | **Region preference is mostly inert** | Phones request an Asia cell on ~19 % of attempts and get one ~6 % of the time; the sticky lane wins silently, so Asia users ride the US path more than intended (memory `relay-region-preference-mostly-inert`). | Let a region preference override stickiness when the preferred region has headroom; measure with `orca_relay_runtime_metrics` region counters. | 2–3 days | -| 4.3 | **Desktop lease-rotation waves** | A cell recreate seeds a fleet-wide 1006/4408 reconnect burst ~54 min later, every ~54 min (Finding 3). | Jitter the desktop control lease renewal by ±10 % so the cohort spreads out. | half a day, desktop + wire-compatible | -| 4.4 | **Raise `postgres_retries` gate calibration** | The 300 bar was recalibrated (PR #18580) but should track the post-lock-fix baseline once 4.1 lands. | Re-derive from a week of `orca_relay_postgres_transaction_retry` counts. | 1 hour | - -## 5. Observability still missing - -| # | Item | Why | How | -|---|---|---|---| -| 5.1 | **Cell crash-rate alert** | 201 process exits in 48 h with no page (Finding 6). | Log metric on `container die` for `resource.type="gce_instance"` relay cells, > 3 per 15 min per cell. In `cloud/infra/terraform/relay-observability.tf`. | -| 5.2 | **Page a person for auth alerts** | Today's four auth policies (orca-cloud #475) route to the relay Slack channel only. A repeat of 2026-09-04 deserves a page. | Add a PagerDuty/phone notification channel to `auth_alert_notification_channels` for refresh rejections and latency. | -| 5.3 | **Pruning job alert** | See 1.5. | | -| 5.4 | **Dashboard that puts the four signals side by side** | Diagnosis took hours because checkpoint state, NAT drops, auth 401 rate, and fleet controls live in four consoles. | One Cloud Monitoring dashboard: `orca_relay_cloud_sql_wal_checkpoint`, NAT `dropped_sent_packets_count`, `orca_auth_refresh_401`, summed `controls`. | - -## Landed on 2026-09-04 (for completeness) - -- Auth service cap 2 → 20 (service-level manual scaling removed); Cloud SQL disk 49 → 250 GB PD-SSD; - `max_wal_size` 16384; partial index `refresh_tokens_family_unrevoked` built concurrently by hand. -- orca-cloud #474: the above in Terraform + deploy workflow; replayed dead token answers 401 without - re-revoking or re-auditing. Deployed as `orca-cloud-auth-00031-tox` 21:04Z. -- orca-cloud #475: auth alerts (refresh 401 > 100/5 min, 429 > 20/5 min, 5xx > 10/5 min, p99 > 10 s). Applied. -- orca-cloud #476: batched `refresh_tokens` pruner (disabled), auth pool `statement_timeout` 10 s, schema - DDL on an untimed connection. -- stablyai/orca #18693: both relay NATs on dynamic port allocation 64..4096 (applied US 21:01Z, Asia 21:05Z); - alerts for Cloud SQL WAL-checkpoint loop, disk > 70 %, NAT `OUT_OF_RESOURCES` drops. Applied. -- stablyai/orca #18694: desktop learns of a revoked session immediately, panes re-fetch on mount, pairing - notice says "Sign in again to use Orca Relay". -- stablyai/orca #18698: phone shows "Desktop signed out — sign in to Orca on your desktop to reconnect" via - the WebSocket close reason (only additive slot old phones tolerate). -- Director on image 519f4914; c7 on 85bf6799; other 22 cells still on the old image (see 1.1). diff --git a/cloud/docs/relay-incident-monitor.md b/cloud/docs/relay-incident-monitor.md deleted file mode 100644 index 699835b5d0e..00000000000 --- a/cloud/docs/relay-incident-monitor.md +++ /dev/null @@ -1,396 +0,0 @@ -# Relay incident monitor - -Status: monitor core and dedicated identity boundaries are implemented locally; -the targeted production bootstrap and live negative-permission proof remain -required before dispatch. - -This monitor is read-only. It probes Relay endpoints and reads Cloud Monitoring, Compute -inventory, and authenticated aggregate director status. It does not drain, restart, resize, -deploy, change admission, or write to Google Cloud. - -## Production workflow - -Run `Monitor Relay Production` manually. Choose: - -- `dry-run` for the 15-minute pre-drain gate the rehome enable and the production candidate, - multi-target, and capacity mutations consume. -- `monitor` for a 90-minute incident watch. - -Use the default `strict` migration policy for ordinary mutations. Select -`recover-forward` only after a drain attempt has durably registered migrations -and enter its exact existing-only recovery source cell. Recovery evidence tolerates -only the aggregate count of registered migrations whose target control is not -currently active for that source. Blocked or expired/unregistered migrations and every other -health threshold remain unchanged. - -Enter the exact selector generation and all three disjoint membership sets: -`existing-only`, `migration-only`, and `general`. Every configured cell must -appear exactly once. Generation zero is the only mode that reads the legacy -boolean admission field; selector-era generations use only the durable -tri-state membership. Any generation or membership mismatch freezes the gate. -The workflow verifies dependencies and restored evidence before -authentication. It has no shared-deploy fallback and accepts only the dedicated -monitor provider and service-account production environment variables. Do not -dispatch it until the targeted identity bootstrap is applied and its live -negative-permission checks pass. - -The job polls every 60 seconds and writes private aggregate evidence at 0, 5, 15, 30, 45, 60, 75, -and 90 minutes where applicable. It uploads state JSON, checkpoint JSONL, and Markdown for 14 -days. No tokens, request bodies, logs, user IDs, host IDs, or relay device IDs are recorded. -Reruns keep one stable incident ID, restore the immediately preceding private -artifact, verify its commit/run/attempt provenance and content hashes, and pass -`--restart`. A missing or mismatched artifact fails closed. A missing, stale, -or collector-failed sample is durably recorded. Up to two consecutive such -samples per source are tolerated and the window keeps running; a third resets -the active continuous window, and the next fresh sample starts a new 15- or -90-minute window under the same incident lineage. A pre-drain dry run must reach -a verdict within 35 minutes of its lineage start. - -Exit code `2` means the gate froze or a dry run failed. Missing, stale, malformed, unauthorized, or -unavailable telemetry fails closed. - -## Same-cap rolls sample inline - -`Deploy Relay Production Same-Cap` does not consume this dry-run. Each `apply` wave samples -the fleet itself right before it isolates its cell, with this monitor's evaluator, thresholds, -and tolerances, for a window sized to the hosts the drain will re-place (3, 5, or 8 minutes), -plus three lookback rules: no container exit in 10 minutes on a general or migration-only cell -other than the one being rolled, no minute with more than 500 -director 503s in 10 minutes, and director concurrency p99 within this monitor's bar over 4 -minutes. See [pre-drain fleet-health sample](./relay-workflows.md#pre-drain-fleet-health-sample). -The break-glass override that used to skip this gate for same-cap is gone with it. - -## Local use - -The active `gcloud` identity must be a service account that can mint an ID token for the exact -audience `https://relay.onorca.dev/v1/admin/drain`, and it needs read access to the monitored GCP -resources. A user account normally needs Token Creator on an approved service account. - -For a short run, an already minted JWT may instead be supplied through -`ORCA_RELAY_ADMIN_ID_TOKEN`. The token must remain valid for the whole run and is never persisted. -Use refreshable service-account credentials for the 90-minute mode. - -```sh -pnpm incident:relay -- \ - --environment production \ - --incident-id relay-incident-20260728 \ - --expected-selector-generation 2 \ - --expected-existing-only-cells production-gce-c1 \ - --expected-migration-only-cells production-gce-c2 \ - --expected-general-cells production-gce-c3,production-gce-c4,production-gce-c5,production-gce-c6 -``` - -Add `--pre-drain-dry-run --duration-minutes 15` for an ordinary gate. Add -`--migration-policy recover-forward --recovery-source-cell-id ` only -for a committed forward-recovery gate. Durable files default to -`.relay-incidents/`. If the process stops, rerun the identical command with `--restart`. A runner -gap resets the active window at the next fresh sample and preserves the prior -window evidence. A threshold freeze never clears automatically. - -A signal that reads missing or stale may miss up to two consecutive samples -without restarting the window. The sample still counts and is still checked -against every threshold it can read, and each tolerated gap is recorded in -`continuityEvents` with `tolerated: true`. A third consecutive miss of the same -signal, a failed collector, a runner gap, or any threshold breach restarts or -freezes as before. - -A production candidate or multi-target mutation must download the exact -dry-run artifact by workflow run ID and attempt. It verifies the artifact -hashes and provenance, requires a green completed 15-minute state no older -than five minutes, then -rechecks the live selector and one complete fresh sample of every safety -signal immediately before running the mutation command. -The signed state binds `strict` evidence to ordinary mutations and -`recover-forward` evidence to the exact recover-forward source; neither can -authorize the other. -The monitor and mutation jobs share one production lock. A passing dry-run is -durably marked consumed before mutation and cannot authorize another run. - -## Freeze thresholds - -| Signal | Freeze condition | -| --- | ---: | -| Active probe age | over 60 seconds | -| Cloud Monitoring data age | over 330 seconds | -| Relay log and director admin data age | over 180 seconds | -| Cell heartbeat age | over 45 seconds | -| Endpoint latency | over 2,000 ms | -| Cloud SQL CPU | over 80% | -| Cloud SQL memory | over 90% | -| Cloud SQL backends | over 320 (65% of the 490 usable of `max_connections` 500) | -| Cloud SQL waiting backends | over 20 | -| Cloud SQL deadlocks | over 0 | -| Relay pool waiters | over 800 | -| Relay pool wait | over 2,500 ms | -| PostgreSQL retries in five minutes | over 2,000 | -| Exhausted PostgreSQL retries in five minutes | over 300 | -| Director instances | outside 5–6 for more than two consecutive samples | -| Director CPU or memory | over 80% | -| Director concurrency | over 64 | -| Unexpected director 5xx in five minutes (excludes 503) | over 15 | -| Auth 5xx in five minutes | over 0 | -| Connections per cell process | over 500 | -| Queued bytes per cell process | over 48 MiB | -| Blocked or expired/unregistered migration | over 0 | -| Registered migration with inactive target | over 0, except in `recover-forward` evidence | - -Expected enabled cells must also have a powered runtime, healthy and ready endpoints, fresh -heartbeats, and matching live admission. - -Two readings are the exception to the freeze-on-first-breach rule above. - -A cell's endpoint readings are the first. Health, ready and latency are a single HTTP -round trip from one runner, so they must fail more than two consecutive samples before -they freeze the run; the streak is keyed by cell, so one cell's three readings share it. - -The director instance count is the second. Cloud Run replaces an instance in place -rather than holding the count, so the reading leaves the band for about one sample -roughly twice a day, and a deploy that briefly serves two revisions raises it the same -way. Neither is an unhealthy fleet. The minimum and maximum share one streak, so a count -that alternates above and below the band still freezes the run. - -Absorbed breaches are recorded in the state artifact under `toleratedProbeEvents`. Every -other signal, including the director and auth health probes, freezes on the first bad -sample. The live preflight that runs before each mutating wave re-samples on the same -tolerance. - -## Region placement alert policies - -Cloud Monitoring alert policies, not monitor freeze bars: these page from -`cloud/infra/terraform/relay-observability.tf` on the shared relay channel in -`relay_alert_notification_channels`, and they do not gate any workflow. All -three exist because US desktops sat on asia-east2 cells for weeks in 2026-08 -with every existing bar green. - -| Alert policy | Condition | -| --- | ---: | -| Orca Relay: far-cell phone accept latency | per cell, median 30-second `clientAcceptTotalMsP95` over 15 minutes above 2,000 ms with at least 20 completed accepts | -| Orca Relay: cell control round trip | per cell, median `controlRttMsP50` over one hour above 150 ms with at least 500 samples | -| Orca Relay: region hint skew | fleet-wide, asia-east2 share of hinted requests over one hour more than 2x and more than 15 points above its share of actual placements, with at least 500 hinted requests | - -Threshold basis: - -- Accept latency. An in-region phone accept completes in 0.3-0.6 s and a - cross-Pacific one in 5-10 s, so 2,000 ms sits outside in-region noise and - well under the far-cell floor. The 20-accept minimum keeps one slow accept - on a quiet cell off the pager. The p95 is the published value, so the - window aggregate is its median, not its max. -- Control round trip. In-region is tens of milliseconds; a US desktop on an - asia-east2 cell is 200 ms or more. Only the p50 is used. The desktop echoes - the pong on its main thread, so the published p95 and max track renderer - stalls rather than distance. 500 samples per hour is about two - continuously connected hosts at the 15-second control ping. Tuning risk: EU - desktops on us-central1 sit at 100-130 ms, so a cell whose population is - mostly European can approach the bar while correctly homed. Check where the - hosts are before reading a first breach as mis-homing. -- Region hint skew. This compares two shares of the same hour rather than - testing one absolute share, because an absolute bar is wrong at both ends. - Measured over twelve hours on 2026-09-07, while the desktop region probe - was still mis-picking: asia-east2 was 33.8% of the 33,800 hinted requests - and only 7.9% of the 45,364 assignments, a divergence of 4.27x and a gap of - 25.9 points. A fixed 40% bar would have stayed silent through that, and - once the probe is fixed the genuine APAC share climbs past any such bar and - pages forever on the correct end state. The 2x and 15-point bars sit inside - the broken state and outside a healthy one. `unhinted` requests are - excluded from the denominator: they were 27% of all requests, so a client - change that always sends a hint would move the number with no behaviour - change at all. The two bars are cross-multiplied rather than divided. An - hour that placed nobody in the region is the most extreme skew there is, - and it happens whenever the region is drained, fenced, or at capacity, but - dividing by that zero placement share makes MQL drop the row and lose the - series before any other clause runs. - -Expect the skew alert to stay lit after a client fix until the mis-homed -backlog is rehomed. Sticky assignment never re-consults the hint, so a -desktop already on an asia cell keeps being placed there whatever it now -asks for; the ratio clears only once the rehome sweep has drained. - -All three conditions are written in MQL rather than the metric filters the -other relay policies use. Every runtime metric is a DELTA DISTRIBUTION, and -the only scalar aligners a filter condition can apply to one are percentiles; -each of these alerts needs the sum of the extracted values as a volume floor, -which is `sum(value.)` in MQL and unreachable otherwise. None of the -metrics they read exists in the project yet, so what was checked against -production is the query shape: the same MQL run over existing metrics of the -same kind confirmed the distribution sum, the join arity, the unit literals, -and the condition clause. - -The skew shares are built from one log-based metric per region for hints and -one per region for placements. They read flat `requestedRegionDelta` -and `selectedRegionDelta` fields that the relay publishes as zeros in -every interval, not the nested region maps: a log-based metric would need a -quoted field path to reach a hyphenated map key, and an absent key would drop -a series out of the inner join. The region list lives in Terraform as -`relay_region_keys` and is pinned to relay-contract's `RELAY_REGIONS` by -`dev/scripts/relay-region-hint-metrics.test.mjs`. Both sides spell the field -name segments out as literal maps rather than deriving them, so the same test -compares the two declarations directly. Adding a region to the contract -without its segment is a compile error in relay-contract, not a silent gap. - -## Relay lock contention alert policies - -Two paging policies in `cloud/infra/terraform/relay-observability.tf` alert the -relay channel when one transaction holds the relay cell table long enough to -stall the fleet. `Orca Relay: cell table lock held over 1 second` fires on any -30-second runtime sample from a cell whose `cellInventoryHoldMsMax` is at least -1,000 ms, labelled with the cell's `cell_id`. `Orca Relay: lock timeout burst` -fires when Postgres cancels at least 20 relay statements in one minute for -waiting out their lock timeout. Auth traffic on the shared instance and -fail-fast refusals from background sweeps are excluded. Replayed over -2026-09-20 14:00 to 2026-09-22 15:00 UTC, the cell hold policy matched all 93 -holds from asia-east2 rehome commits, about 3.6 s each, and every one of the 88 -burst minutes overlapped one of them. First response to either alert: if a -cell hold names an asia-east2 cell, pause regional rehoming through -`Operate Relay Production Rehome` with action `pause`. - -Director holds of 1-2.5 s recur a few times a day even with rehoming paused. -They go to `Orca Relay: director cell table lock held over 1 second`, which has -no notification channel. If the burst alert fires with no cell hold in the same -minute, look at that director policy rather than pausing rehome; on 2026-09-23 -at 04:23 UTC a 2,457 ms director hold produced 25 cancels while rehoming was -paused. Do not drain or restart cells for a director hold. A cell that has -stopped reporting emits no hold sample, so these policies catch lock convoys, -not outages. - -Since the step-1 observability deploy, director holds also include the -regional target rows a drain return locks (`cellInventoryHoldMaxSite: -isolated-replacement`), so the director policy can fire during a roll's drain. -That is expected and needs no action; `isolatedReplacementHoldMsP99` is the -per-site view. The same deploy adds that lock's NOWAIT refusals and bounded-wait -timeouts to the director's `cellInventoryLockUnavailable` and -`cellInventoryLockTimeouts`, so compare those fields across the deploy only with -that site subtracted; `orca_relay_cloud_sql_lock_timeouts` is unaffected. Drain -returns run only on directors, so the paging cell policy is unchanged. - -## Implementation log - -- Gave `collector_failed` the same two-consecutive-sample tolerance as an unread - signal and raised the pre-drain lineage cap from 25 to 35 minutes - (2026-09-17). Basis: dry-run 35258662628 sampled a healthy fleet clean for - 13 minutes, then a single unreadable Cloud Monitoring sample restarted the - window, and the restarted window ran past the 25-minute cap at 1 500 002 ms, - so a healthy fleet produced no verdict. One failed collector round trip is - evidence about that round trip, not about the fleet, and it cannot freeze the - gate on its own because it carries no threshold breach. `monitor_gap` keeps - zero tolerance: it means the run stopped sampling, so the window has a real - hole. 35 minutes fits a 15-minute window plus one restart: a reset on the - window's last sample restarts at minute 16 and finishes at 31. The job - timeout is already 100 minutes. -- Recalibrated the Cloud SQL backends freeze from 250 to 320, the unexpected - director 5xx freeze from 3 to 15, and gave per-cell endpoint probes a - two-consecutive-sample tolerance (2026-09-17). Basis: the pre-roll dry-run had - frozen 39 times out of 39, every time on a chronic production condition - unrelated to the roll it gates, so it was adding delay rather than safety. - Measured over the 24 h to 2026-09-17 through the Cloud Monitoring API with the - monitor's own aggregation. `cloud_sql.backends` latest-sum per minute: p50 118 / - p90 165 / p95 212 / p99 262 / max 282, so the old bar of 250 sat under the - observed peak and tripped 1.95% of minutes and 21.8% of 15-minute gates; 320 - clears every healthy minute with 13% headroom and still fires at 65% of the 490 - usable connections, leaving 170 in hand for the runaway that exhaustion actually - is. `director.errors` non-503 5xx per rolling five minutes: p50 0 / p90 3 / - p95 5 / p99 9 / max 52, so the old bar of 3 sat on the p90 and froze 9.2% of - windows and 29.0% of gates on the chronic `/v1/assign`, `/v1/regions` and - `/v1/resolve` 500 bursts that accompany the recurring Cloud SQL stall; 15 clears - the chronic p99, drops the gate-freeze rate to 1.5%, and deliberately leaves the - exceptional 20-52 bursts detectable. The director serves roughly 50 requests a - minute in 503s alone, so a genuinely broken director lands in the hundreds per - window. For the cell probes, the asia-east2 cells run readiness as `SELECT 1` - against Cloud SQL in us-central1 over a 176 ms round trip behind a 2 s timeout, - so a saturated pool makes the load balancer answer "no healthy upstream" for - about 30 s; 7 of the last 14 freezes were that. It arrives as a real HTTP 503, so - provenance cannot separate it from a cell serving `health=0` and persistence has - to: at the 60 s interval it spans one sample and at worst two. The streak is - keyed by cell rather than by signal, because keyed per signal a cell that - alternates between slow and unanswered holds every streak at one and never - reaches the tolerance. The live preflight before each mutating wave re-samples on - the same tolerance, so a blip cannot fail a wave there either. Thresholds stay - code constants sealed into every checkpoint rather than workflow inputs, so a - green run stays auditable. Re-tighten the backends bar when the auth connection - model lands (#21165). -- Recalibrated the relay pool freezes from 30 waiters / 1,000 ms to - 800 waiters / 2,500 ms (2026-08-27). Basis, measured from - `orca_relay_runtime_metrics` (`databasePoolWaitersMax`, - `databasePoolWaitMsMax`): healthy fleet-wide bursts reach 43 waiters and - 2.03 s several times an hour (52 burst-minutes over three days), a cell - roll's reconnect surge peaks at 676 waiters, and the 2026-08-23 incident - peaked at 356 waiters without ever crossing 2.5 s — amplitude does not - separate incident from routine operation in either direction, and a - 15-minute gate had roughly one-in-six odds of freezing on a burst. The - retry signals discriminate that incident at ~10x separation and keep their - thresholds; the pool bars now fence only unbounded queueing. -- Recalibrated the Cloud SQL backends freeze from 160 to 250 (2026-08-26). - Basis, measured from `cloudsql.googleapis.com/database/postgresql/num_backends` - latest-sum over 24 healthy hours: mean ~100, 1-minute spikes to 216, with - 10 minutes over the old bar of 160 — enough to freeze roughly one in ten - 15-minute pre-drain gates on baseline noise. 250 cleared the healthy peaks - measured then and still fired well before the 400-connection ceiling assumed at - the time (the live instance measured 500 on 2026-09-16); - pool waiters and pool wait latency keep their strict thresholds. Superseded by - the 2026-09-17 entry above, which re-measured a grown baseline against the - 490-connection budget. -- Recalibrated the PostgreSQL-retry freeze from 20 to 300 per five minutes - (2026-08-26). Basis, measured from - `jsonPayload.event="orca_relay_postgres_transaction_retry"` in production - logs: healthy-day bursts reach 234/5min with zero exhausted retries and 26% - of five-minute windows over 20, while the 2026-08-23 lock-contention - incident ran roughly 2,200–3,000/5min by raw log-line count (the gate's - own `orca_relay_postgres_retries` metric read 1,510 for that window; see the - 2026-09-04 entry). -- Recalibrated the PostgreSQL-retry freeze from 300 to 2,000 per five minutes - (2026-09-04). Basis: the global `relay_cells FOR UPDATE` lock made - successful retries a steady-state rate. Measured fleet-wide (director + - cells, summed per five minutes from the `orca_relay_postgres_retries` - log metric) over 2026-09-03T05Z..2026-09-04T05Z: p50 430 / p90 924 / - p99 1,320 / max 1,504; 55% of windows over 300; only 22% of 15-minute gates - clean at 300 versus 100% at 2,000. Three read-only dry-runs on 2026-09-04 - froze on this bar (runs 33836470590, 33838698725) or on a genuine six-cell - crash storm (33837160275), blocking the same-cap roll that carries #18521 - and the `beginProof` crash guard to the 23 cells. The 2026-08-23 incident - on this metric peaked at 1,510 then 646, so retries alone no longer - separate it from today's baseline; the exhausted-retry bar (incident peak - 467 vs bar 300), director concurrency, and the pool bars carry that role. - Re-tighten after the fleet is on the 500 ms lock wait. -- Raised the Cloud Monitoring freshness bar from 180 s to 330 s and let a - freshness-only failure miss up to two consecutive samples without restarting - the window (2026-09-05). Basis: Google's metric list documents Cloud Run - `request_count`, `container/instance_count`, `container/cpu/utilizations`, - `container/memory/utilizations` and `container/max_request_concurrencies` as - "Sampled every 60 seconds. After sampling, data is not visible for up to 120 - seconds", and Cloud SQL `database/cpu/utilization`, - `database/memory/utilization`, `database/postgresql/num_backends`, - `database/postgresql/backends_in_wait` and `database/postgresql/deadlock_count` - as "up to 165 seconds", so the newest visible point is up to 180 s and 225 s - old respectively. Window-sum signals age further: `observedAt` is the newest - point in the 5-minute query window, so a label series that stops emitting - reads as 300 s old while its summed value is complete. The old bar sat under - all three. Production on 2026-09-04/05 restarted healthy 15-minute windows at - 181 s and 255 s (`auth.errors`, run 33928912676) and at 189 s - (`cloud_sql.lock_waits`, run 33944873727), and the last of those then blew the - 25-minute lineage cap at 1 500 004 ms, so a green fleet produced no verdict. - The director admin bar stays at 180 s and the nonzero lock-wait carry window - stays at 180 s; both publish on our own cadence. -- Recalibrated the exhausted-PostgreSQL-retry freeze from 0 to 300 per five - minutes (2026-09-04). Basis: #18521 cut the request-path cell-inventory - lock wait from the 1 s pool `lock_timeout` to 500 ms, so contended waiters - now fail fast (one `/v1/assign` 503 with `Retry-After`) instead of - succeeding slowly, and `orca_relay_postgres_transaction_exhausted` became - a steady contention rate. Measured fleet-wide per five minutes over - 2026-09-03T03Z..2026-09-04T02Z: 236 of 236 windows non-zero; quiet hours - p50 2 / max 36; pre-#18521 daytime p50 10 / p90 25 / max 87; post-#18521 - p50 42 / p90 147 / max 220; the 2026-08-23 incident peaked at 467. Every - pre-drain dry-run since the director deploy froze at minute one on this - bar, which blocked the cell roll that carries the same fix to the 23 GCE - cells. `/v1/assign` 503 share was unchanged by #18521 (13.9% vs 12.3%). -- Added a fail-closed state machine with latched threshold freezes, - generation-scoped checkpoint boundaries, continuity-reset evidence, cadence - accounting, restart-gap recovery, and the 15-minute pre-drain gate. -- Added Cloud Monitoring, active-probe, relay runtime, and authenticated director collectors. -- Added exact Cloud SQL instance and Cloud Run service filters, five-minute - DELTA aggregation, and serialized aggregate admin reads so monitoring cannot - load the director's three-connection database pool. -- Added private atomic state, idempotent JSONL checkpoints, and secret-safe Markdown evidence. -- Added the manual production workflow. It has not been dispatched. - -### Director error allowance (2026-09-12) - -The serving-cell rollout observed three unexpected director 500 responses among approximately 33,600 responses in an hour, all two-second PostgreSQL connection timeouts. CPU remained near 30–37% and the zero-error bar repeatedly prevented any cell mutation. The five-minute allowance was set to three non-503 director 5xx here, and was superseded by the 2026-09-17 recalibration to 15 recorded above. Auth errors, data freshness, active probes, SQL/pool pressure and other limits are unchanged. This is a bounded operational allowance, not a calibrated SLO or proof that intermittent failures are resolved; persistent low-frequency errors below this limit still require diagnosis. diff --git a/cloud/docs/relay-reconnect-2026-09-findings.md b/cloud/docs/relay-reconnect-2026-09-findings.md deleted file mode 100644 index efb23f380bd..00000000000 --- a/cloud/docs/relay-reconnect-2026-09-findings.md +++ /dev/null @@ -1,1032 +0,0 @@ -# Relay reconnect investigation: findings and evidence - -Working notes for the 2026-09-04 mobile relay reconnect incident and the cell roll that follows. -Kept current across context compactions. Newest section first. All times UTC. Host ids are log digests, -never raw ids. Nothing here is a production mutation record unless the "Mutations" section says so. - -## Status board - -| Item | State | Where | -|---|---|---| -| PR #18565 relay accept abandonment + lease jitter + desktop rotation spread + phone probe fail-fast | Open, CI fully green again after the doc move (05:45Z), CodeRabbit + Pullfrog cleared, 3 review rounds; not merged (owner has not asked) | https://github.com/stablyai/orca/pull/18565 | -| PR #18569 monitor `relayPostgresRetryExhausted` 0 -> 300 | **Merged** 2026-09-04 ~04:20Z as 4101505b6b | https://github.com/stablyai/orca/pull/18569 | -| Same-cap `verify` of c7 (read-only) | **Passed** run 33836527159 | confirms identities, selector gen 110, rehome gen 12, protocol 1, digests | -| Monitor dry-run #1 | Froze min 5: `relay.postgres_retries` 380 > 300 | run 33836470590 | -| Monitor dry-run #2 | Green to min 13, froze 04:49Z: `director.concurrency` 76.7 > 64 (six-cell crash storm, Finding 6) | run 33837160275 | -| Monitor dry-run #3 | Froze min 3 at 05:01Z: `relay.postgres_retries` 339 > 300; no crash, concurrency 5–8 | run 33838698725 | -| Owner decision 2026-09-04 ~05:10Z | **Option B approved**: "you can raise the bar. or remove it altogether ... whats the most logical move". Kept the bar (removal would leave contention unwatched during the roll) and recalibrated from measured data. | this thread | -| PR #18580 monitor `relayPostgresRetries` 300 -> 2000 | Open, awaiting CI; mutation-checked (300 fails the new test) | https://github.com/stablyai/orca/pull/18580 | -| PR #18565 CI | Was red on `root directory guard` because this findings file sat at repo root; moved to `cloud/docs/` in 8ebff89106 | | -| PR #18580 | **Merged** 2026-09-04 05:23Z as 79d5fb469a (Pullfrog cancelled by the merge; independent Opus review requested instead, per owner) | | -| Monitor dry-run #4 | Froze min 12 at 05:37:35Z: `cell.production-gce-c27.health`/`.ready` = 0. Retries green all 12 samples under the new 2000 bar. Cause: c27 (asia-east2) container died 3x 05:37:00–05:38:01Z, Finding 6 crash class. | run 33840364323 | -| Monitor dry-run #5 | Froze at sample 1 (05:41Z): c27 health/ready still 0. MIG autoheal `recreateInstance` on c27 fired 05:38:12Z after the 3 crashes; instance RECREATING, process up with 0 controls (was ~395). Second c27 recreate in 7 h (Finding 3 seed pattern). Waiting for c27 to settle before dry-run #6. | run 33841327879 | -| Monitor dry-run #6 | **Passed** 06:06:31Z: 16 samples, no freeze (started 05:47:42Z) | run 33841783747 attempt 1 | -| c7 `canary-apply` | **Succeeded.** Dispatched 06:07:15Z; drain 06:10Z; MIG recreate 06:16–06:23Z; new image listening 06:23:42Z; verify + trust proof passed; restored to `admission=general` 06:25:21Z; canary authority sealed. c7 is on `85bf6799…`. | run 33843071283 | -| PR #18581 doc reconcile (Aug 23 figure: 2,200–3,000 raw log lines vs 1,510 on the gate metric) | **Merged** | https://github.com/stablyai/orca/pull/18581 | -| Same-cap `verify` c7 target=519f4914 rollback=85bf6799, gen 112 | **Passed** (read-only) | run 33856355648 | -| Monitor dry-run #7 (gen 112) | Froze at sample 1 (09:05:31Z): `director.errors` 4 > 0, the four 2.0 s pg-connect 500s from the 09:00 cascade still inside the 5-min delta window. Dispatched 4 min too early. | run 33856521278 | -| Monitor dry-run #8 (gen 112) | Green for 15 of 16 samples (09:09:38–09:24), froze on the final sample 09:25:22Z: `director.errors` 1 > 0. The one 500 was `/v1/admin/evacuation-status` at 09:23:50Z, 2.01 s latency = director pg-connect timeout, called by **the monitor's own collector** (`incident-monitor-sources.ts:492`). First evacuation-status 500 since Sep 1. The gate froze on a request it made itself. | run 33856905229 | -| Monitor dry-run #9 (gen 112) | Froze: c13/c23 crashed 50 s after dispatch, then c14/c20/c9 at 09:34. | run 33858650691 | -| Monitor dry-run #10 | Dispatched 09:46:13Z; froze at sample 5 (09:56:59Z): `director.errors` 12. All twelve at 09:55:17–21Z, 0.8–2.1 s latency, 10 on `/v1/regions` + 2 on `/v1/assign`; c16 and c8 crashed at 09:55:19 in the same second. A single 4-second Postgres connect stall hit director and cells together. | run 33859947207 | -| Monitor dry-run #11 | Froze at sample 2 (10:08:07Z): `director.concurrency` 79.8 > 64, the c8/c20 re-dial. They crashed 10:05:54, 3 s before the waiter's quiet check passed (log ingestion lag). | run 33861578009 | -| Monitor dry-run #12 | Dispatched 10:17:38Z after 10 quiet min; froze at sample 2 (10:19:24Z): `cell.production-gce-c16.health` 0. c16 did **not** crash (no container die, MIG NONE/HEALTHY, readiness=true throughout, `/health` 200 in 230 ms at 10:21). At 10:19:07–16 it logged "control activity renewal failed" x4 and a burst of 1006 closes, sqlFailures 1 -> 14, sqlLatencyMsMax 2588: a pg stall on the old image that did not reach the unhandled path. The probe's single fetch (30 s timeout) came back unavailable during that stall and `unavailableIsZero` turned it into health=0. | run 33862504601 | -| Monitor dry-run #13 | Green 14 of 16 samples (10:48:38–11:03), froze 11:04:43Z: c9 crashed 11:04:23, c28 11:04:25 (then looped 11:05:04, 11:05:41); c15 probe also read 0 (stall, no crash). Missed by ~90 s. **Dispatched by hand 10:48:15Z** into a 43-min crash lull (last die 10:05:54; last director 500 10:31:49). The re-armed waiter never fired: its MIG-stable check used `grep -vc True`, which exits 1 when nothing matches, so `&&` short-circuited on the *healthy* case. Waiter armed 10:20Z: 10-min quiet + every MIG stable + 60 s recheck, then dispatch, then canary c7 on green. Held at 10:24 and 10:31 by lone director `/v1/assign` 500s (2 s pg-connect stalls, no cell crash). Director 500 events since 08:46: 6 (gaps 2.7/21/31/29/7.6 min). At 10:39 the waiter was re-armed with a 6-min director-500 window (the monitor's own delta is 5 min) instead of 10, since the gate only needs the 15 min *after* dispatch to be clean. Cell crashes have stopped since 10:05 (33+ min, longest gap since 08:40). 12 dry-runs: 1 pass (#6), 11 freezes, none on a real fleet-health regression. | Cascade gaps since 09:00: 31, 2.9, 5.1, 16.1, 4.0 min (median 5); a 15-min clean window is ~28% per attempt at this rate. | | -| Monitor dry-run #14 | Dispatched 11:26:53Z by the fixed waiter (first autonomous dispatch); c14, c23, c25, c15, c24, c19 died 11:30:59–11:31:08 (six cells, 13 min after the last cascade). Froze on c8 (and others) health/ready probes. Waiter re-armed 11:06Z (grep bug fixed: `grep -c` under `|| true`), same chain; held through the 11:17 cascade and c14/c28 recreates. 13 dry-runs: 1 pass, 12 freezes. Since 08:40: 10 cascades, 75 container dies, gaps 20/31/3/5/16/4/6.5/58/13 min; only 3 windows of >=17 clean minutes existed in 2.6 h, and dry-runs hit two of them (#6 passed, #13 lost the third by 90 s). | -| Monitor dry-run #15 | Waiter armed 11:33Z (6-min director-500 window, 8-min crash window, all MIGs stable), chained canary; still holding at 12:04Z. Since 11:00: 8 cascades, 98 dies, gaps 13/13.6/3.6/14.5/4.4/6.1/3.0 min, **max gap 14.5 min**, so no 15-min clean window has existed in the last hour. 14 dry-runs: 1 pass, 13 freezes. | -| Monitor dry-run #15 verdict | Dispatched 12:28:49Z; froze at sample 2 (12:30:41Z): **12 cells** health/ready = 0 at once (c4, c5, c7, c10, c15, c16, c18, c20, c22, c25, c27, c28), including c4/c5 (0 controls all day, `/health` 200 in 190 ms a minute later) and c7 (new image). Six old-image cells also crashed 12:30:02–21. This was a fleet-wide SQL stall, not a cascade: every cell's `sqlLatencyMsMax` hit 4–6 s (c7 4865, director 5140), director pool waiting 1258, 15 cell pg-connect timeouts, director sqlFailures 92. Cloud SQL CPU 0.73, backends 160, new connections normal, memory 0.46, so the *instance* was not saturated; something held the database for ~5 s. Postgres log 12:31:23–28 shows a burst of `could not obtain lock on row in relation "relay_cells"` from NOWAIT (single-row and full-inventory) sweeps, i.e. the row locks were held during recovery. Cloud SQL transactions/min flat (~30k), reads flat, -network flat: the database was neither busy nor saturated, it was *waiting*. The stall bracket -(12:30:02–12:30:41) is where every cell's SQL max hit 4–6 s at once. Lock retries in that window were -ordinary (49/29/13 per min). Best reading: a ~5 s Postgres-side wait event shared by every session -(lock on a hot row held across a long transaction, or an instance-level pause), not CPU/IO. Cell -`sqlLatencyMsMax` was already 1.5–2.2 s fleet-wide in the four minutes before, i.e. the old cells' 1 s -`lock_timeout` plus queueing. | run 33872946111 | -| Monitor dry-run #16 | Dispatched 12:38:57Z; froze at sample 1 (12:40:11Z): `cell.production-gce-c27.latency_ms` 2071 > 2000, a fifth distinct freeze signal, the probe's own round-trip absorbing a checkpoint sync. **Loop stopped by me at 12:41Z**: with the disk in the checkpoint loop (Finding 10) no bar can hold for 15 min, so further dry-runs only burn the shared rollout lease. 16 dry-runs: 1 pass, 15 freezes. Re-arm after the disk change lands. | -| Cloud SQL checkpoint loop | **Broke on its own 12:39–12:45Z**: disk writes 48 -> 4 MB/s at 12:39 with transactions and network flat and no Cloud SQL operation; 12:40:17 checkpoint synced 0.047 s; 12:45:53 checkpoint was `time`-triggered again (first since 11:55) with sync 0.096 s and write spread over 269 s. Cause of the break unknown (most likely WAL fell back under `max_wal_size` once a burst of full-page writes aged out). It can re-enter the loop on the next large checkpoint; the disk-size fix remains the durable one. | -| Monitor dry-run #17 | Dispatched ~12:49Z (all guards clean); froze at sample 1 (12:52:05Z): `director.errors` 4, from the c9/c22 crash loop that began 12:50:34, ~90 s after dispatch. Checkpoints stayed healthy (85 ms), so this is the old image's baseline crash rate, not the disk. 17 dry-runs: 1 pass, 16 freezes. | -| Monitor dry-run #18 | **Dispatched by mistake 13:48:56Z into the outage**: my gcloud credentials expired ~13:45Z, every guard query returned empty, and the waiter's `grep -c . || true` read empty as "quiet". Froze at sample 1 (13:49:43Z) on `director.ready=0`, `auth.health=0`, and cell probes; no canary dispatched, no production mutation. All waiter loops killed at 13:51Z. Lesson: a quiet-window check must fail closed when its data source errors. Waiter had been re-armed 12:53Z. | -| Gate decision | Owner asked at 09:36Z to choose: A keep looping / B recalibrate `directorErrors` 0 -> small n / C human bypass. Ten dry-runs, four froze on this bar. Recommendation B+A. Note: B alone would not have passed #9 or #10 (cell health probes and a 12-error burst); it fixes the single-500 false freezes (#7, #8) only. | | -| Batch roll | **Deferred by plan**: roll once with the lock-fix image instead of twice. | | -| PR #18606 lock removal (root cause) | **Merged** 09:2xZ as 7b108abf71 after review, fix, re-verify; CI green | https://github.com/stablyai/orca/pull/18606 | -| Image publish for 7b108abf71 | **Done** 08:36:49Z run 33854111305: `sha256:519f4914217f08cabcdcd34825965db8473ec37c6591553a3af0d65dcdeeb183` | | -| Director deploy on 519f4914 | **Succeeded** 08:45Z run 33854355791; serving `orca-cloud-relay-00570-siv`, rollback tag on 00569-ret (also 519f4914), 00565-fes (85bf6799) still deployable. Dispatched 08:37:45Z (blue/green; prior revision 00565-fes on 85bf6799 kept as rollback). Note: `predecessor-image-digest` is a required input even with bootstrap=false; pass the serving digest. | `cloud-deploy-relay-production-director.yml` | -| c7 on new image, 2 h in | 817 controls, **0 container die** since restore (was ~1 per 15 min on old image); `sqlLatencyMsMax` still 1.0 s = lock wait unchanged, which #18606 targets | | -| Terraform alert `relay_postgres_retry_exhausted` at `> 0` | Firing continuously since #18521; recalibration not done (own change) | `cloud/infra/terraform/relay-observability.tf:447,469` | - -## Mutations performed (complete list) - -1. Merged PR #18569 to main (code/docs only). -2. Merged PR #18580 and #18581 to main (monitor bar + docs). -2b. Merged PR #18606 to main (relay lock change; no serving effect until the image is deployed). -2c. Dispatched `cloud-publish-relay-production` for 7b108abf71 (builds and pushes an image; changes nothing serving). Done: 519f4914. -2d. Dispatched `cloud-deploy-relay-production-director` on 519f4914 (preserve placement, no prune, rehome gen 12). Succeeded 08:45Z; serving revision 00570-siv. Rollback: `gcloud run services update-traffic orca-cloud-relay --region us-central1 --to-revisions orca-cloud-relay-00565-fes=100` (85bf6799, still Ready). Not needed so far. -3. 2026-09-04 06:07:15Z: dispatched `cloud-deploy-relay-production-same-cap` `canary-apply` for production-gce-c7 only (run 33843071283). Completed successfully 06:26Z: c7 isolated, drained (807 controls re-dialed), template + MIG rolled to 85bf6799, verified, restored to general admission. Selector generation advanced 110 -> 112 (isolate + restore). -4. Nothing else. Both monitor dispatches were `mode=dry-run` (read-only). The same-cap dispatch was `mode=verify` (read-only, confirmed by step gates `if: inputs.mode != 'verify'` on every mutating step). - -## Finding 6 (2026-09-04 ~05:00Z): the old cell image crashes the whole process on a Postgres connect timeout - -**This is the most important open finding.** The 23 GCE cells run image `sha256:5aedbca5…` = orca-cloud -commit e3e92d95d3 (2026-08-14). In that build `beginProof` is called as `void this.beginProof(...)`. -When `verifyCellAssignment` inside it throws (pg-pool `timeout exceeded when trying to connect`, 2 s -`connectionTimeoutMillis`), the rejection is unhandled and Node exits 1. Docker restarts the container -in ~1 s, but every control on that cell (~800 hosts) drops and re-dials `/v1/assign` at once. - -Evidence, cell c7 instance 4545742188814054238, 2026-09-04: - -``` -04:46:47.951 stderr [orca-relay] control activity renewal failed (x5) -04:46:49.527 stderr Error: timeout exceeded when trying to connect - at pg-pool/index.js:45:11 - at async PostgresPoolPressure.connect (postgres-pool-pressure.js:30:20) - at async PostgresDatabase.query (database.js:645:24) - at async RelayAssignmentStore.verifyCellAssignment (assignment-store.js:2024:22) - at async HostSessionRegistry.beginProof (host-session-registry.js:376:15) -04:46:49.527 stderr Node.js v24.19.0 -04:46:49.835 dockerd: container die … exitCode=1 image=…relay@sha256:5aed… -04:46:50.258 dockerd: container start -04:46:52.761 stdout [orca-relay] listening on https://c7.relay.onorca.dev -``` - -2026-09-04 05:36:59–05:38:01Z: c27 died 3x in 62 s plus one other instance (5464389947731541178); this froze dry-run #4 on c27's health probe. - -Fleet-wide `container die … exitCode=1` on the relay image, last 48 h: **201 events on 19 instances** -(c28 x38, c29 x37, c27 x19). Hourly counts track the lock-contention curve (peak 23/h at 21Z Sep 3). -Every one has the same `Node.js v24…` crash banner. On 2026-09-04 04:46:35–04:47:41Z six cells -(c7, c8, c19, c21, c22, c25) died within 66 s: ~4,800 hosts re-dialed, `/v1/assign` returned 16,321 -503s in one minute (baseline ~20), director concurrency hit 85 (Cloud Run cap 80), Cloud SQL -`new_connection_count` 119 -> 287/min. Fleet recovered by 04:51Z. That is what froze dry-run #2. - -Fix status: `guardSessionTask` wrapping `beginProof` landed in orca-cloud #436 (2026-08-27) and is in -the target image `sha256:85bf6799…` (main 11aace8dec). The roll is the fix. Not caused by anything in -this session: the same-cap verify finished ~04:25Z and never reached a mutating step; no compute -operations exist for those instances; heap/event-loop were flat before the crash. - -Autoheal amplifier: MIG health check is `/health` every 10 s, timeout 5 s, unhealthy after 3, so a -crash loop of ~30 s+ triggers `compute.instances.repair.recreateInstance`. All ~20 recreates in the -48 h to 2026-09-04 05:40Z were the three Asia cells (c27 x6, c28 x7, c29 x8; gcloud prints local --07:00 times). c27 recreated 05:38:12Z after 3 crashes in 62 s; its ~395 controls went to 0 and the -monitor's `cell.production-gce-c27.health/ready` probe read 0 for the whole recreate (~several min), -freezing dry-runs #4 and #5. Each recreate also seeds a Finding 3 rotation cohort. Rolling the Asia -cells early in the batch phase should be weighed against the canary-first rule; c7 stays the canary. - -Implication for the gate: the monitor's `director.concurrency` freeze is *correctly* detecting these -crash storms. A dry-run only passes in a 15-minute window with no cell crash, roughly 1 in 3 windows -at current rates. Retrying in quiet hours is legitimate; the bar is not wrong. - -## Finding 5: `relay.postgres_retries` at 300 is 3x under today's baseline - -Retries per 5 min, cells + director, last 24 h: p50 579, p90 1039, p99 1398, max 1505; **65% of -windows over 300**. Quiet hours (03–08Z) p50 235, max 512. When the 300 bar was set (2026-08-26) -healthy bursts reached 234. Baseline has roughly tripled in 10 days. Skill notes say do not raise this -bar; I have not. Best odds for a clean 15 min are 02–04Z and 17–18Z (9/12 five-minute windows under -300 in each). - -## Finding 4: exhausted-retry bar was the wrong single blocker (fixed) - -`relayPostgresRetryExhausted: 0` never cleared after #18521 reached the director (22:12Z Sep 3): 236/236 -five-minute windows non-zero; post-#18521 p50 42 / p90 147 / max 220; Aug 23 incident peak 467. -Recalibrated to 300 in #18569 (merged). Dry-run #1 immediately revealed Finding 5 behind it. - -## Finding 3: the 00:50Z control-close wave was desktop lease rotation, not a rollout - -2026-09-04 00:49–00:51Z: 2,745 control closes on 19 instances; 1157/1632 code 1006 and 973/1030 code -4408 `control rebound` had ageMs in the 53-minute bin. Relay grants a flat 55 min lease; desktops -rebind 60–120 s early; so every host that (re)connected in the same minute rebinds as one cohort -forever. Seed: c27 MIG autoheal recreate 23:23Z (`compute.instances.repair.recreateInstance`) dumped -~420 controls. Harmonics at 23:55, 00:04, 00:25, 00:49Z. Each rebind is an `activateControl` -transaction that can take the inventory lock. Fix in #18565: relay lease 55 min ± 5 min (symmetric, -so mean rebind rate unchanged), desktop early window 1–6 min. - -## Finding 2: fleet-wide lock contention, worse on Sep 3 - -| window | 55P03 retries/h (cells) | cell sqlFailures/h | -|---|---|---| -| Sep 2 18Z – Sep 3 07Z | 660–1470 | 680–1620 | -| Sep 3 08Z–16Z | 3600–7100 | 3700–7700 | -| Sep 3 23Z | 7468 | 7585 | - -100% of sampled retries are 55P03; director phase is `cell-inventory`. Every cell pins -`sqlLatencyMsMax` at 1.0–1.2 s = the pre-#18521 1 s pool `lock_timeout`. Not load (controls flat -~26k, Cloud SQL CPU 46–53%). No `cloud-*` workflow explains the 08Z step. The lock is a global -`SELECT * FROM relay_cells FOR UPDATE` (23 rows) taken by assignment, control activation, activity -acquire, and sweeps, held to COMMIT. - -## Finding 1: root cause of the phone's 24 s hang (the original symptom) - -`acceptClient` runs four serialized Postgres calls; the fourth (`acquireActivity`) contends for the -global lock. Under contention the cell finishes after the phone's 12 s bound, then -`PendingHostDataReservation.bind` throws `host_data_reservation_already_bound` because the phone's -close already released the reservation. Every "first frame handler failed already_bound" line is that -post-mortem (31 events 23:06–01:01Z across 12 instances). Fix in #18565: abandon the accept after each -DB step once the socket is closed; new event `orca_relay_client_accept_abandoned {stage, elapsedMs}` -and metric fields `clientAcceptsAbandonedByStageDelta` / `clientAcceptAbandonedMsMax`. Phone side: -direct probe now fails fast on `reconnecting` so relay recovery is not queued behind three doomed -LAN redials (~3.5 s saved per foreground). #18518 (merged, not yet on the phone) covers the -stage-aware dial bound. - -Host 666077865f2e: stable throughout. 4408 rotation 00:27:45Z; 1006 quit 00:52:24Z on old adhoc; -sticky reassignment to c27 00:52:35Z on new build; rotation closes 01:44:55Z and 02:23:15Z with -splices intact. No drain/4404/wrong-cell. - -## Finding 7 (2026-09-04 ~05:10Z): retries bar recalibration basis (PR #18580) - -Chose 2000 over removal. The metric is the gate's own source (`orca_relay_postgres_retries` -log metric, director + cells summed per five minutes, ALIGN_DELTA 300 s): - -| window | p50 | p90 | p99 | max | > 300 | -|---|---|---|---|---|---| -| 2026-09-01 | 56 | 105 | 206 | 456 | 0% | -| 2026-09-02 | 109 | 186 | 294 | 377 | 1% | -| 2026-09-03 | 430 | 924 | 1320 | 1504 | 55% | -| 2026-09-04 to 05Z | 285 | 1012 | 1211 | 1211 | 44% | - -15-minute pass rate, last 24 h: bar 300 -> 22%, 800 -> 66%, 1000 -> 86%, 1500 -> 99%, 2000 -> 100%. -Aug 23 incident on this metric: 1510 then 646 (single windows), so retries no longer separate an -incident from baseline; exhausted (467 vs bar 300; healthy 72 h max 184), director concurrency, -and pool bars carry that role. Note: my earlier "p99 1398 / 65% over 300" in Finding 5 came from -raw log line counts; the metric-based numbers above are what the gate actually evaluates. -Baseline tripled between Sep 2 and Sep 3 with no deploy; still unexplained (Finding 2). - -## Decision needed from the owner (resolved: B) - -The same-cap roll is blocked only by the monitor gate, and the gate is blocked by `relayPostgresRetries: 300` -(Finding 5: 65% of windows breach it; even the 04:55Z quiet window hit 339). Three options: - -- A. Keep waiting for a naturally quiet 15 min. Odds per attempt ~1 in 3 in quiet hours, lower by day. - Each attempt is free and read-only. Could take hours. -- B. Recalibrate `relayPostgresRetries` from measured data, same method as #18569: 24 h p99 is 1398, the - Aug 23 incident ran 2200–3000, so ~1500 clears healthy windows with ~1.5–2x incident separation - (less margin than the exhausted bar had). Overrides the "do not raise" note in the skill facts. - Argument for: the roll being gated is the thing that reduces retries. Argument against: the bar is - doing its job of saying contention is high. -- C. A human dispatches the roll with a different gate policy. Not something I can or should do. - -My recommendation: B, with the number chosen from the table in Finding 5 and the roll following -immediately so the bar can be re-tightened after the fleet is on the 500 ms lock wait. - -## Finding 12 (2026-09-04 13:12Z): **INCIDENT IN PROGRESS. The auth service is at its 2-instance cap and rejecting 90% of desktop token calls with 429; the relay fleet has emptied.** - -Timeline: 13:04–13:06 the old-image cascades and NAT stalls drove ~1,400 desktops to re-dial. Their relay -JWTs (5-min TTL) expired mid-storm, so they hit `orca-cloud-auth` `/v1/desktop/auth/refresh` and -`/v1/desktop/auth/relay-token` together. The auth service is Cloud Run `maxScale=2`, `concurrency=80`, -1 vCPU throttled (`auth_max_instances = 2` in orca-cloud `infra/terraform-apps/environments/production.tfvars`, -applied by `deploy-auth-production.yml`). Both instances pinned at concurrency 85 from 13:02; from 13:07 -Cloud Run's front door returns **429 "no available instance"** (0 s latency, never reaches the container): -12,045 at 13:07, 54,292 at 13:08, 46,025 at 13:08, 42,529 at 13:09. Sep 3 total auth 429s: **0**. -Without a fresh relay token every desktop's `/v1/assign` gets 401 (1,433 distinct hosts 401'd, 0 got 200 -since 13:07) and every cell closes its control with `4401 relay authorization expired`. Fleet controls: -13,375 (12:55) -> 7,633 (13:08) -> **249 (13:12)**, splices 1. Auth container CPU 0.15–0.5, so the cap is -the limit, not the code. Every desktop is now in its refresh-retry loop hammering the same 2 instances: -this is a self-sustaining thundering herd and will not clear on its own. At 13:14Z: fleet **30 controls** -across 23 cells; successful relay-token issuance 5,000–6,500/min until 13:05, then 1,059 / 734 / 733 / -443 / 220 / 214 / 148 / **4** per minute through 13:13; auth 429s 54k -> 25k/min only because desktops -are backing off, not because the service recovered. Note `AUTH_MAX_INSTANCES: 2` is also hardcoded in -orca-cloud `.github/workflows/deploy-auth-production.yml` (lines 33–34), so a redeploy would re-pin it; -change both the workflow env and the tfvars. - -**Immediate mitigation (owner action, not applied):** raise the auth service's max instances. Fastest: -`gcloud run services update orca-cloud-auth --region us-central1 --max-instances 20` (or `10`, matching -the other apps' `max_instances = 10`), then land the same in `auth_max_instances` so Terraform does not -revert it. Auth is stateless behind Cloud SQL (`refresh_tokens` table); backends 210 of 400, so 20 -instances x a small pool is within budget. Also consider the desktop's refresh backoff: it re-dials on -401 immediately with no jitter, so a 429 storm sustains itself. - -**13:51Z status: my gcloud session lost auth at ~13:45Z; all production monitoring from this session is -blind until re-authenticated (`gcloud auth login`, interactive). Last confirmed state 13:40Z: fleet 0 -controls, auth maxScale 2, 7,600 auth 429/min. All autonomous dispatch loops are stopped.** - -**17:19Z–17:21Z MITIGATION APPLIED (owner said "fix it NOW").** State at 17:19Z, four hours in: all 23 -cells at 0 controls, auth 429 ~2,000/min, auth 2xx ~40/min, and the 2xx that got through took 13–28 s -(both instances saturated). Mutation 1: `gcloud run services update orca-cloud-auth --max-instances 20` -created revision `orca-cloud-auth-00018-4jc` (same image `auth@sha256:1710ff6c`, same env/concurrency, -only maxScale 2 -> 20) but the service pins traffic to `00023-qud` **by revision name**, so the new revision -was immediately `Retired` and nothing changed. Mutation 2 (17:21:30Z): `gcloud run services update-traffic ---to-revisions orca-cloud-auth-00018-4jc=100`. Lesson: the auth service's traffic block is name-pinned -(the deploy workflow does an explicit traffic switch), so a bare `services update` never reaches users. -Terraform still says `auth_max_instances = 2`; the next `deploy-auth-production.yml` run will revert this -unless the tfvars and the workflow's `AUTH_MAX_INSTANCES` are changed first. - -## Finding 13 (2026-09-04 17:19Z–18:10Z): **the auth outage is a database problem, not (only) a Cloud Run cap; `refresh_tokens` has 63 M rows and reuse-revokes scan whole families** - -Mutations this window (all online, no restarts, all by hand in project onorca-cloud): -1. 17:19Z `gcloud run services update orca-cloud-auth --max-instances 20` → new revision `00018-4jc`, but traffic is - pinned by revision name so it was `Retired`; 17:21:30Z `update-traffic --to-revisions 00018-4jc=100`. -2. Still 2 instances at 17:31Z: the SERVICE has its own `scaling.maxInstanceCount=2` in **manual scaling mode** - (`run.googleapis.com/maxScale: '2'` on service metadata, set by Terraform `infra/terraform-apps/auth.tf`), which - overrides the revision cap. `--scaling=auto` then `--max 20` at 17:31:45Z. Instances 2→20 by 17:38Z; 429s fell - 6,000/2 min → 60/2 min at 17:36Z and controls briefly reached 11. -3. Then latency, not capacity, became the wall: every refresh took 100+ s inside Postgres (desktop client timeout - is 30 s, `CLOUD_REQUEST_TIMEOUT_MS`), so 20 instances × 80 concurrency filled again with requests nobody was - waiting for, and 429s returned (~1,500/2 min from 17:40Z). -4. 17:27Z Cloud SQL disk 62 GB → 250 GB (IOPS ceiling 1,470 → ~7,500). 18:00Z `max_wal_size` 1.5 GB → 16 GB - (the checkpoint loop: `checkpoint starting: wal` every 45–60 s since 13:06Z). -5. 18:07Z `CREATE INDEX CONCURRENTLY refresh_tokens_family_unrevoked ON refresh_tokens(family_id) WHERE - revoked_at IS NULL` (an earlier attempt with `AND rotated_at IS NULL` was wrong for the revoke predicate; its - invalid remnant `refresh_tokens_family_live` was dropped). - -Evidence: `refresh_tokens` = 63.3 M live tuples, 16 GB table + 10 GB indexes; every refresh inserts a row and -nothing ever deletes (30-day TTL rows are never pruned). Query Insights 17:33–17:39Z: `UPDATE refresh_tokens SET -revoked_at = $1 WHERE family_id = $2 AND revoked_at IS NULL` = 21,000 s of execution per 6 min, ~90–120 k rows -updated per minute; io_time 15,000 s read; pg_stat_activity 180+ backends in `IO/DataFileRead` on that statement, -200 backends total for orca_auth (20 instances × pool max 10). `session-refresh-reuse-detected` audit events per -hour: ~100 all day → 8,805 (13Z), 15,511, 19,486, 24,897, 26,935 (17Z). Mechanism: a desktop's refresh times out -client-side at 30 s, the server had already rotated the token, the desktop retries with the same token, the -server calls that reuse and revokes the family (Bitmap scan on `refresh_tokens_family` + heap filter over every -row the family ever had), then the desktop retries the dead token again, and each retry re-runs the same -full-family scan (already-revoked families short-circuit nowhere). Reuse-detected 401 also **signs the user out** -on the desktop (`isOrcaCloudAuthFailure` → `clearCloudSessionIfUnchanged`), so every user who hit this during the -outage must sign in again. - -Durable fixes (orca-cloud PR in preparation on branch `auth-revoke-only-live-tokens`): `AUTH_MAX_INSTANCES` and -`auth_max_instances` → 20; Terraform disk 250 + `max_wal_size=16384`; the partial index in the schema; an -`already-revoked` short-circuit in `rotateRefreshToken` that skips the family UPDATE and the audit insert. Still -open after that: prune `refresh_tokens` (expired or revoked rows older than N days), a server-side statement -timeout shorter than the desktop's 30 s so the client and server agree on failure, and an alert on auth 429s. - -**19:11Z RESOLVED at the database layer.** `refresh_tokens_family_unrevoked` went valid at 19:11:17Z (build -18:07–19:11, two full table scans of 2.1 M blocks under load). Within 60 s: refresh latency 100 s → 0.1 s, auth 429 -→ 0, active orca_auth backends 200 → 2, checkpoints back on the 5-min timer (`checkpoint starting: time` at 18:35, -18:41, 19:00, 19:11). Director `/v1/assign` returning 200. Fleet controls 0 → 17 by 19:14Z. - -**Residual: mass sign-out.** 19:11–19:14Z: 3,857 refresh 401s from 3,829 distinct IPs, then near zero. Every one is -a desktop whose family was revoked by reuse-detection during the outage; the desktop clears its cloud session on -401 (`clearCloudSessionIfUnchanged`) and stops retrying. Those users must sign in again before the relay sees -them. Fresh `/session` sign-ins: 1, 5, 3 per minute at 19:10–19:12. Recovery of controls is now paced by users -signing in, not by infrastructure. Total `session-refresh-reuse-detected` events 13:00–19:00Z ≈ 100k, against a -~100/hour baseline. -**Affected-user count (19:22Z, from `refresh_tokens`):** 23,318 live token families revoked in the window, -**21,605 distinct users**. Only ~3,800 desktops had seen their 401 by 19:15Z; the rest were closed or asleep -and will find themselves signed out on next launch, so sign-ins will trickle for days. - -**Desktop UX finding (owner's own Mac, 19:22Z):** a revoked desktop keeps showing the account card as -"Connected" and the pairing pane as "Orca Relay: Unavailable" / `relay_control_not_active` indefinitely; the -local trace writes no relay events. Only quit + relaunch surfaced the sign-out prompt, after which sign-in → -relay-token → `/v1/assign` 200 (0.15 s) → working pairing, all within 10 s. Follow-ups: the relay coordinator's -401 path should flip the account card to reconnect-required immediately, and the pairing error should say "Sign -in again to use Relay" when the cause is an auth failure. Announcement wording: "If Relay shows Unavailable, quit -and reopen Orca, then sign in when prompted." - -orca-cloud PR #474 (branch `auth-revoke-only-live-tokens`): caps → 20, disk 250 / max_wal_size 16384 in -Terraform, partial index in the schema, `already-revoked` short-circuit. Do not deploy auth to any environment -with a large `refresh_tokens` before building the index concurrently there. - -**Wave 1 of the roadmap (2026-09-04 21:35Z onward):** five Opus agents in isolated worktrees: 3.1 grace window -(orca-cloud), 4.1+2.3 relay locks + pool timeout, 3.2+4.3 desktop refresh/jitter, 5.1+5.4 observability, -2.1 private IP (plan only, both repos). First back: stablyai/orca PR #18717 (crash alert + dashboard). Its key -finding: cell exits log to `cos_system` with uppercase `jsonPayload.MESSAGE` and `SYSLOG_IDENTIFIER=docker`, -so every earlier `jsonPayload.message:"container die"` count in this doc that read 0 was querying the wrong -field. Verified: 87 exits 12–13Z on the agent's filter, 0 in the last 6 h. Monitor dry-run 33922255205 -dispatched 21:41Z as the Roll 1 gate. -Dry-run 33922255205 froze at 21:46Z on `signal_missing cloud_sql.backends`. Cause: Cloud Monitoring published -no `num_backends` point for the auth instance between 21:40 and 21:46 (every other minute of the last 100 has -one; measured directly via the timeSeries API). A Google-side publish gap, not a database or monitor defect; -the monitor's freeze-on-missing rule is correct. The 12–13Z monitor failures were a different cause (active -probes reading 0 during the crash cascade). Re-dispatched at 21:50Z. -Dry-run #2 (33922844671) froze at 21:52:21Z on `auth.health observed 0` — verdict read from the state.json -artifact, not the log (the log only prints checkpoints). Auth served `/health` 200 continuously, including the -21:52:05 probe. Cause: the probe requires `/health` AND `/ready` on the first attempt; auth has no `/ready` -(404 by design), so every auth sample takes the forced 11 s retry, and on the third sample the retry fetch threw -at the network layer on the runner (no request reached Cloud Run) and `check()` recorded the exception as -health=false. Neither freeze was fleet health. Fix delegated (relay-ops: a thrown fetch is not a reading; auth -does not require `/ready`). **Sequencing constraint for Roll 1:** monitor evidence must be < 5 min old at -canary dispatch, so the owner's go must precede the dry-run, and a green dry-run must be followed by the -canary dispatch immediately. - -stablyai/orca PR #18719 (3.2 + 4.3, desktop): the replay engine was not the refresh function but -`RelayAuthCoordinator.scheduleRetry`, since `shouldRetryRelayConnectionError` treats any non-HTTP error -(including a refresh `TimeoutError`) as retryable and re-reads the same stored token on backoff. Fix: refresh -gets one 60 s attempt; an ambiguous failure (no status line) records the token and blocks re-sending it for -30 s (bounded, not permanent); definitive 5xx gets exactly one retry after re-reading the store; a 401 on an -ambiguously-attempted token logs `orca_cloud_refresh_possible_replay`. Lease renewal gets ±10 % full jitter -(base shrunk so the latest sample stays ≥ 90 s before expiry); server resets the full 55-min TTL on any rebind -(`host-session-registry.ts:736-743`) so early renewal is free. Verified the retry-path claim and both server -cites against main. - -2.1 private IP: orca-cloud PR #477 (foundation: servicenetworking API, /24 peering range 10.42.128.0, private -network on the instance, `prevent_destroy`; real production plan 3 add / 1 in-place change, staging unchanged) -and stablyai/orca PR #18720 (relay: `relay_cloud_sql_private_ip` variable, conditional `--private-ip` in the -cell startup template; default false renders byte-identical to main). Findings that change the plan: Google -states the private-IP change **restarts the instance** with no in-place path, and it is a one-way door (cannot -disable private IP or remove the network link). The director uses the Cloud Run built-in connector, not the -relay VPC NAT, so it never consumed the exhausted ports and is out of scope. Disabling public IP later breaks -the local proxy workflow and the director. #18720 merges (inert); #477 held for owner decision. - -4.1 + 2.3 relay: stablyai/orca PR #18722. Premise correction: #18521 and #18606 had already bounded and -narrowed most of the fleet-wide lock before today; what remained were the sticky-refresh retry (all 23 rows → -the one pinned row), reservation reconciliation (23 → the 2 involved rows), a dead pool-default fallback, and -an absolute counter write (→ delta with capacity guard). Placement (`assignOnce`) deliberately keeps the -ordered inventory lock: least-loaded selection is fleet-wide and dynamic target-only locking previously caused -cross-cell cycles; converting it to optimistic snapshot + conditional delta is the remaining 55P03 floor and a -follow-up. Pool `statement_timeout` was already 5 s but hardcoded; now env-configurable, `57014` added to the -retryable set (it was terminal before), schema DDL on an untimed max:1 pool. Independently re-ran the new and -adjacent suites here against 55440: 66/66. Harness note: 55440 is not idempotent across full runs (2 -pre-existing failures on a second run); reset the schema between runs. Rollout: director first, watch -`orca_relay_postgres_transaction_exhausted` and `cellInventoryHoldMsP95` before cells. - -#18719 first CI run failed only on `windows-host-job.win32.test.ts` (EPERM on temp-dir cleanup), a Windows -PTY test the PR does not touch and which no other recent run failed on; rerun dispatched rather than waved. - -3.1 grace window: orca-cloud PR #478 merged (not yet deployed; deploy is an owner gate because the startup -schema apply adds a nullable column to `refresh_tokens` with a brief ACCESS EXCLUSIVE). Semantics: within -`ORCA_CLOUD_REFRESH_ROTATION_GRACE_MS` (60 s default, 300 s cap, 0 = off) a re-presented rotated token gets the -SAME successor refresh token + a fresh access token, no revoke, no audit, provided the successor is still the -live head. Third presentation / outside window / revoked family: unchanged (revoke + audit). Successor plaintext -is stored sealed (AES-256-GCM, key = HKDF of the predecessor token; the DB never holds the key). Cost stated -plainly: a stolen token replayed inside 60 s is served once instead of tripping detection; DB-read + stolen -predecessor recovers the successor offline until pruned. Rotation now runs in one transaction (proved by a -forced-INSERT-failure rollback test; the 8-way race alone did not kill the non-transactional mutant). Verified -locally 27/27 incl. the Postgres suite against 55440, and CI ran it on PG 16 and 17 (4/4 each, not skipped). -Deploy wiring: env is set by BOTH Terraform and the deploy workflow, with a test pinning all three sources to -one value. **Pre-existing bug surfaced:** the deploy script strips every env var it does not own, so the -Terraform-set `ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` (from #476) silently reverts to the compiled default on each -release. Latent only because both defaults are 30. Follow-up: add it to `authEnvironment` + the workflow env. - -Monitor probe fix: stablyai/orca PR #18723. A thrown fetch (DNS/TCP/TLS/8 s abort) is now "no reading" and is -re-asked once after 1 s; only a second throw is `false`. A non-ok HTTP answer is still `false` with no extra -retry. `latencyMs` is the slowest answering round trip, never a sleep. `requiresReady` is per endpoint: auth -(no `/ready` by design) is judged on `/health` + latency; director and cells unchanged. No threshold or rule -touched; `auth.ready` had no consumer. 81/81 relay-ops tests and 9/9 evidence-script tests locally. The monitor -runs at `main` head, so once merged the next dry-run uses it. - -Applying #18717 (22:10Z): the cell-exit log metric `orca_relay_cell_process_exit` is created; the alert policy -raced descriptor propagation (404) and is being retried. **Not applied, deliberately:** the dashboard. Its -targeted plan drags in `google_logging_metric.relay_snapshot[*]`, and that plan is `32 to add, 21 to destroy`: -the Terraform source adds a `region` label to every runtime metric (`EXTRACT(jsonPayload.region)`) which the -live metrics do not have, and a label change on a log metric is a delete+create. Replacing 21 live metrics -resets their history and would blank the 14 existing relay alert policies during the swap. That is -pre-existing drift in the relay root (unapplied since the region work), not something #18717 introduced. It -needs its own reviewed apply in a quiet window, ideally with the runtime-metric replacement acknowledged as -intentional. Dashboard apply waits on that. - -**Wave 1 closed 22:20Z.** Merged: orca-cloud #478 (grace window); stablyai/orca #18717 (crash alert + -dashboard TF), #18719 (desktop no-replay + jitter), #18720 (private-IP flag, off), #18722 (relay per-cell -locks + pool timeout), #18723 (monitor probe fix). Applied to production: cell-exit log metric + alert policy. -Held for owner: orca-cloud #477 private IP (restart, one-way); the dashboard apply (behind the runtime-metric -label drift); the auth deploy carrying #478; Roll 1. Every wave-1 code change now sits on main un-deployed: -the next relay image build carries #18722 + #18723's monitor runs at main head already; the next auth deploy -carries #478. - -**Landing (2026-09-04 20:50Z–21:02Z, owner: "if you are confident the cloud changes are valid, you can land them"):** - -- Merged: orca-cloud #474, #475, #476; stablyai/orca #18693, #18694, #18698. Neither repo has branch - protection or environment reviewers; `verify` / `cloud-verify` green on main after each. -- Applied to production by targeted saved plans (each plan asserted create-only / exact-attribute before - apply, via `terraform show -json`): 4 relay resources (WAL-checkpoint log metric + 3 alert policies), 8 auth - resources (3 log metrics, propagation sleep, 4 alert policies), and the us-central1 NAT - (`enable_dynamic_port_allocation` false→true, ports 64..4096). Google's docs: switching to dynamic does not - break existing connections when max ≥ 1024 and max ≥ old min; only lowering max or reverting to static is - disruptive. asia-east2 NAT deliberately left for after a US soak. -- Not applied: the untargeted apps-root plan also carries 4 unrelated drifts (`ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` - env on the auth service from #476, a skill log exclusion filter change, skill pressure threshold 16→8, an - artifacts bucket lifecycle rule) and fails on the 1Password Cloudflare data source locally. The foundation - root plans clean (disk 250 / max_wal_size already match). Those drifts belong to whoever runs the next full - apps apply in CI. -- `deploy-auth-production` on main 8034955 (run 33919143723) **succeeded 21:04Z**: serving revision - `orca-cloud-auth-00031-tox` at 100%, previous `00018-4jc`, cap 20, smoke passed on both URLs. First 15 min on - the new revision: 31×200 / 1×401 on `/refresh`, max latency 56 ms, no 5xx. The new - `refresh_token_prune_cursor` table exists, so the new schema applied. -- US NAT soak (21:01–21:06Z): 0 drops, 0 proxy dial errors, 0 cell exits, port_usage 11, sqlMax ~1.07 s. - Asia NAT then applied 21:05:28Z from the pre-verified saved plan (same three attributes). The deploy script strips env vars it does not own, so the Terraform - TTL var will not be on the new revision until the full apps apply lands; the auth code defaults to 30 d. -- Terraform locally needs `GOOGLE_OAUTH_ACCESS_TOKEN="$(gcloud auth print-access-token)"`; ADC is stale. - -**Alerting + NAT follow-ups (19:58Z, superseded by the landing block above):** - -- stablyai/orca PR #18693 (`relay-nat-ports-and-sql-alerts`): both relay NATs switch to dynamic port - allocation (64–4096 per VM); new relay-channel alerts for the Cloud SQL WAL checkpoint loop (log metric on - `checkpoint starting: wal`, > 3 per 5 min), Cloud SQL disk > 70%, and NAT `OUT_OF_RESOURCES` drops. No - existing workflow applies these resources; the PR body carries the targeted plan. -- orca-cloud PR #475 (`auth-observability-alerts`): log metrics + policies for auth refresh 401 (> 100 per 5 - min; Sep 3 baseline 20–80 per hour), 429 (> 20 per 5 min; baseline 0), 5xx (> 10 per 5 min), and Cloud Run - p99 latency > 10 s. Production routes to the relay Slack channel. -- Desktop stale auth-status fix: stablyai/orca PR #18694 (`desktop-cloud-session-revoked-status`). Main pushes - an auth-status-changed IPC when a 401 clears the session; panes re-fetch on mount; the pairing notice says - "Your Orca account session expired. Sign in again to use Orca Relay" and hides Retry. StrictMode regression - test verified red on the old guard. Does not help desktops already revoked today (session cleared before - this code); it fixes every future revocation. -- orca-cloud PR #476 (`auth-refresh-token-pruning`): batched `refresh_tokens` pruner as a scheduled Cloud Run - job (revoked rows kept 30 d, rotated rows 60 d against a 30 d TTL, 5k-row batches, 200 ms pauses, persisted - cursor, per-run budget) plus a 10 s `statement_timeout` on the auth request pool with schema DDL on an - untimed connection. Merges cleanly onto #474 and does not need its index (walks the primary key; - EXPLAIN-asserted no seq scan). CI ran the Postgres integration tests for real on PG 16 and 17. Ships - `auth_token_pruner_enabled = false` in both environments: enabling needs an image digest from a build that - contains the new entrypoint. Operating rules once enabled: monitor the run summary's `stopReason` and - `deletedRows`, not the exit code (a run that only ever times out exits 0); ~48 M rows drain in ~10 days at - 200k/hour; deleting them leaves dead tuples, so the 16 GB is not reclaimed without a separate VACUUM FULL or - pg_repack pass, which is its own change. -- Phone-side copy when the desktop is signed out: stablyai/orca PR #18698 (`phone-desktop-signed-out-reason`). - Real path traced: the director resolves the phone to the host's last cell (durable assignment row), and the - cell's `acceptClient` rejects with 4404. The only additive slot every shipped peer tolerates is the WebSocket - close *reason* (relay-hello and resolve schemas are zod strict; a new close code drops old phones off the - host-offline cadence). Desktop closes its control with reason `signed-out` only when the cloud session is gone - (null context after a 401, or explicit sign-out); quit and relaunch stay reasonless. Cell remembers it per - host for the dormant-assignment TTL, forgets on re-auth, and echoes it as the 4404 close reason; phone - renders "Desktop signed out — sign in to Orca on your desktop to reconnect" with the same retry cadence. - Old×new matrix in the PR body; nothing changes for any old peer. Merges cleanly with #18694. - -## What actually blocks the roll now (12:58Z summary for the owner) - -0. **Cloud NAT ports** (Finding 11, found 12:55Z): every us-central1 cell reaches Cloud SQL's public IP - through a NAT with the default 64 ports/VM; port_usage pinned at 64 and 1,514 dropped SYNs to - Cloud SQL:3307 in one 4-min window. This is the 2 s connect stall that kills old-image cells and is - still active after the disk loop broke. Fix: `min_ports_per_vm = 1024` (or dynamic allocation) on - `google_compute_router_nat.relay_gce` in `cloud/infra/terraform/relay-gce-foundation.tf`, targeted - apply; durable fix is a private IP on the Cloud SQL instance. Online, no VM restart. -1. **Cloud SQL disk** (Finding 10): 49 GB PD-SSD saturated since 11:58Z, checkpoint loop, fleet-wide - 4–6 s stalls every ~45 s. Fix: bigger disk and/or `max_wal_size`. Owner: `stablyai/orca-cloud` - `infra/terraform-foundation/database.tf` `google_sql_database_instance.auth` (no `disk_size`, - `disk_autoresize`, or `database_flags` set today, so Terraform is at defaults: 10 GB initial, autoresize - grew it to 49 GB). Add `disk_size = 200` (+ `disk_autoresize = true`) and optionally - `database_flags { name = "max_wal_size" value = "4096" }`; production tfvars are - `infra/terraform-foundation/environments/production.tfvars`; applied by `deploy-production.yml` in - that repo. Online, no restart for disk; `max_wal_size` is also a non-restart flag. Note Terraform - `disk_size` below the live 49 GB would be a destructive shrink, so 200 is safe and 49 is the floor. **This is now the first thing to do**; nothing else can pass a - 15-min gate while it persists, and it is also what is killing the old-image cells several times an hour. -2. **Old cell image** (Finding 6): dies on every stall. Fixed by rolling 519f4914 (canary inputs ready). -3. **Gate policy**: `directorErrors: 0` and per-cell health probes freeze on any single stall. Recalibrate - after 1 and 2, or bypass by hand for the canary. - -## Plan agreed with the owner (2026-09-04 ~06:45Z), in execution order - -Owner: "feel free to improve operations to make things more effective ... continue driving everything e2e -until this process is complete." Owner has had multi-day experiences with cell rolls and does not want a -9-hour sequential roll. - -1. **Lock-removal PR** (root cause). *Status 08:55Z: pushed as branch `relay-single-row-reservation` - (2 commits). Opus adversarial review found one real defect: `acquireActivity` moving a client-chosen - activity id across cells locked the old cell's row before the new one, cycling with placement's - ascending inventory lock (reviewer reproduced it as paired 55P03s on real Postgres; no 40P01 because - lock_timeout == deadlock_timeout == 1 s). Fixed with `lockCellRows` (ordered, 500 ms bound); census now - fails on any inline `relay_cells FOR UPDATE` outside the named helpers. Three-cell Postgres test moves - an activity high->low while the target row is held; 5/5 revert-mutants fail it. 480 SQLite tests + - tsc green. Also fixed a pre-existing test leak (`relay_cell_connection_snapshots`) that made - `assignment-control-supersession-postgres` fail on reruns. Reviewer re-verified 65569be3de: cycle - repro completes in 7 ms (was 1022 ms + paired 55P03); no remaining out-of-order pair in the store; - flagged two evasions in the new census guard, closed in the third commit (whole-statement scan, - covers query() too, mutation-checked with both evasions). Headroom Postgres test's one failure is - pre-existing on main (verified by swapping in main's store).* Make `activateControl` superseded-control cleanup, `acquireActivity` - existing-lease branch, and `changeActivity` use the existing single-row - `adjustCellReservationAtomically` instead of the 23-row `lockCellInventory`. Keep the global lock only - for placement (`resolve`/assignment) and sweeps. Real-Postgres contention test on port 55440. -2. **Faster same-cap rollout workflow.** (a) paced drain instead of `graceMs: 0` so a cell's ~800 hosts - re-dial over minutes, not one second (director cap is 5 x 80 = 400 in-flight); (b) cells in a batch run - in parallel once drains are paced; (c) post-canary batches use a short freshness check instead of a new - 15-min dry-run, since the in-job safety recheck already runs before each drain; (d) job timeout > 75 min. - Target: 22 cells in ~6 batches x ~25 min. -3. **Build image** with (1) merged, then one roll of the fleet with (2). Asia cells c27/c28/c29 first. -4. Re-tighten the monitor retries bar; recalibrate the Terraform exhausted alert. -5. Consider deleting the 55-min control lease rebind entirely (no recorded reason; liveness is the 75 s - watchdog + 90 s activity lease). Separate PR after (1) so its effect is measurable. - -## Faster same-cap rollout: design (step 2 of the plan), from reading the real limits - -What actually bounds parallelism today (measured on the c7 canary, run 33843071283): - -| step | c7 duration | bound by | -|---|---|---| -| prechecks (recheck, backend init, resolve, verify) | 43 s | none | -| isolate + drain + transition wait | 7 min | drain is `graceMs: 0`; `verify-relay-capacity-transition --activity restart-safe` polls until leases drain | -| Terraform template + MIG recreate + wait-until stable | 8 min | GCE recreate; per cell, independent | -| verify new incarnation + trust proof + restore | 1.5 min | none | - -Real constraints: (1) the director is 5 x 80 = 400 in-flight `/v1/assign`; a `graceMs: 0` drain of ~800 -hosts pins it at cap for ~2 min (observed 79.75/84.75 p99). (2) `production-cloud-sql-rollout` lease and -workflow concurrency group serialise the whole run, by design, and the per-cell job shares it via -`holder-key`. Nothing else forbids parallel cells. - -Changes, smallest first: -1. **Paced drain.** `HostSessionRegistry.drain(graceMs)` already sends `drain {graceMs}` and closes each - session after `graceMs`, but the desktop's `handleDrain` re-dials immediately regardless of graceMs - (`relay-origin-pool.ts:150-162`), so graceMs only delays the *close*, not the stampede. Fix on the - cell: stagger the drain *send* across sessions over a window (e.g. 800 sessions over 120 s = ~7/s), - which needs no desktop change and works for every desktop version in the field. New admin body field - `spreadMs` (optional, default 0 keeps today's behaviour); canary script passes `spreadMs: 120000`. - Requires the cell to be on an image with the change, so it applies to batches after the first - post-lock-fix roll, not to this one. -2. **Parallel cells in a batch.** In `cloud-deploy-relay-production-same-cap.yml` make `cell_2..cell_4` - `needs: [gate]` instead of chaining, gated on the same evidence (drop the `+75 min x wave-index` - allowance, it exists only because of chaining). Each job already takes the rollout lease with the - run's `holder-key`, so they re-enter it rather than fail. With paced drains, 4 cells x ~800 hosts - over 120 s is ~27 dials/s, well under the director cap. Raise `timeout-minutes` to 90. -3. **Post-canary batches skip the 15-min dry-run.** The in-job "Recheck aggregate SQL, pool, - reconnect, migration, and selector safety" step (`pnpm incident:relay-preflight`) already runs a - live one-shot check before each drain. For `batch-apply` with a sealed `canary-run-id` from the - same commit, accept a dry-run of any age (the canary's) plus that live recheck; keep the 15-min - requirement for `canary-apply`. Change lands in `relay-monitor-evidence.mjs verify-authority` + - `relay-production-same-cap-wave.mjs` + their node:test suites. - -**Correction after reading the cell job (07:35Z):** (2) parallel cells is not a flag flip. Each cell job -asserts the exact selector generation `expected + 2 x wave-index` and exact memberships derived from -predecessors having completed (`ISOLATED_*`/`RESTORED_*` in the job, `applyExactAdmissionSelector` -compare-and-swap), and all cells share one Terraform state lock. Making that concurrent means a batch-level -isolate/restore in the gate and a rewrite of the 650-line job's expectations. That is the multi-day trap -the owner described. Deferred. - -What is cheap and removes most of the wall-clock: (3). The per-batch 15-min dry-run costs 15 min each -*and* fails ~50% of the time on old-image crashes, which is where hours go. Implement: `batch-apply` with a -verified canary authority accepts a passed dry-run up to 6 h old and may re-use one already consumed -(the consumed-marker check exists to stop replaying stale evidence; the canary binding plus the in-job -live preflight at drain time replace it). Files: `relay-monitor-evidence.mjs` (`--after-canary`), -`incident-live-preflight-cli.ts` (same flag), the same-cap workflow + job, and both test suites. -Revised expectation: 22 cells = 6 sequential batches x ~70 min = ~7 h wall-clock but *unattended-safe* -and with one dry-run total, versus today's 6 dry-runs at ~50% each. (1) paced drain rides the lock-fix -image. - -## Recommended next steps (superseded by the plan above; kept for history) - -1. Resolve the gate decision above, then: monitor dry-run -> c7 `canary-apply` only -> verify -> stop. - Each rolled cell leaves the Finding 6 crash class. -2. Merge #18565; publish; a later same-cap roll carries it to cells. -3. Remove the global inventory lock from per-connection paths (`acquireActivity` existing-lease - branch, `activateControl` superseded-control cleanup, `changeActivity`) by using the existing - `adjustCellReservationAtomically` single-row update. Own PR, after the roll. -4. Recalibrate the Terraform alert `relay_postgres_retry_exhausted` to 300/300 s (observability root). -5. Whether to raise `relayPostgresRetries` is a human call; the data is in Finding 5. - -## Canary blast radius (read before dispatching c7) - -- What `canary-apply` does to c7, in order: isolate (selector -> migration-only, no new - assignments), `/v1/admin/drain graceMs:0` (every control on c7 re-dials the director and is - reassigned), Terraform template + MIG update to the target image, wait stable, verify new - incarnation + exact digest + protocol, prove per-host trust, restore c7 to general admission. - On any failure c7 is left isolated (migration-only) with rehome disabled; nothing else is touched. -- c7 at 05:20Z: 788 controls, 5 splices, 800 connections. So ~790 desktops re-dial once. The fleet - already absorbs this exact event 201 times / 48 h uncontrolled (Finding 6); the controlled version - isolates first, so no new assignment lands on c7 mid-roll. Expect a director concurrency blip, not - a freeze-class one (six cells at once gave 85; one cell should stay well under 64). -- Precedent: the identical workflow (pre-move, in orca-cloud) ran 9 successful `apply` canaries and - batches on 2026-08-27 (last: c20 -> 5aedbca5). Its failures that day all stopped at the read-only - "Recheck aggregate SQL..." or "Require durable rehome disabled" step, before `MUTATION_STARTED`. - The moved copy in this repo has one run: the read-only `verify` of c7 (passed, including WIF auth). -- c7 side note: MIG autoheal recreated the c7 instance four times on 2026-09-01 08:02-08:42 PDT - at ~13 min spacing. Same crash class as Finding 6 (health check failing during restart loops). - -### Canary observed effect (c7 drain, 2026-09-04 06:10Z) - -- c7 807 controls -> 0 between 06:08:52Z and 06:10:52Z. Director `/v1/assign`: 200s 32 (06:09) -> 2628 (06:10) - -> 340 (06:11); 5xx 1969 (06:10) -> 31 (06:11). Director max-concurrency p99 7.9 -> 79.75 (06:10) -> 84.75 - (06:11), i.e. at the Cloud Run cap of 80 for ~2 min. My pre-dispatch estimate ("well under 64") was wrong. -- Confounder: c10 (us-central1, instance 2803000337345335589) crashed 06:09:56Z on the old-image class - (Node.js banner + container die), so ~1,600 hosts re-dialed in the same minute, not ~800. Coincidental; - the fleet has one of these every ~15 min. -- Recovery: 06:13 903 / 06:14 1471 assign 200s from 640 distinct desktop IPs; 503s 78 -> 183 -> 29/min. - No cell crash 06:12–06:16Z. Drain step passed ~06:16Z; template/MIG apply started. -- 06:16:03–06:17:08Z, during c7's template apply (not its drain): c27 (x4) and c29 (x3) crash-looped on the - old-image pg-pool connect timeout in `beginProof`, both MIGs autoheal-recreated (c27's second recreate in - 40 min). Fleet 23 -> 21 reporting cells, controls 13286 -> 12462, assign 503s 1000/min at 06:17, director - concurrency p99 74.8. Cloud SQL CPU 0.70 max, backends 174 max (bar 250). Same multi-cell pattern occurred - at 01:31Z (4 cells) and 04:47Z (5 cells) with nothing rolling; the c7 drain's SQL load 6 min earlier may - have nudged the pool timeouts but the class is pre-existing. c7 MIG RECREATING onto new template - `…20260904061618…` = the expected image swap. -- 06:20Z: 849 assign 503s. Closes 06:19:30–06:21: 162x1006 age<5min (hosts bouncing off the recreating - c27/c29), 73x4408 + 53x1006 in the 50-min age bin (Finding 3 rotation cohort). Not roll-caused. - c7 MIG `recreating=1` on the new template since 06:16:18Z; c27 and c29 MIGs also RECREATING (autoheal). -- 06:23:16Z c7 instance restarted in place (MIG RECREATE keeps name/id relay-c7-bwjc / 4545742188814054238), - pulled `relay@sha256:85bf6799…` 06:23:37Z, listening + readiness true 06:23:42Z. Apply step passed 06:24Z; - verify step running. Isolate -> ready on new image took ~14 min end to end. -- Post-restore c7 on new image (06:25:42–06:26:42Z): controls 143 -> 273 -> 377 refilling, sqlQueries - ~1,500/30 s, `sqlLatencyMsMax` 518 -> 1003 -> 1155 ms, still 55P03 `cell-inventory` retries. So the new - image alone does not remove lock waits; the request-path 500 ms cap from #18521 applies to the director's - paths, and cell-side `acquireActivity`/`activateControl` still ride the global lock (step 3 in next steps). - Watch: does c7's sqlLatencyMsMax settle below the old 1.0–1.2 s pin once refill finishes, and does c7 stop - appearing in `container die` (the real win: guardSessionTask). -- 08:25Z (2 h after restore): c7 817 controls, 0 crashes since 06:25Z. Fleet crashes last 2 h: c27 x6, - c28 x5, all old-image Asia cells. The new image stops the crash class as predicted; it does not move - lock latency (c7 sqlLatencyMsMax 1005 ms), which is #18606's job. -- Implication for the batch phase: every drain will push director concurrency past the monitor's 64 bar - for ~1-2 min. The batch job rechecks safety *before* it drains (read-only step), so that is fine per wave, - but never run a monitor dry-run concurrently with a wave, and prefer batches of 2 over 4 until the fleet - is on the new image and the crash class is gone. - -## Post-merge dispatch plan for #18606 (image -> director -> cells) - -1. `gh workflow run cloud-publish-relay-production.yml --ref main -f mode=publish` (after the squash lands - on main). Resolve the digest by tag, never by parsing the log (it mixes relay and fence-broker digests): - `gcloud artifacts docker images describe us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay:sha- --format='value(image_summary.digest)'`. -2. Director: `gh workflow run cloud-deploy-relay-production-director.yml --ref main -f image-digest= - -f regional-placement-mode=preserve -f prune-incompatible-revisions=false -f expected-rehome-generation=12 - -f bootstrap-runtime-identity=false -f predecessor-image-digest=` - (no monitor evidence needed; requires rehome disabled at gen 12, which it is). Last run 33826514754 used - the same shape. Watch director `orca_relay_postgres_transaction_retry` per minute before/after. -3. Cells: same-cap `verify` c7 with target=, rollback=85bf6799; fresh dry-run; `canary-apply` c7; - then batches (3 per batch, Asia c27/c29/c28 first). Each batch: new dry-run unless the batch-reuse - change (design section above) has shipped. - -## Finding 8 (2026-09-04 08:40Z): ten-cell crash cascade during the director deploy, not caused by it - -Timeline: candidate revision 00570-siv created 08:38:39Z, first log 08:39:20Z; traffic still 100% on -00565-fes through 08:43 (assign logs by revision). Cell crashes: c28 (5031087219978409220) looped 08:37:55– -08:40:07 (9x), then at 08:40:20–08:40:45Z **ten** instances died within 25 s (c10 2803…, 5110…, 532…, 5464…, -7536…, 7726…, 8671…, 8928…, 8966…). All old-image `beginProof` pg-pool timeouts. Fleet controls 13,423 -> -6,157 by 08:43; assign 503s 3,912 (08:42) and 4,624 (08:43) per minute, director concurrency 85 (cap 80), -Cloud Run autoscaled 5 -> 10 instances, Cloud SQL CPU 0.55 -> 0.99. Deploy finished cleanly at 08:45Z with -the new director taking the tail of the storm; by 08:46 503s were ~30/15 s, controls 7,913 and rising, -director lock retries 29/min (vs 105–157/min pre-deploy) and exhausted 2/min (vs 65/min at 08:36). -Same class as 01:31Z (4 cells) and 04:47Z (5 cells) today; this was the biggest. c7, on the new image -since 06:25Z, did not crash. What triggered the pool timeouts fleet-wide at 08:40 is not established; Cloud -SQL CPU was 0.78–0.88 in the minutes before, the highest of the day, so the cells' 2 s connect timeout is -the plausible tipping point under a busy database. Every cell still on 5aedbca5 remains exposed to this. - -## Finding 9 (2026-09-04 08:56Z): #18606 on the director cut lock retries ~10x - -`orca_relay_postgres_retries` per 5 min, director only: 08:21–08:41 windows 419–689 (old image, incl. the -crash storm); 08:46/08:51/08:56 (new image 519f4914, refilling ~7k hosts): **61 / 69 / 54**. Exhausted: -104–178 -> **11 / 14 / 12**. Inventory hold p95 ~200 ms, max 255 ms, ~366 holds/min. Cells (still old -image) 17–44 -> 0–3, because the director no longer holds the 23-row lock on their behalf. This is the -first direct measurement of the root-cause fix under real load. Cloud SQL CPU peaked 0.99 during the -cascade and is decaying (0.86 at 08:55); the monitor freezes above 0.80, so no dry-run until it clears. - -Fourth cascade 09:00:12–09:00:18Z: c23, c8, c16, c26, c22 (five cells, 11 container-die events in 6 s, -all `5aedbca5`, exitCode 1, Node banner, pg-pool `client closed the connection` burst right before). Cloud -SQL CPU 0.84 -> 0.78 in the preceding minutes, director concurrency 18–22 (idle), so this one fired -*without* a database or director spike. Fleet had just recovered to 13,015. Cadence today: 01:31 (4), -04:47 (5), 08:40 (10), 09:00 (5), 09:31 (c13, c23), 09:34 (c23 again, c14, c20, c9; c14/c20 crash-looping), -09:39 (c21, c24), 09:55 (c16, c8), 09:59 (c20), 10:05 (c8, c20), 10:19 (c16 stalled, no crash), then a 58-min -lull, 11:04 (c9; c28 died 13x in 4 min, autoheal recreate 11:09Z, its 3rd recreate today), 11:17 (c10, c28 -again, c22, c23, c14 x9 looping; 23 dies in ~90 s; fleet 13.3k -> 10.8k), 11:31 (c14, c23, c25, c15, c24, c19), 11:34 (c20, c26, c29 x4, c14, c27 x3, c25; fleet 13.1k -> 10.3k). -Three cascades in 17 min. 11:38–11:45 c27 crash-looped 17x and c28 4x (Asia cells), c29 recreating. -11:59 (c21, c9, c10, c23), 12:02 (c19; 4,109 assign 503s that minute, mostly hosts bouncing off the -recreating cells, code 1006 age<5min x217), 12:09–12:12 (c19, c27 x6, c28 x5, c13, c22, c15, c26, c14; -8 cells, c27/c28 recreating again). Cloud SQL CPU 0.62–0.85 through it. 12:20 (six more cells). Cascade -cadence since 11:00 is now ~every 8 min; the waiter has held correctly the whole time and there has been -no dispatchable window. Loop continues unattended; findings stop logging each cascade from here unless the -class changes. Every cell that has died today is on 5aedbca5; c7 (85bf6799, -5.5 h) has not. Cell dies per hour today: -01Z 5, 02Z 7, 03Z 4, 04Z 7, 05Z 4, 06Z 9, 07Z 11, 08Z 30, 09Z 26, 10Z 2, 11Z 68+ (to 11:42). -Director concurrency pinned at 85 for 09:32–09:33; 503s 4,141 and 4,396 per minute. 09:39: c21, c24 -(2,870 503s). Crashes per instance 08:10–09:40Z: c28 x14, c27 x5, c23 x5, c22 x4, c14 x4, c13/c20 x3, -then c26/c9/c24/c16/c8 x2. Mean gap between cascades since 08:40: ~12 min. Every 15-min gate attempt -now has well under even odds; the c7-style canary that ends this needs a gate it can pass. The old image is now cascading roughly hourly regardless of load; the -only cell on a fixed image (c7) has 0 crashes in 2.5 h across all four. - -Director 500s: 4 in the 09:00 window, all 2.0 s latency on `/v1/assign` or `/v1/resolve` = pg-pool connect -timeout surfacing as a 500. Pre-existing (Sep 3: 03h/08h/16h one each, same 2.0 s shape; 06:09Z today on -the old image during the c7 drain). The monitor's `directorErrors: 0` bar freezes on any of these, so a -dry-run needs a 15-min window with none; at ~1 per cascade that is a real but modest constraint. - -**Gate observation (09:26Z):** `directorErrors: 0` counts every non-503 5xx on the director, including -the monitor's own admin calls. The director on 519f4914 still sees an occasional 2.0 s pg-pool connect -timeout (~1 per 20 min under today's Cloud SQL load), which surfaces as a 500 on whichever request drew -it. Two consecutive dry-runs (#7, #8) froze on exactly this: one, isolated, 2 s 500. That bar was set for -"unexpected director 5xx"; a single connect timeout that the client retries is not an incident. Candidate -recalibration (own PR, not done): `directorErrors` 0 -> 2 per 5 min, or exclude the monitor's own -user-agent. Not changing it unasked; noting that at ~3 per hour the 15-min gate passes ~1 in 2 attempts. - -**Did the director deploy make cells crash more? (checked 09:45Z)** Cell `container die` per 30 min: -06:00 9, 07:00 2, 07:30 9, **08:30 30** (director candidate 08:38, traffic 08:43–08:45; the 10-cell burst -was 08:40:20, before the move), 09:00 11, 09:30 12. Per hour today 05:4 06:9 07:11 08:30 09:23 vs Sep 3 -same hours 2/7/8. So today is 2–3x worse than yesterday and was rising before the deploy; after the deploy -it is ~11–12 per 30 min, in line with 06:00–07:30. Cloud SQL backends (~230 max) and new connections -(~5k/30 min) are flat across the deploy. Latest crash (c21 09:39:11) is `Connection terminated due to -connection timeout` with cause `Connection terminated unexpectedly` in `verifyCellAssignment` <- -`beginProof`, the same unhandled path. Conclusion: no evidence the deploy worsened it; the old image's -crash rate simply climbed all day. Director lock retries stayed ~10x lower after the deploy. - -**Checkpoint-phase check (10:00Z, negative result):** Postgres checkpoints complete every 5 min at ~:07. -Cell crashes bucketed by phase within that 5-min cycle show a mild :00–:29 s cluster today (22 of 103) -that is absent on Sep 3 (7 of 114), so checkpoints are not the trigger. Disk write bytes in cascade -minutes are at or below the median except 09:00. Cloud SQL memory 0.47, transaction rate flat. The -09:55 stall (11 director + 4 cell pg-connect timeouts in the same 4 s) came with `could not obtain lock -on row in relation "relay_cells"` from a NOWAIT sweep at 09:55:36, i.e. someone was holding the full -inventory at that moment. On the new director that can only be placement or a sweep; on the old cells it -is still every rebind. What stalls *connections* (not locks) for 2 s fleet-wide remains unexplained; -Cloud SQL is `db-custom-4-15360` REGIONAL PD_SSD 49 GB at 0.5–0.75 CPU when it happens. - -**Stall census (10:01Z):** 33 pg-connect-timeout stall events today (clusters of timeouts < 20 s apart). -Before 08:35 they were 1–9 timeouts each and 10–60 min apart; from 08:35 the big ones are 16, 22, 21, -17 timeouts and 5–30 min apart. No second-of-minute phase (start seconds spread across all buckets), so -not a fixed timer. Cloud SQL backends by state at 09:55: active peaked 42 at 09:52, idle-in-transaction -≤ 10, nothing near the 400 ceiling; memory 0.47; disk normal. Each stall is a few seconds where *new* -connections to Cloud SQL (via the auth proxy socket) time out at the 2 s `connectionTimeoutMillis`, -hitting every process that happens to need a fresh pool connection in that window. Old-image cells die -on it (unhandled), new-image director logs a 2 s 500 and continues. Root cause of the stall itself is -outside the relay code (Cloud SQL proxy or instance); not chased further here. - -## Finding 10 (2026-09-04 12:40Z): Cloud SQL disk write saturation since 11:58Z is driving the stalls - -`orca-cloud-auth-db` is `db-custom-4-15360` on a **49 GB PD-SSD** (81% used). PD-SSD performance scales -with size: 49 GB gives roughly 1,470 write IOPS and ~23 MB/s write throughput. Measured: - -| | before 11:58Z | 11:59Z onward | -|---|---|---| -| disk write MB/s | 4–6 | **30–50** (over the ~23 MB/s cap) | -| disk write IOPS | 500–800 | 800–1,475 (at the ~1,470 cap in 11:59, 12:15, 12:24, 12:34) | -| checkpoint `sync=` | 0.07–0.2 s (Sep 3 max 0.65 s, 290 checkpoints) | 2–20 s; 27 of 39 checkpoints in 12Z were >= 2 s | -| checkpoints per hour | 12 (timed, every 5 min) | 39 (WAL-triggered, every ~45 s; `write=` fell from 270 s to 30 s) | -| Cloud SQL CPU / memory | 0.5–0.8 / 0.47 | same (not the bottleneck) | - -Every 4 s+ fleet-wide SQL stall since 11:04 (11:04, 11:17, 11:31, 11:34, 12:09, 12:10, 12:18, 12:20, -12:30) sits inside a slow checkpoint `sync` window; the 12:30:49 checkpoint synced 5.88 s (longest file -5.47 s), matching the 12:30:02–41 stall. During fsync the WAL writer stalls and every session waits, which -is why the stall hit all 23 cells and the director at once regardless of the relay lock changes. The -old-image cells then die on the pool timeout; the new image survives. What raised write volume ~8x at -11:58Z is not established (autovacuum ran on every relay table 11:55–11:57 and checkpoints are being -forced by WAL volume, so a write amplifier inside Postgres is the leading candidate; relay transaction -rate and Cloud SQL network bytes were flat). This is the first cause found today that is *upstream* of -the relay code and it explains the afternoon acceleration (11Z 68 dies, 12Z 47 by 12:34). - -Corrections after digging (12:45Z): relay query volume, renewals, reconnects, and assignments per 5 min -were **flat** across 11:58 (sqlQ ~330k, renewals ~115k), so the relay did not start writing more. WAL -recycling per checkpoint went 7 -> 10–11 files (16 MB each) at 45 s intervals, i.e. WAL output rose from -~0.4 MB/s to ~4 MB/s while data-file writes rose to 30–50 MB/s; checkpoints switched from `time` to `wal` -triggered at 11:58:24. No Postgres slow-statement or "checkpoints too frequently" lines. This is -write amplification inside Postgres (full-page writes after each of the now-frequent checkpoints on -hot pages, plus autovacuum on every relay table each minute) on a disk too small for its IOPS ceiling, -not new relay load. Instance label `managed_by=terraform`, created 2026-07-09; the instance resource is -**not** in `cloud/infra/terraform` (only the database, user, and secret are, via -`local.relay_database_instance_name`), so it lives in the other Terraform root (orca-cloud, per -[[orca-cloud-terraform-split-findings]]). `storageAutoResize=true` with limit 0, so Cloud SQL will grow -the disk only when it fills, not when IOPS saturate; disk is 81% full. - -Onset precisely: the 11:55:37 `time` checkpoint wrote 67,258 buffers (10.5% of shared_buffers, the -day's largest) over 163 s and completed 11:58:24. Every checkpoint since has been `wal`-triggered at -~45 s spacing (`max_wal_size` reached), each writing 13–20k buffers with 9–11 WAL files recycled. This is a -self-sustaining loop: a checkpoint completes -> every subsequent write to a hot page emits a full-page -image into WAL -> WAL fills `max_wal_size` in ~45 s -> next checkpoint -> repeat. The relay's hot rows -(`relay_cells`, `relay_assignments`, activity leases, cell runtime) are updated tens of thousands of -times a minute, so full-page-write amplification is large. Before 11:58 the 5-min timed checkpoints kept -WAL well under the limit; a one-off larger checkpoint tipped it over and the disk's write ceiling keeps -it there. Query Insights: io_time +30% in the 12:00 bucket, lock_time flat. - -**Owning workflow / mitigation (not applied):** raise the Cloud SQL data disk (PD-SSD IOPS and MB/s scale -linearly with GB; 49 -> 200 GB roughly quadruples the ceiling, online, no restart) in the Terraform root -that owns `google_sql_database_instance` for `orca-cloud-auth-db`, applied through that root's workflow. -A second, flag-level lever is raising `max_wal_size` (default 1 GB) so timed checkpoints resume; that is -also a Cloud SQL instance setting in the owning Terraform root. Per the standing rule, not applied from -this session. Until then the fleet-wide 4–6 s stalls recur on -every slow checkpoint sync, the old-image cells die on each one, and no 15-min gate window will exist. - -## Finding 11 (2026-09-04 12:55Z): **Cloud NAT port exhaustion** on the us-central1 cells is the second stall class - -`google_compute_router_nat.relay_gce` (us-central1, `AUTO_ONLY` IPs, no `min_ports_per_vm`, no dynamic -port allocation, i.e. the default **64 ports per VM**). `router.googleapis.com/nat/port_usage` per VM -hit **64 = the cap** in exactly the minutes the cells' Cloud SQL proxies logged `dial tcp -35.188.82.89:3307: i/o timeout` (12:20–12:22, 12:41–12:43, 12:51–12:53), and -`nat/dropped_sent_packets_count` went 0 -> 56/552/590, 82/272/133, 395/1565/1842 in those same minutes. -Hourly: port_usage max was 25–50 all of Sep 3 and until 10Z today, 64 in 11Z and 12Z; dropped packets 0 -until 11Z (219), then 5,491 in 12Z. Open NAT connections rose 400–600 -> 815–874. Every cell's Cloud SQL -traffic egresses through this NAT to the instance's public IP (the instance has no private IP: -`ipv4Enabled=true`, `privateNetwork` unset). When a VM's 64 ports fill, new TCP SYNs to 3307 are dropped, -the proxy's dial times out, and the relay pool's 2 s `connectionTimeoutMillis` fires: that is the exact -2 s stall the old image dies on and the new director surfaces as a 500. The dial timeouts hit c7 and c8 -hardest because they carry the most controls and open the most DB connections. - -What raised port demand today: each old-image crash re-opens a full pool through fresh NAT ports, the -autoheal recreates do the same, and the 55P03 retry storms keep more connections mid-transaction, so -crashes and NAT exhaustion feed each other. This is why the afternoon accelerated even after the disk -loop broke at 12:39. - -**Owning change (not applied):** `cloud/infra/terraform/relay-gce-foundation.tf` -`google_compute_router_nat.relay_gce` (this repo): set `min_ports_per_vm = 1024` (or enable -`enable_dynamic_port_allocation = true` with `max_ports_per_vm = 4096`) and, if needed, add manual NAT IPs -(each IP supplies 64,512 ports across VMs). Online change, no VM restart. The durable fix is giving the -Cloud SQL instance a **private IP** and pointing the proxy at `--private-ip`, which takes DB traffic off -NAT entirely; that is a Cloud SQL instance change in the orca-cloud foundation root plus a startup-script -flag here. Per the standing rule, not applied from this session. - -Direct proof: `resource.type="nat_gateway" AND jsonPayload.allocation_status="DROPPED"` shows **1,514 -dropped allocations to 35.188.82.89:3307** in 12:50–12:54 alone, every one of them the Cloud SQL public -IP. The NAT has zero manual IPs (AUTO_ONLY) and no port settings in Terraform, so it is at Google's -default 64 ports/VM. No workflow in this repo applies `relay-gce-foundation.tf` broadly (the roll -workflows apply cell templates with `-target`), so the NAT change needs a targeted apply of -`google_compute_router_nat.relay_gce`, which is an owner-run Terraform step. - -Original write-up of the symptom before the NAT correlation follows. - -The 12:50:30–12:50:50 stall (every cell 3.7–3.9 s SQL max, six old-image cells died) happened with -checkpoints healthy (85 ms) and disk at 6 MB/s, so it is not Finding 10. The cells' Cloud SQL Auth Proxy -logged `failed to connect to instance: dial error: dial tcp 35.188.82.89:3307: i/o timeout`. Count of -those per hour today: 08Z 1, 11Z 15, **12Z 416**; all of Sep 3: 4. Cloud SQL `up`/backends/connections -did not blip. So new TCP connections to the instance's public IP on 3307 are timing out from the cells' -proxies in bursts, which is exactly the "2 s connect timeout" the old image dies on. Query Insights for -12:49–12:54 attributes 1,380 s of lock wait to the placement CTE (`WITH assignment_state AS -MATERIALIZED …`) and 469 s to the single-row reservation UPDATE: the lock queue is the *consequence* of -connections stalling mid-transaction, not the cause. Not chased further; candidates are the proxy's -connection churn under the crash loops (each recreated cell opens a fresh pool) and the instance's -public-IP path. Relay code cannot fix this; it is Cloud SQL / network. Dial timeouts by minute today: 12:20 24, 12:21 -66, 12:41 22, 12:42 6, 12:51 160, 12:52 137, i.e. bursts of 20–160 s each, and they hit c7 (new image, -89 today) and c8 (93) hardest, so it is not the old image's connection churn either. Cloud SQL `up`=1 -throughout. The proxy dials the instance's public IP `35.188.82.89:3307`; a burst of i/o timeouts to a -healthy instance points at the path (public-IP egress / NAT / proxy connection limits), not at Postgres. -That is the same 2 s that the old image dies on and that the new director surfaces as a 500. - -## Roll inputs (verified by the read-only `verify` run) - -**Image census from instance templates, 2026-09-04 21:45Z (authoritative, read from `gcloud compute -instance-templates`):** 20 serving cells on `5aedbca5` (c8, c9, c10, c13–c16, c19–c29) — the image that exits -the process on a Postgres connect timeout (Finding 6); c7 on `85bf6799`; c4, c5, c17, c18 (draining / -migration-only) on `0e83408b` / `36a56b10`; c1, c2, c3, c6, c11, c12 (existing-only) on Jul/Aug images. Target -for Roll 1 is `519f4914` (director already on it). Monitor dry-run dispatched 21:45Z as the roll gate; waves -require owner go. - - -- target-image-digest `sha256:519f4914217f08cabcdcd34825965db8473ec37c6591553a3af0d65dcdeeb183` (lock fix; supersedes 85bf6799 as target) -- previous target `sha256:85bf67993869a769642995d0863f4c2b6b569c3850c2d8390ec2ca5f2b179e28` (c7 is on this; use as c7's rollback) -- rollback-image-digest `sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563` -- target/rollback rehome protocol 1 / 1; expected-rehome-generation 12; selector generation **112** (110 before the c7 canary) -- existing-only c1,c11,c12,c2,c3,c4,c5,c6; migration-only c17,c18; general c10,c13–c16,c19–c29,c7,c8,c9 -- confirmation for canary: `ROLL_RELAY_SAME_CAP production-gce-c7` -- monitor evidence is single-use and must be < 5 min old at dispatch (plus 75 min per predecessor wave) -- monitor dry-run dispatch (read-only, runs at `main` head so a merged bar change applies immediately): - `gh workflow run cloud-monitor-relay-production.yml --ref main -f mode=dry-run -f expected-selector-generation=110 - -f expected-existing-only-cells= -f expected-migration-only-cells=production-gce-c17,production-gce-c18 - -f expected-general-cells= -f migration-policy=strict -f recovery-source-cell-id=none -f capacity-cell-id=none` - -## Queries that worked (copy-paste) - -- Cell metrics: `resource.type="gce_instance" AND jsonPayload.event="orca_relay_runtime_metrics"` -- Container crashes: `resource.type="gce_instance" AND jsonPayload.MESSAGE:"container die" AND jsonPayload.MESSAGE:"relay@sha256"` -- Crash banner: `resource.type="gce_instance" AND jsonPayload.message:"Node.js v24"` -- Retries: `jsonPayload.event="orca_relay_postgres_transaction_retry"` (no resource filter to get both) -- Director lines are `textPayload`; cell lines are `jsonPayload.message` -- Cloud Run concurrency: Monitoring API `run.googleapis.com/container/max_request_concurrencies` -- Dry-run final state: download artifact `relay-monitor-dry-run--`, read `*.state.json` (the log's `schemaVersion` lines are only checkpoints, not the final verdict) - -## 2026-09-04 22:50Z onward: owner go received; driving the gates - -Owner: "sure, feel free to drive these." Sequence chosen: Roll 1 first (highest uplift), auth deploy with -#478 second, pruner enable third, label drift resolved by matching Terraform to live state, #477 still held. - -| Step | Result | -| --- | --- | -| Monitor dry-run #19 (gen 112, strict) | **Passed** 23:07:53Z, run 33927238469 attempt 1. First green since the probe fix (#18723). 16 samples, no freeze. Dispatched 22:51:33Z after confirming: 0 `container die` in 3 h, director 5xx in the last 4 h were all 503s (excluded by the `director.errors` filter). | -| c8 `canary-apply` onto 519f4914 (rollback 5aedbca5) | **Failed at 23:09:07Z before any mutation**: `relay monitor evidence provenance does not match` in `verify-authority`. Run 33928330631. Gate job passed, `cell_1 / rollout` failed on the manifest check, `seal_canary` skipped, lease released. Cause: the manifest binds `commitSha`; the dry-run ran at main `264c9ed8d2`, the canary dispatched at `--ref main` resolved to `4fab8e2f15` because unrelated PRs merged to main during the 15-minute gate. Verified no side effects: c8 MIG still on template `…c8-20260827…` (5aedbca5), stable, 25 controls; no `/v1/admin/drain` or isolate calls in the director log. | -| Constraint learned | Both workflows must run at the **same main commit**. The production environment's deployment branch policy allows only `main`, and the job gates on `github.ref == 'refs/heads/main'`, so a pinned tag/branch is not an option. Any merge to stablyai/orca main during the 15-minute dry-run invalidates the evidence. Mitigation for the retry: dispatch the canary within seconds of the green, and do not merge anything to stablyai/orca main myself during the window. A durable fix (accept evidence whose commit is an ancestor with identical workflow/script content) is a follow-up, not a same-day change to a safety check. | -| Label drift (5.x) | Resolved by dropping the `region` label from Terraform to match the 21 live metrics (stablyai/orca #18734, merged). Targeted plan asserted `27 no-op, 9 create, 0 destroy`; applied 23:11Z: 8 `orca_relay_control_*` renewal metrics that had never been applied, plus `google_monitoring_dashboard.relay_incident`. `orca_relay_controls` createTime unchanged (2026-07-13), label extractors unchanged. | -| Pruner enable (1.2) | orca-cloud #479 merged: `auth_token_pruner_enabled = true`, image digest of `00031-tox`, `max_rows_per_run = 20000`. Targeted plan asserted 9 create / 0 change / 0 destroy (job, scheduler at `41 * * * *` UTC, two service accounts, five IAM grants). **Not yet applied**: waiting until the roll canary has landed so the first hourly run does not overlap a drain. | -| Auth deploy with #478 (3.1) | Dispatched 23:13Z from orca-cloud main `f0fa4b5` (run 33928663526). Candidate startup adds nullable `successor_material` under a brief ACCESS EXCLUSIVE lock. | -| Auth deploy result | **Succeeded** 23:15:37Z: `orca-cloud-auth-00035-gos` serving 100 %, cap 20 preserved, 0 5xx. `refresh_tokens.successor_material` present (nullable text); 298 sealed successors written in the first 15 min against 924 rotations; `session-refresh-reuse-detected` at baseline (5 / 15 min). Grace window is live. | -| Monitor dry-run #20 | Froze 23:35:38Z on `runtime_power_unknown cell.production-gce-c11.powered`. Two window restarts earlier (23:24, 23:25) on `signal_stale auth.errors` (Cloud Monitoring publish lag 181–255 s vs 180 s bar). Cause: one transient rejection of the per-cell MIG GET in `readResourceInventory` yields `targetSize: null` → `runtimeKnown=false` → hard freeze. c11 is a parked existing-only cell (MIG size 0, stable) and was fine. Not fleet health. Fix delegated: stablyai/orca #18740 (retry the MIG read once, mirroring #18723). Run 33928912676. | -| Monitor dry-run #21 | **Green** 23:54Z at main `8064d1f991`, but main had moved to `0a821e5bc8` during the window; the chain re-gated instead of dispatching (the canary would have failed provenance again). Run 33930229711. | -| Monitor dry-run #22 | **Green** 00:10Z at `0a821e5bc8`; main moved to `2e80972450`. Re-gated. Run 33931177390. | -| Monitor dry-run #23 | Froze 00:18:31Z on `cell.production-gce-c29.latency_ms` 2635 > 2000, the probe's own round-trip from a US runner to asia-east2; c29 controls 17→19 and `sqlLatencyMsMax` flat ~1050 through the minute, no crash, no checkpoint stall. c29 probe max was 0 in the three previous gates, so a one-off. Run 33932092775. | -| Blocking constraint | Main receives unrelated merges every 5–10 min (23:08, 23:15, 23:17, 23:40, 23:42, …). A 15-min gate bound to an exact commit cannot be consumed under that traffic. Delegated a durable fix: `verify-authority` accepts evidence whose commit is an ancestor of the canary commit **and** has no diff on the monitor/deployer trusted paths; fails closed on shallow clones or unknown commits. Chain re-armed on dry-run #24 (run 33932679796) meanwhile. | -| Monitor dry-run #24 | Froze 00:28:00Z on `director.instances` 4 < 5. Cloud Run active-instance count read 4 for exactly one minute (00:27), 5 in every other minute for 3 h; min/max scale is pinned at 5; no new revision. A routine single-instance recycle. Not fleet health. Bar `directorInstancesMin: 5` with `latest-sum` cannot tolerate that; recalibrate to 4 or use a 3-min window minimum (follow-up, not same-day). Run 33932679796. Chain dispatched #25 (run 33933193511) at `86cd327749`. | -| Monitor dry-run #25 | **Green** 00:46Z at `86cd327749`; main moved to `8096cb2803`. Fourth green gate lost to unrelated main traffic (#19, #21, #22, #25). Run 33933193511. Chain's re-gate #26 (run 33934079533) cancelled by me. | -| Fixes merged 00:55Z | stablyai/orca #18740 (MIG inventory read retried once before `runtime_power_unknown`; 2 tests) and #18754 (`verify-authority` and the batch canary authority accept evidence sealed at an **ancestor** commit when every trusted monitor/deployer path is byte-identical; fails closed on shallow clones and unknown commits; deploy/rehome jobs now check out with `fetch-depth: 0`; 5 new tests, 18/18 pass). Reviewed both diffs; trusted-path set verified to exist on main. | -| Monitor dry-run #27 | Dispatched 00:56Z at `74ad08ec66` (first gate whose evidence the new rule can consume). Run 33934541092. Chain re-armed with the same ancestor + identical-trusted-code rule so an unrelated merge no longer forces a re-gate. | -| Monitor dry-run #27 | **Green** 01:11:35Z at `74ad08ec66`; main had moved to `38bde20121` with identical trusted code, so the new rule (#18754) let the chain dispatch. Run 33934541092. | -| c8 `canary-apply` #2 (run 33935407461) | Provenance check **passed** (first consumption of ancestor evidence). Isolate → gen 113, drain, template+MIG applied 01:14–01:22, new c8 came up on `519f4914` and `relay_capacity_transition_verified` (migration-only, image exact, heartbeat fresh) at 01:23:50. Then the step's next call, `curl --fail-with-body` to c8 `/v1/admin/runtime-status`, got a **503 with a 27-byte body** at 01:23:51 and the step exited 22. Director `cell-status` at 01:23:50.8 returned 200; c8's own logs show nothing at that second; c8 health/ready both 200 seconds later; backend HEALTHY (the health check had just flipped TIMEOUT→HEALTHY at 01:22:16 and UNKNOWN→HEALTHY at 01:23:47 as the new instance warmed). Read: a single 503 at the load-balancer/warm-up edge on a curl with no retry, on a cell that was already verified healthy one line earlier. Failsafe ran: c8 kept **migration-only**, rehome control disabled, selector gen 113. c8 is serving (40 controls at 01:39, sqlLatencyMsMax ~30 ms) on the target image, just not admitted for general traffic. Nothing to roll back. | -| Recovery | The job has an explicit resume path: `mode=rollback` with `rollback-image-digest` = the image the cell already runs skips isolate/apply, verifies, and restores general admission (`ROLLBACK_RESUME=true`). Dispatched gate #28 (run 33936966508) at gen 113 with c8 in migration-only; on green the chain dispatches that resume for c8 with rollback digest `519f4914` and target `5aedbca5` (the validator only requires them to differ). | -| Follow-up | The verify step's bare `curl --fail-with-body` needs the same "no reading is not a verdict" retry the monitor got (#18723/#18740); a 503 immediately after `verify-relay-capacity-transition` passed is not evidence of a bad cell. | -| Monitor dry-run #28 | **Green** 01:58:59Z at gen 113 with c8 in migration-only. Run 33936966508. | -| c8 recovery (run 33937756402, `mode=rollback`, rollback digest = 519f4914) | **Succeeded** 02:02Z. `ROLLBACK_RESUME=true` path: isolate/apply skipped, converged-Terraform check passed, verify passed (`relay_capacity_transition_verified` general, image `519f4914`, heartbeat fresh), activate → **gen 114**, c8 general. No restart, no drain. c8 at 43 controls, sqlLatencyMsMax 36 ms. **c8 is the second cell on 519f4914** (with c7 on 85bf6799). Because the recovery ran as `rollback`, `seal_canary` was skipped, so no canary authority exists for a `batch-apply`; the next cell runs as another `canary-apply`. | -| Merged 02:05Z | stablyai/orca #18769: bounded retries on every admin-endpoint curl/fetch in the same-cap job and the rehome/canary/verify scripts (`--retry 3 --retry-delay 2 --retry-connrefused`, per-attempt bodies to a file; script helper 2 attempts on network error or 500/502/503/504 only; 4xx never retried; 650/650 tests). Trusted-path change, so the next gate runs at a commit containing it. | -| Pruner enabled (1.2) | Terraform applied 02:06Z (8 creates, then the deploy-identity job IAM grant after a propagation 404, 9/9). Job `orca-cloud-auth-token-pruner`, image `343a0915…`, scheduler `41 * * * *` UTC, budget 20 000 rows/run. First run by hand (exec `sf5ct`): cold start 3m20s, then `stopReason: time-budget` at 480 s: 73 batches, 365 000 scanned, **1 040 deleted** (1 021 revoked, 19 expired, 0 rotated), ~6.4 s/batch of 5 000, `completedFullPass: false`. No errors, no lock-wait or checkpoint alert. Scan-bound, not budget-bound: at this pace a full pass over the table takes many hourly runs, and the row budget is never the limiter. Leave the budget alone; watch hourly runs for `stopReason` and a rising `deletedRows` as the cursor reaches the rotated backlog. | -| Monitor dry-run #29 | **Green** 02:20:58Z at gen 114, main `e2b70a5eba` (contains #18740, #18754, #18769). Run 33938052374. | -| c9 `canary-apply` (run 33938818286) | **Succeeded end to end** 02:21–02:34Z: isolate → gen 115, drain, template+MIG to `519f4914`, verify passed on the first try (retry-hardened step), trust proof, activate → **gen 116**, general. `seal_canary` **succeeded**: batch authority now exists. c9 at 38 controls, sqlLatencyMsMax 33 ms. No `container die` in 30 min. Three cells on new images (c7 `85bf6799`, c8 and c9 `519f4914`); 17 serving cells still on `5aedbca5`. | -| Monitor dry-run #30 | Dispatched 02:36Z at gen 116 (run 33939533990). On green the chain dispatches **batch 1**: `batch-apply` c10,c13,c14,c15 bound to canary run 33938818286 (sealed at gen 116, same commit `e2b70a5eba`). Preflight: all four on `5aedbca5`, MIGs stable, no crash in 20 min. Sequential cells inside the job (wave-index 0..3), each with its own isolate/drain/apply/verify/restore, so ~12 min per cell, ~50 min total. | -| Monitor dry-run #30 verdict | **Green** 02:52:15Z at gen 116, `e2b70a5eba`. | -| Batch 1 (run 33940290163) | Dispatched 02:52:27Z: `batch-apply` c10,c13,c14,c15, canary authority run 33938818286, same commit. | -| Batch 1 attempt 1 (run 33940290163) | **Failed at 02:54:39Z in the live preflight, before any mutation**: `relay live preflight failed: cloud-monitoring/signal_stale`. The step's `--retry-freshness` (5 attempts, 15 s apart, freshness-only codes) is passed only for `WAVE_INDEX != 0`; the first cell takes a single sample, so one Cloud Monitoring publish lag > 180 s at that instant fails the batch. Every candidate series was current again by the time I checked. c10 untouched (template `…c10-20260827…`, 47 controls), no selector write, gen still 116, failsafe no-op. Gate #31 dispatched 02:57Z (run 33940508865); chain re-dispatches the same batch (canary authority 33938818286 still valid: same gen 116, same commit). Fix delegated: wave 0 gets the same freshness retry. | -| Monitor dry-run #31 | **Green** 03:13:26Z at gen 116; main at `cb7f7dd11a` with identical trusted code. Run 33940508865. | -| Batch 1 attempt 2 (run 33941253533) | Dispatched 03:13:38Z: c10,c13,c14,c15, canary authority 33938818286. Runs at `cb7f7dd11a` (batch authority is accepted across the ancestor since trusted paths are unchanged). | -| Merged 03:14Z | stablyai/orca #18778: `--retry-freshness` on every same-cap wave including the first, and the retry loop now stops before the next wait would push evidence past the wave's age bound (it was checked only at entry before). Twin carve-out in the capacity job filed as a follow-up. | -| Batch 1 cell 1 (c10) | **Succeeded** 03:14–03:27Z (preflight, drain, apply, verify, restore). c13 started 03:27Z. | -| Batch 1 cell 2 (c13) | **Succeeded** 03:27–03:38Z. c14 started 03:38Z. | -| Batch 1 cell 3 (c14) | **Succeeded** 03:38–03:50Z. c15 started 03:50Z. | -| Batch 1 complete (run 33941253533) | **All four succeeded** 03:13–04:00Z: c10, c13, c14, c15 on `519f4914`, selector **gen 124**. Fleet at 936 controls, 23 cells. Two `container die` at 03:35:41/44 were **c13's new container** exiting during boot (`applyPostgresSchema` → `Connection terminated due to connection timeout`, exit 1, 2 s runtime each) because the `cloud-sql-proxy` sidecar had not finished starting; the third start at 03:35:45 succeeded and c13 has been serving since (57 controls). A boot-order race in the container spec, not a serving-cell crash. Follow-up: schema pool should wait for the proxy socket, or the container should depend on the proxy's readiness. **8 cells on new images** (c7 85bf6799; c8, c9, c10, c13, c14, c15 519f4914), 12 on `5aedbca5`: c16, c19–c26 (US), c27–c29 (Asia). | -| Monitor dry-run #32 | **Green** 04:19:50Z at gen 124, main `436ef827dd` (contains #18778). Run 33943539025. | -| c16 `canary-apply` (run 33944255902) | Dispatched 04:20:02Z. On success it seals the authority for batch 2 (c19,c20,c21,c22). | -| c16 canary (run 33944255902) | **Succeeded** 04:20–04:32Z, activate → gen 126, batch authority sealed. 9 cells on new images. | -| Monitor dry-run #33 | Failed 04:58:56Z on `continuity_deadline_exceeded` (1 500 004 ms > 1 500 000 ms). One `signal_stale cloud_sql.lock_waits` at 04:46 (189 s vs 180 s bar, Cloud Monitoring publish lag) restarted the 15-min window at sample 12; the restart could not complete inside the 25-min continuity cap. No health failure at any sample; no `container die` since c16's own boot race at 04:30. Run 33944873727. Chain re-gates. Note for recalibration: `cloudDataMaxAgeMs: 180000` vs observed Cloud Monitoring publish lag of 181–255 s has now cost three gates (#20 twice, #33). | -| Freshness recalibration | stablyai/orca #18798 (open, merge after batch 2 dispatch): `cloudDataMaxAgeMs` 180 s → 330 s, derived from Google's documented visibility delays (Cloud Run 60+120 s, Cloud SQL 60+165 s) and the 5-min window-sum query (a label series that stops emitting reads as up to 300 s old while its sum is complete, which is the 255 s `auth.errors` case) plus ~30 s collect latency. Director-admin and the lock-wait carry keep their own 180 s pins. A freshness-only failure may miss 2 consecutive samples without restarting the window; the sample still counts and is still threshold-checked; a 3rd miss, collector failure, runner gap, or any breach restarts/freezes as before. 92/92 tests. | -| Monitor dry-run #34 | **Green** 05:17:31Z at gen 126, `436ef827dd`. Run 33946093029. | -| Batch 2 (run 33946819345) | Dispatched 05:17:43Z: c19,c20,c21,c22, canary authority 33944255902 (c16). | -| Merged 05:19Z | stablyai/orca #18798 (freshness bar 330 s + two-sample tolerance). Next gate runs at a commit containing it. | -| Batch 2 cell 1 (c19) | **Succeeded** 05:19–05:32Z. c20 started. | -| Batch 2 cell 2 (c20) | **Succeeded** 05:32–05:43Z. c21 started. | -| Batch 2 cell 3 (c21) | **Succeeded** 05:43–05:59Z. c22 started. | -| Batch 2 complete (run 33946819345) | **All four succeeded** 05:17–06:12Z: c19, c20, c21, c22 on `519f4914`, selector **gen 134**. Fleet at 1 090 controls, 23 cells, refresh 401s at baseline (1–4 per 3 min). One `container die` at 06:08:45 was **c22's new container** exiting during boot (exit 1, 2 s runtime; started 06:08:43, restarted 06:08:46 and serving since), the same proxy-sidecar boot race seen on c13 and c16. No serving-cell crash. **Census: 15 of 23 serving cells on new images** (c7 `85bf6799`; c8–c10, c13–c16, c19–c22 `519f4914`), 7 on `5aedbca5`: c23–c26 (US), c27–c29 (Asia). Next: gate at gen 134 → canary c23 → batch c24,c25,c26; then canary c27 → batch c28,c29. | -| Monitor dry-run #35 | **Green** 06:32:01Z at gen 134, `b33d1972bc` (contains #18798, first gate at the 330 s freshness bar). Run 33949334606. | -| c23 `canary-apply` (run 33950075843) | Dispatched 06:32:13Z at main `b0c67eaf88` (ancestor gate SHA, identical trusted code). On success it seals the authority for batch 3 (c24,c25,c26). | -| c23 canary (run 33950075843) | **Succeeded** 06:32–06:46Z, activate → gen 136, batch authority sealed. No `container die` during boot. 16 of 23 serving cells on new images; 6 on `5aedbca5` (c24–c26 US, c27–c29 Asia). | -| Monitor dry-run #36 | Dispatched 06:46Z at gen 136, run 33950746574 (`58553bfe1c`). On green the chain dispatches batch 3 (c24,c25,c26) under canary authority 33950075843. | -| Monitor dry-run #36 result | **Green** 07:02:49Z at gen 136, `58553bfe1c`. | -| Batch 3 (run 33951468008) | Dispatched 07:03Z: c24,c25,c26, canary authority 33950075843 (c23). | -| Batch 3 cell 1 (c24) | **Succeeded** 07:04–07:18Z. c25 started. | -| Batch 3 cell 2 (c25) | **Succeeded** 07:18–07:31Z. c26 started. | -| Batch 3 complete (run 33951468008) | **All three succeeded** 07:03–07:44Z: c24, c25, c26 on `519f4914`, selector **gen 142**. Fleet at ~1 230 controls, 23 cells, refresh 401s at baseline. **Zero `container die`** during the batch (no boot race on c24–c26). **All 20 US serving cells now on new images** (c7 `85bf6799`; c8–c10, c13–c16, c19–c26 `519f4914`). Remaining on `5aedbca5`: c27, c28, c29 (asia-east2, probe hard cap 3000 ms). | -| Monitor dry-run #37 | Dispatched 07:48Z at gen 142, run 33953555224 (`4c5077d57a`). On green the chain dispatches the c27 canary (first Asia cell). | -| Monitor dry-run #37 result | **Green** 08:04:24Z at gen 142, `4c5077d57a`. | -| c27 `canary-apply` (run 33954264945) | Dispatched 08:04Z, first Asia cell (asia-east2-a). On success it seals the authority for batch 4 (c28,c29). | -| c27 canary (run 33954264945) | **Failed closed before any mutation** 08:07:25Z at "Verify exact current generation, digest, cap, and rollback point": `runtime predecessor mismatch fields=regionalRehomeProtocol`. **Operator input error, not a cell fault**: the chain script hardcoded `target-rehome-protocol=1 / rollback-rehome-protocol=1` for every cell, but `relay_region_rehome_source_cell_ids` lists only the 16 US cells (c7–c10, c13–c16, c19–c26), so the Asia startup template omits `ORCA_RELAY_REHOME_*` and c27–c29 report protocol 0 by design. `MUTATION_STARTED` never set, failsafe no-op, selector stays gen 142, c27 still serving on `5aedbca5`, no `container die`. Gate #37 evidence consumed. Fix: chain script now takes `PROTO`; Asia round dispatches with protocol 0 (the per-host trust proof step is protocol-gated and skips, as designed for non-source cells). Follow-up: the job already reads `relay_region_rehome_source_cell_ids`; it could derive the expected protocol from membership instead of trusting the operator input. | -| Monitor dry-run #38 | Dispatched 08:12Z at gen 142, run 33954621425 (`e95d247be1`). On green the chain dispatches the c27 canary with protocol 0. | -| Monitor dry-run #38 result | **Green** 08:28:36Z at gen 142, `e95d247be1`. | -| c27 `canary-apply` #2 (run 33955359385) | Dispatched 08:28Z with `target/rollback-rehome-protocol=0`. | -| c27 canary #2 (run 33955359385) | **Failed closed, no mutation** 08:31:19Z. Predecessor check passed with protocol 0; the isolate step then died at argument parsing: `production capacity target is not approved`. The same-cap job shells out to `prepare-relay-production-capacity-canary.mjs` for isolate/drain/activate, whose `PRODUCTION_CAPACITY_CELL_IDS` allowlist is the 16 US capacity cells (c7–c26), while the same-cap wave validator (`SAME_CAP_CELLS`) approves all 19 serving cells including c27–c29. The Asia cells have never been through this job (their Aug 14 rollout used the asia-topology workflow). Both the isolate step and the failsafe threw before any HTTP call, so `MUTATION_STARTED=true` was written but nothing was isolated: selector stays gen 142, c27 general and serving on `5aedbca5`, no `container die`. Gate #38 evidence consumed. Fix: stablyai/orca #18811 (`--approved-cells same-cap` on all four invocations, default unchanged for the US capacity job, census test over every `SAME_CAP_CELLS` member × isolate/drain/activate + the job's cell-shape bash block; 525/525 script tests). Sweep of the other job scripts found no further Asia blocker; gate #39 (run 33955668701) dispatched at gen 142 to prove the selector is unchanged before the next attempt. | -| Monitor dry-run #39 | **Green** 08:51:28Z at gen 142: independent proof the selector was untouched by both failed c27 attempts. Not used for dispatch (its commit predates #18811). | -| Merged 08:51Z | stablyai/orca #18811 → main `12e05203a4`. | -| Monitor dry-run #40 | Dispatched 08:51Z at gen 142 on main `12e05203a4` (contains #18811), run 33956408337. On green the chain dispatches the c27 canary, protocol 0, third attempt. | -| Monitor dry-run #40 result | **Green** 09:08:03Z at gen 142, `12e05203a4`. | -| c27 `canary-apply` #3 (run 33957151726) | Dispatched 09:08Z, protocol 0, on main containing #18811. | -| c27 canary #3 (run 33957151726) | **Failed after isolate; failsafe held** 09:17:21Z. Live check 09:26Z: c27 at 0 controls (drained), template still `…20260814235757`, c28/c29 absorbed the hosts (37 each), fleet 1 404 controls / 23 cells, refresh 401s baseline, no `container die` in 60 m. Predecessor check and allowlist passed; isolate → **gen 143** (c27 migration-only), drain sent (graceMs 0, hosts reconnected via director to c28/c29/US). Terraform plan built correctly (template replace + MIG update to `519f4914`), then `validate-relay-capacity-plan.mjs --mode same-cap-cell` rejected it: `cell plan does not contain the reviewed image and capacity`. Its same-cap rule demands exactly one `ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT` and one `ORCA_RELAY_REHOME_AUDIENCE` printf in the startup script; Asia templates omit both because c27–c29 are not rehome sources (same root as attempt 1, third US-only assumption in the job). **No apply ran**: c27 template unchanged, still `5aedbca5`, isolated and draining (drain is one-way in-process; only a restart clears it). Failsafe re-asserted migration-only at gen 143 and rehome disabled. Recovery plan: fix validator (protocol-0 path: require the rehome lines *absent*), merge, gate at gen 143, then `mode=rollback` with rollback-image=`519f4914` (the failed-canary re-entry path; accepts draining + migration-only) to restart c27 onto the target image and restore it; then single-cell canaries for c28 and c29 (batch needs ≥2 cells). | -| Plan-validator fix | stablyai/orca #18818 (merged 09:41Z → main `9f2a9a248e`): `validate-relay-capacity-plan.mjs --regional-rehome-protocol 0|1` in same-cap-cell mode; protocol 0 requires the rehome lines *absent*, protocol 1 unchanged; both plan-validation calls in the job pass `DESIRED_REHOME_PROTOCOL`; census test now validates a correct plan for every `SAME_CAP_CELLS` member at its tfvars-derived protocol. 529/529. Residual: the operator-supplied protocol is still unbound for Asia cells (no `SOURCE_CELLS` cross-check outside us-central1), so a wrong value fails late at plan validation rather than early; deriving it from membership is the checklist follow-up. | -| Monitor dry-run #41 | Dispatched 09:42Z at gen 143 (c27 expected migration-only) on main `9f2a9a248e` (contains #18811 + #18818), run 33958728141. On green: c27 recovery via `mode=rollback`, rollback-image `519f4914`, protocol 0, confirmation `ROLL_BACK_RELAY_SAME_CAP`. | -| Monitor dry-run #41 result | **Green** 09:58:51Z at gen 143, `9f2a9a248e`. | -| c27 recovery #1 (run 33959789773, `mode=rollback`) | **Failed closed, no mutation** 10:09:21Z at `Verify monitor evidence provenance`: `relay monitor dry-run authority is incomplete or stale`. The dry-run authority is valid for 5 min after `completedAt` at wave 0 (`EVIDENCE_MAX_AGE_MS`); the gate completed 09:58:51Z but the operator poller (20 s `gh run view` loop) only observed completion at 10:07:09Z during a local network outage, so the dispatch landed at 10:07:11Z, 8 m 20 s after completion. Failed before the rollout lease, isolate, or any Terraform step; c27 unchanged (migration-only, drained, `5aedbca5`, gen 143). Every prior canary dispatched ≤15 s after gate green, so this is a dispatch-latency miss, not a job defect; the freshness bound behaved as designed. | -| Monitor dry-run #42 | Dispatched 18:39Z at gen 143 on main `af82126058` (trusted paths byte-identical to `9f2a9a248e`), run 33984753269. Recovery script re-armed behind it (same `mode=rollback` onto `519f4914`, protocol 0). | -| Monitor dry-run #42 result | **Green** 18:55:47Z at gen 143, `af82126058`. | -| c27 recovery #2 (run 33985902062, `mode=rollback`) | **Failed closed, no mutation** 19:05:02Z, same `authority is incomplete or stale`. Dispatch landed 19:02:39Z, 6 m 52 s after the gate completed. Root cause of both misses is the operator laptop sleeping during the 15 min gate wait (`pmset -g log`: asleep 18:52:28Z → 19:02:17Z; the morning miss coincided with a sleep/dark-wake cycle too), so the 20 s poller never ran inside the 5 min window. Not a job or evidence defect: the freshness bound did its job. Operator fix: poller now runs under `caffeinate -i`. | -| Monitor dry-run #43 | Dispatched 19:06Z at gen 143 on main `af82126058`, run 33986121849. Recovery armed behind it under `caffeinate`. | -| Monitor dry-run #43 result | **Green** 19:22:53Z at gen 143, `af82126058`. | -| c27 recovery #3 (run 33986948522, `mode=rollback`) | **Failed closed, no mutation** 19:25:48Z. Dispatched 13 s after gate green (authority accepted this time), then the live preflight recheck failed: `relay live preflight failed: active-probe/threshold_max`. That is the 2 000 ms `endpointLatencyMs` bar on one endpoint's slowest /health or /ready round trip from the runner (8 s fetch timeout, one retry). The error names no endpoint and the job log prints none; gate #43 had zero failures across 16 samples, so this was a transient probe slow-down in the ~3 min between gate and preflight. Live probe 19:32Z from the operator: director and auth ~130–190 ms, US cells ≤540 ms, Asia cells 690–1 315 ms (c28/c29 /health ~1.3 s, the closest to the bar; c27 ~0.9 s). Existing-only cells c1–c3, c6, c11, c12 return 503 on both paths as expected (unpowered). Failed before the rollout lease, isolate, or any Terraform step; c27 unchanged. Follow-up (checklist): preflight should print the failing signal and observed value. | -| Monitor dry-run #44 | Dispatched 19:33Z at gen 143 on main `062db77118`, run 33987646501. Recovery re-armed behind it. | -| Monitor dry-run #44 result | **Frozen red** 19:50:01Z after 13 samples: `active-probe/threshold_max cell.production-gce-c27.latency_ms observed=2568 threshold=2000`. No other failure, no continuity event, no `container die` fleet-wide in 60 m. `/health` is a static JSON reply (`app.ts`), so the slow round trip was `/ready` (the probe reports the max of the two) or the path to the cell. Cloud SQL logs for 19:49:38Z–19:51:58Z show six `could not obtain lock on row in relation "relay_cells"` errors and a time-triggered checkpoint completing at 19:50:36Z (write phase 270 s, the spread target, not a stall). c27 is drained with 0 controls, so its `/ready` dependency check was the only thing it was doing. Recovery script stopped as designed (no auto re-gate). Operator probe 19:53Z: c27 and c28 both bimodal, ~0.27 s or ~0.89 s per `/health` from the US, identical shape, nothing c27-specific. Attributing the one 2.6 s sample to the same shared-DB contention that produced the lock errors is the best available reading; the retry at gate #45 tests whether it recurs. | -| Monitor dry-run #45 | Dispatched 19:53Z at gen 143 on main `062db77118`, run 33988383401. Recovery re-armed behind it. | -| Monitor dry-run #45 result | **Frozen red** 19:54:47Z after 3 samples, same signal: `cell.production-gce-c27.latency_ms observed=2668 threshold=2000`. Two gates in a row now attribute a >2 s round trip to c27 while every other cell passes. | -| c27 `/ready` tail analysis | `/health` is static; `/ready` (`relay-readiness.ts`) fetches the auth JWKS (2 s timeout) then runs `SELECT 1`, cached 10 s. Operator probes 19:57Z–20:00Z, 15 each from the US: c27 and c28 have the **same** tail (0.27 s / 0.88 s modes, then 1.3 s, then 2.17–2.27 s at the top); US cells c8/c20 sit at 0.08–0.18 s. Auth JWKS latency over the last hour: 400 requests, max 20 ms, none over 1 s. So the tail is cell→Cloud SQL (US) round trips plus the runner→Asia hop, not auth and not c27-specific; c27 is drained (0 controls) so nothing local competes. Cloud SQL `could not obtain lock on row in relation "relay_cells"` runs at 17–78 per 10 min all day (NOWAIT inventory locks, expected under placement bursts) with no spike in the failing minutes. The bar (`endpointLatencyMs` 2 000 ms, one shot per minute, max of two paths) leaves Asia cells ~10% of samples from tripping; the gate got unlucky twice on c27 and lucky on c28/c29. Not a health finding. | -| Monitor dry-run #46 | Dispatched 20:01Z at gen 143 on main `062db77118`, run 33988810139. Recovery re-armed behind it. If this also freezes on an Asia probe, the next move is a per-region latency bar (or p50 over the window) in `incident-monitor.ts`, reviewed and merged before further Asia gates rather than retrying blindly. | -| Monitor dry-run #46 result | **Frozen red** 20:07:44Z after 7 samples, third time on `cell.production-gce-c27.latency_ms` (observed 2 685). Operator 40-sample `/ready` probe per Asia cell at 20:10Z: c27 p50 0.88 s / p90 2.15 s / max 2.26 s / 6 over 2 s; c28 p50 0.88 / p90 1.25 / max 2.25 / 1 over; c29 p50 0.88 / p90 0.89 / max 1.27 / 0 over. All 200. `/ready` (`relay-readiness.ts`) fetches the auth JWKS in us-central1 then `SELECT 1` on Cloud SQL in us-central1, so an Asia cell's readiness is two trans-Pacific hops plus the runner→Asia hop; the fleet-wide 2 000 ms bar was calibrated on US cells (0.08–0.5 s). c27 being drained and idle has no local load, so this is path latency, not health. **Stopped retrying gates.** Fix in flight: per-region `cell..latency_ms` bar (us-central1 stays 2 000, asia-east2 4 000; hard faults still caught by the health/ready equal-1 checks and the 8 s probe timeout) plus attributable preflight failure messages, via review + CI before the next Asia gate. | -| Merged 20:33Z | stablyai/orca #18877 → main `a3c1d32995`: per-region `cellEndpointLatencyMs` (us-central1 2 000, asia-east2 4 000; director/auth rules and the `endpointLatencyMs` key unchanged), region carried from tfvars onto every cell expectation, preflight failures now print `source/code signal observed= threshold=`. relay-ops 95/95, cloud suite 633 + 529 + 148 green. | -| Monitor dry-run #47 | Dispatched 20:34Z at gen 143 on main `a3c1d32995` (first gate with the per-region bar), run 33989896150. Recovery re-armed behind it. | -| Monitor dry-run #47 result | **Green** 20:38:09Z at gen 143 on `a3c1d32995`: first gate under the per-region bar, 16/16 samples, no Asia latency failure. | -| c27 recovery #4 (run 33990715317, `mode=rollback`) | **Success** 20:51Z. Dispatched 13 s after gate green. Isolate re-asserted migration-only at gen 143 (already isolated, no change), Terraform applied the same-cap template `…20260905204141` and the MIG replaced the instance, new incarnation on `519f4914`, protocol 0, transition verifier passed at migration-only (1 180 assignments carried, hard cap 3 000, heartbeat fresh), then activate → **gen 144**, c27 general, verifier passed again. No `container die` fleet-wide 19:55Z–20:52Z. c27 now runs the target image; c28/c29 remain on `5aedbca5` (template `…20260814235757`). | -| Monitor dry-run #48 | Dispatched 20:53Z at gen 144 (c27 back in general, MIG = c17,c18) on main `61ebffa86e` (trusted paths identical to `a3c1d32995`), run 33991385880. On green the chain dispatches the c28 `canary-apply`, protocol 0. | -| Monitor dry-run #48 result | **Green** 21:08Z at gen 144, 16/16 samples, no Asia latency failure. Main had moved to `5cec2c2dfc`; the chain verified the trusted paths were identical to the gate commit and dispatched 12 s after green. | -| c28 canary (run 33992169289, `canary-apply`) | **Success** 21:27Z. Isolate → migration-only at **gen 145**, drain already clear, verifier passed on the old image (1 220 assignments carried, hard cap 3 000, heartbeat fresh), Terraform applied same-cap template `…20260905211352`, new incarnation on `519f4914` at protocol 0, verifier passed again at migration-only, activate → **gen 146**, c28 general, verifier passed (1 219 assignments). Seal step recorded the canary. No `container die` fleet-wide 21:08Z–21:30Z. Only c29 remains on `5aedbca5`. | -| Monitor dry-run #49 | Dispatched 21:33Z at gen 146 (c28 back in general, MIG = c17,c18) on main `dce5ebd83d` (trusted paths identical to `a3c1d32995`), run 33993075948. On green the chain dispatches the c29 `canary-apply`, protocol 0, the last Roll 1 cell. | -| Monitor dry-run #49 result | **Frozen red** 21:52:24Z, `active-probe/continuity_deadline_exceeded observed=1500005 threshold=1500000`. One continuity event at 21:41:27Z, `cloud-monitoring/collector_failed` (a Cloud Monitoring read failed, not tolerated), which reset the continuous window at sample 14; the restarted window reached 10 samples before the 25-minute lineage cap (`INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS`) expired. No health failure in any of the 25 samples, no Asia latency failure, no `container die`. Monitor-side transient, not a fleet finding. The chain re-gated automatically after its 2-minute back-off. | -| Monitor dry-run #50 | Dispatched 21:54Z at gen 146 on main `51eed5a1bc`, run 33994385666. **Green** 22:10Z, 16/16 samples. Main had moved to `d7767fb196`; trusted paths identical to `a3c1d32995`. Chain dispatched the c29 `canary-apply` (run 33995164002, protocol 0) 12 s after green. | -| c29 canary (run 33995164002, `canary-apply`) | **Success** 22:27Z. Isolate → migration-only at **gen 147**, verifier passed on the old image (1 199 assignments), Terraform applied same-cap template `…20260905221622`, new incarnation on `519f4914` at protocol 0, verifier passed at migration-only, activate → **gen 148**, c29 general, verifier passed (1 199 assignments carried). No `container die` fleet-wide 22:11Z–22:30Z. | -| **Roll 1 complete** | Image census 22:30Z from MIG templates: c8–c10, c13–c16, c19–c29 on `519f4914` (18 cells); c7 on `85bf6799` (the earlier rehearsal image, carries the same fix); existing-only c1–c6, c11, c12 and migration-only c17, c18 untouched by design. No serving cell remains on `5aedbca5`. Selector gen 148, membership unchanged from the start of the roll. Zero relay container exits fleet-wide across the roll (01:14Z–22:30Z). Gates used: #19–#50; freezes were all monitor-side (provenance, freshness, flat Asia latency bar, one Cloud Monitoring collector failure), none a fleet health finding. Roll 2 (fresh image with #18722 + #18720) is the next data-plane step and waits on the owner's private-IP window decision. | - -## Roll 2 (image `4916ed67`, 2026-09-06) - -| Step | Result | Evidence | -|---|---|---| -| Docs split | #18958 merged `3bb038a185` (findings, checklist, roadmap, Roll 2 plan). | | -| Code PR | #18959 merged `61b09b7a02` (rebase of #18565 onto main; desktop rotation change dropped since #18719 shipped a proportional version). Two Opus review rounds: round 1 caught the mobile fail-fast rejecting on any socket close (one AP flap would book the 60 s cooldown) → 2 s grace, re-armed once on `handshaking`; round 2 caught a removed jitter assertion that let a one-sided jitter pass → exact pin on the top of the band. Control lease 55 min → 6 h ± 30 min. | | -| Image publish | run 34002233801 → `sha256:4916ed676d8389f694a648e750f1112d9002d68c84a1e0c7af828d5af129de62`; mirrored to staging (run 34002326150). | | -| Staging cell smoke | **Dropped.** Staging C4 is pinned to the Asia launch digest by `relay-staging-c4-refresh-workflow.test.mjs` (with production c27–c29 tfvars and the C4 recovery workflow) and the only C4 image-refresh path pins its accepted predecessor to an older digest. Re-pinning all of it for a smoke widens into the Asia launch machinery; #18969 closed. Roll 2 follows the Roll 1 path: director first, c7 as the rehearsal cell. | | -| Director deploy | run 34002673626 **success** 01:02Z: serving `orca-cloud-relay-00575-leq` on `4916ed67`, `00574-wag` (same image) tagged `selector-rollback`, `00569-ret` (`519f4914`) still deployable. Baseline before: 1 director Postgres retry in the prior hour, 0 `container die`. | | -| c7 `verify` (read-only) | run 34002885408 **success** (gate success, cell_1 rollout success, release_lease success), target `4916ed67`, rollback `85bf6799`, protocol 1, gen 148. | | -| Director go/no-go (01:02Z–07:00Z, 6 h on `00575-leq`) | **Go.** Presence confirmed (13.8k assign 200s, 410 cell + 90 director `runtime_metrics` rows/30 min). Postgres retries 13 (all `55P03` lock_timeout) vs 85 on `00570-siv` in the prior 6 h. `/v1/assign` mix 200/401/503 = 13820/5557/623 vs 14081/5256/663 before the deploy; 503s are the placement/sticky admission `Retry-After` path and cluster by source (top source 351), same shape as before. 0 `container die`, cell `sqlFailuresDelta` sum 0. The earlier all-zero read at 01:28Z was a dead gcloud credential, not a quiet fleet, and was discarded. | | -| Monitor dry-run (Roll 2 gate 1) | run 34018071984 dispatched 07:03Z at gen 148, **green** 07:18Z at `1326d6b40c`; main had moved to `b51bbf3fc6` with identical trusted code. | | -| c7 `canary-apply` (run 34018804481) | **Succeeded** 07:18–07:31Z, protocol 1, rollback `85bf6799`: gate, rollout, seal_canary, release_lease all success. Template `…-20260906072156…` on `4916ed67`; selector gen 148 → 150. Four `container die` at 07:29:16–25Z were the new container exiting during boot (`applyPostgresSchema`/`backfillRelayCellRegions` → `Connection terminated due to connection timeout`, exit 1, 2 s runtime each) while the `cloud-sql-proxy` sidecar warmed up; fifth start at 07:29:26 listening, readiness check passed 07:29:27. Same boot-order race as c13 in Roll 1 batch 1, no serving impact (cell was still drained). 139 controls by 07:34Z and climbing, `sqlFailuresDelta` 0, `sqlLatencyMsMax` ~40 ms. | | -| Monitor dry-run (Roll 2 gate 2) | run 34019568779 dispatched 07:36Z at gen 150, **green** 07:51Z at `57e34c7f03` (main `6494f2a4f0`, identical trusted code). | | -| c8 `canary-apply` (run 34020284092) | **Succeeded** 07:52–08:09Z, protocol 1, rollback `519f4914`: all jobs success. Template `…-20260906075820…` on `4916ed67`; gen 150 → 152. One boot-race `container die` at 08:05:51Z (2 s, exit 1), next start served. 101 controls by 08:10Z, `sqlFailuresDelta` 0. | | -| Monitor dry-run (Roll 2 gate 3) | run 34021119905 dispatched 08:11Z at gen 152, **green** 08:26Z at `ffbf35e0d2`. | | -| Batch 1 `batch-apply` c9,c10,c13,c14 (run 34021868303, canary 34020284092) | **Failed on cell 3 (c13); c9 and c10 succeeded.** c9 08:27–08:43Z → gen 154, c10 08:43–08:58Z → gen 156, both trust-proven and restored general. c13: isolate → gen 157, drain, template `…-20260906090225…` on `4916ed67`, one boot-race exit 09:09:50Z, readiness 09:09:51Z, transition verifier passed at migration-only 09:11:17Z (2 680 assignments, heartbeat fresh, image `4916ed67`), then `probe-relay-rehome-trust` got **409** from the director at 09:11:18Z (157 ms; c9/c10 got 200 in ~178 ms). Failsafe re-asserted migration-only at gen 157 (no change). c14 skipped, lease released. c13 is **serving on the new image but isolated**: 151 controls by 09:18Z, `sqlFailuresDelta` 0, no exits fleet-wide after 09:12Z. The probe script prints only the status, not the director's `error` body, and neither the director nor c13 logs the 409 reason; candidates are the director's source check (`runtime.ready`/`heartbeatFresh`/incarnation read ~1 s after the verifier passed) or c13's `host-drain` rejecting the probe (incarnation mismatch, shared-runtime-identity proof, or the probe host unexpectedly present). Monitor residual: the probe should print the error body. | | -| Monitor dry-run (Roll 2 gate 4) + c13 recovery | Gate run 34024459585 dispatched 09:26Z at gen 157 with c13 in migration-only. On green: `mode=rollback` for c13 with rollback digest `4916ed67` (what it already runs) and target `519f4914`, protocol 1 both ways: `ROLLBACK_RESUME=true` path, no restart, verify + trust probe + restore general. As in Roll 1 (c8 recovery), the rollback mode seals no canary authority, so c14 runs as its own `canary-apply` and the next batch is c15,c16,c19,c20 behind that. | | -| c13 recovery (run 34025225328, `mode=rollback`) | Gate 4 **green** 09:38Z. Recovery **succeeded** 09:38–09:42Z: `ROLLBACK_RESUME=true`, no restart, verifier passed at migration-only (2 679 assignments, heartbeat fresh, `4916ed67`), **trust probe passed** (`host-not-connected` ×2, idempotent, shared runtime identity rejected), activate → **gen 158**, c13 general, verifier passed again. 154 controls, `sqlFailuresDelta` 0, no exits fleet-wide since 09:12Z. The 09:11Z 409 was therefore transient: same cell, same incarnation, same image, ~30 min later the identical probe passed. Most likely the director's source check reading the runtime row within ~1 s of the verifier's pass (a `ready`/heartbeat edge), which a retry in the workflow step would absorb. Residual: retry the trust probe once on 409 and print the error body. | | -| Monitor dry-run (Roll 2 gate 5) | run 34025450523 dispatched 09:44Z at gen 158, **green** 09:59Z at `6933fd70d7` (main `d19be485d3`, identical trusted code). | | -| c14 `canary-apply` (run 34026157631) | **Succeeded** 09:59–10:20Z, protocol 1: trust-proven, gen 158 → 160, canary authority sealed. No boot exits, 102 controls by 10:22Z, fleet `sqlFailuresDelta` 0 over 30 min. | | -| Monitor dry-run (Roll 2 gate 6) | run 34027238190 dispatched 10:23Z at gen 160, **green** 10:38Z at `ec64df335e` (main `adcc30be3b`, identical trusted code). | | -| Batch 2 `batch-apply` c15,c16,c19,c20 (run 34027985784, canary 34026157631) | **All four succeeded** 10:38–11:31Z, protocol 1, four trust proofs, gen 160 → 168. Boot-race exits only: 3 at 10:50Z (c16) and 5 at 11:02Z (c19), all 2–4 s, exit 1, next start served. Controls at 11:32Z: c15 160, c16 164, c19 164, c20 87 (still refilling). Fleet `sqlFailuresDelta` 1 over 30 min. | | -| Monitor dry-run (Roll 2 gate 7) | run 34030557166 dispatched 11:33Z at gen 168, **green** 11:48Z at `adcc30be3b`. | | -| c22 `canary-apply` (run 34031304526) | **Succeeded** 11:48–12:02Z, protocol 1, trust-proven, gen 168 → 170, canary authority sealed. No boot exits, 134 controls by 12:03Z. One correlated 1 s lock-timeout blip at 11:35:17–27Z (c10, c13, c19, c25, c28: one `sqlFailuresDelta` each, `sqlLatencyMsMax` ≈1 000 ms) spanning old and new images, the known lock-wait shape, not roll-related. Director retries 4 in the last hour. | | -| Monitor dry-run (Roll 2 gate 8) | run 34032011250 dispatched 12:05Z at gen 170, **green** 12:20Z at `adcc30be3b`. | | -| Batch 3 `batch-apply` c23,c24,c25,c26 (run 34032799574, canary 34031304526) | **Failed on cell 4 (c26); c23, c24, c25 succeeded** (12:20–13:11Z, gen 170 → 176, three trust proofs). c26: isolate → gen 177, drain, template `…-20260906131159…` on `4916ed67`, one boot-race exit 13:19:17Z, readiness 13:19:19Z, transition verifier passed at migration-only 13:20:42Z (2 604 assignments, heartbeat fresh, `4916ed67`), then the very next call, `admin_post target-runtime` to `c26.relay.onorca.dev/v1/admin/runtime-status`, got **503 `unconditional drop overload`** (27-byte body) and the step failed. That string is not in the relay codebase and c26 logged nothing at 13:20:42Z (readiness at 13:19:19Z, metrics steady), so it is a front-end/LB shed on one request; curl's `--retry 3` logged no retry attempt. Failsafe re-asserted migration-only at gen 177 (no change). c26 is serving on the new image but isolated: 166 controls by 13:25Z and climbing, `sqlFailuresDelta` 0. Residual: the post-apply `admin_post` should retry on 503 (the pre-apply one already tolerates a transient 5xx by comment). | | -| c26 recovery (run 34036875433, `mode=rollback`) | Gate 9 (run 34036059275) **green** 13:41Z at gen 177 with c26 migration-only. Recovery **succeeded** 13:42–13:46Z: `ROLLBACK_RESUME=true`, no restart, verifier + trust probe passed, activate → **gen 178**, c26 general. 176 controls, `sqlFailuresDelta` 0, no exits since 13:25Z. **All 16 US general cells are on `4916ed67`.** | | -| Monitor dry-run (Roll 2 gate 10) | run 34037169783 dispatched 13:48Z at gen 178, **green** 14:03Z at `f952f1ac96`. | | -| c27 `canary-apply` (run 34037973681, Asia, protocol 0) | **Succeeded** 14:03–14:19Z, gen 178 → 180, canary authority sealed (unused; Asia cells roll as single canaries). Template on `4916ed67`, no boot exits, 51 controls by 14:20Z (Asia cell, refilling), `sqlFailuresDelta` 0, `sqlLatencyMsMax` ~1 040 ms (cross-region baseline, c28 on the old image reads ~1 055 ms). Fleet `sqlFailuresDelta` 5 over 30 min: c28 ×3 (~1.17 s), c8 and c9 ×1 (1 s bar), the known lock-wait singles. | | -| Monitor dry-run (Roll 2 gate 11) | run 34038869552 dispatched 14:21Z at gen 180, **green** 14:36Z at `f952f1ac96`. | | -| c28 `canary-apply` (run 34039710735, Asia, protocol 0) | **Succeeded** 14:36–14:53Z, gen 180 → 182. Template on `4916ed67`, no boot exits, 37 controls by 14:55Z (refilling), `sqlFailuresDelta` 0, `sqlLatencyMsMax` ~1 045 ms. Fleet `sqlFailuresDelta` 3 over 30 min. | | -| Monitor dry-run (Roll 2 gate 12) | run 34040698172 dispatched 14:56Z at gen 182, **green** 15:12Z at `1d2e00819f`. | | -| c29 `canary-apply` (run 34041558414, Asia, protocol 0) | **Succeeded** 15:12–15:28Z, gen 182 → 184. No boot exits, 55 controls by 15:29Z. | | -| Census 15:29Z | MIG templates: 18 of 19 general cells on `4916ed67`; **c21 still on `519f4914`**. When c13's recovery re-sealed the canary at c14, batch 2 took c15,c16,c19,c20 and c21 dropped out of the plan's wave (`c15 canary + c16,c19,c20,c21`). Fleet 23 cells, 2 971 controls. Roll 2 exits since 07:00Z: 20, all boot-race (<10 s), 0 serving. Director retries 5 in the last hour. c21 rolls next as a single canary. | | -| Monitor dry-run (Roll 2 gate 13) | run 34042460176 dispatched 15:30Z at gen 184, **green** 15:45Z at `3631f886a7`. | | -| c21 `canary-apply` (run 34043296422, protocol 1) | **Failed at the same post-apply step as c26.** Isolate → gen 185, drain, template `…-20260906155550…` on `4916ed67`, verifier passed at migration-only 16:04:46Z (2 607 assignments, heartbeat fresh, `4916ed67`), then `admin_post target-runtime` to c21 got **503 `unconditional drop overload`** again (27-byte body, ~160 ms after the verifier's own successful read). Failsafe held migration-only at gen 185. c21 serving on the new image, isolated, 111 controls by 16:07Z. Second occurrence in ~3 h on two different cells, both ~1.3 min after readiness: consistent with an edge shed on the first admin request after the LB backend flips healthy. The step needs the same transient-5xx tolerance as the pre-apply read. | | -| Monitor dry-run (Roll 2 gate 14) + c21 recovery | Gate run 34044440616 dispatched 16:08Z at gen 185 with c21 migration-only. On green: `mode=rollback` resume for c21 (rollback digest `4916ed67`, protocol 1). | | -| c21 recovery (run 34045296151, `mode=rollback`) | Gate 14 **green** 16:23Z. Recovery **succeeded** 16:24–16:28Z: no restart, verifier + trust probe passed, activate → **gen 186**, c21 general. 164 controls, `sqlFailuresDelta` 0. | | -| **Roll 2 complete** 16:29Z | **All 19 general cells on `4916ed67`** (c7–c10, c13–c16, c19–c29); existing-only c1–c6, c11, c12 and migration-only c17, c18 untouched. Selector gen 148 → 186. Fleet 23 cells, 2 927 controls. Container exits 07:00–16:29Z: 20, every one a boot-race exit (<10 s, `cloud-sql-proxy` sidecar not yet listening), **0 serving-process exits**. Director on `00575-leq` (`4916ed67`) since 01:02Z: Postgres retries 0 in the last hour (13 over the first 6 h vs 85 on the predecessor), 5xx in the last hour 104 `/v1/assign` 503s (admission `Retry-After` path, at the pre-roll rate). Three waves needed the no-restart `mode=rollback` resume (c13: transient trust-probe 409; c26 and c21: post-apply `runtime-status` 503 `unconditional drop overload`), each recovered in ~4 min with no drain. 14 monitor gates, 14 green, 0 freezes. | | diff --git a/cloud/docs/relay-roll2-plan-2026-09.md b/cloud/docs/relay-roll2-plan-2026-09.md deleted file mode 100644 index fb50acc8950..00000000000 --- a/cloud/docs/relay-roll2-plan-2026-09.md +++ /dev/null @@ -1,159 +0,0 @@ -# Relay Roll 2 and close-out plan (2026-09-05) - -Owner-approved scope 2026-09-05: finish the relay reliability work with one more cell image roll, -deferring the Cloud SQL private-IP move (2.1, orca-cloud #477) to a separate owner decision. Roll 1 -is complete (see `relay-reconnect-2026-09-findings.md`, "Roll 1 complete"); every serving cell runs -`519f4914` except c7 on `85bf6799`. - -Estimate: about two working days of effort over one week of calendar time. The cell roll itself is -6 to 7 hours of mostly unattended wall clock, run in the US night. - -## Phase 0. Land the code (half a day, no production change) - -### 0a. Split PR #18565 - -The branch mixes three relay/mobile/desktop fixes with the operator record. Split so the record -lands regardless of how the code review goes. - -- **Docs PR** (new branch off main): `relay-reconnect-2026-09-findings.md`, - `relay-improvement-checklist-2026-09.md`, `relay-improvement-roadmap-2026-09.md`, this file. - Docs only, merge on CI green. -- **Code PR** (rebase #18565 onto main, resolve two conflicts): - - `cloud/apps/relay/src/host-session-registry.ts`: conflict with #18698 (signed-out signal). - Keep both; the accept-abandonment and lease changes are orthogonal to the signed-out path. - - `src/main/runtime/mobile-cloud-relay/relay-origin-pool.ts`: **drop this branch's version**. #18719 already - merged the desktop early-window jitter (1 to 6 min). Also drop - `relay-session-broker.test.ts` additions that only exercise the dropped change. - - Keep: relay accept abandonment (`orca_relay_client_accept_abandoned` event), relay-side lease - jitter, mobile direct-probe fail-fast, and their tests. - -### 0b. Lengthen the control lease (same code PR) - -In `cloud/apps/relay/src/host-session-registry.ts`: - -``` -CONTROL_LEASE_MS = 6 * 60 * 60 * 1000 // was 55 min -CONTROL_LEASE_JITTER_MS = 30 * 60 * 1000 // was 5 min -``` - -Why 6 h: the lease bounds how long a host stays on a cell after a missed drain and is the only -passive rebalancing; 6 h keeps both and cuts control-activation traffic on the inventory lock by -about 6x. Nothing else depends on it: the relay JWT (5 min) is refreshed by the desktop on its own -schedule and liveness is the 75 s silence watchdog. Wire-safe: the relay sends `leaseExpiresAt` in -the hello ack and old desktops schedule from that value. - -Update the comment above the constants and the three assertions in -`host-session-client-accept.test.ts` that pin the lease arithmetic. Check that nothing in -`cloud/apps/relay-ops` or the monitor thresholds assumes a 55 min rotation period (grep -`55`, `CONTROL_LEASE`, `rotation`). - -### 0c. Review and merge - -Review rounds per the standing process (Opus review, then Codex pass). Merge order: docs PR first -(no dependency), then the code PR. Record the merge SHA of the code PR; that is the Roll 2 image -source. - -## Phase 1. Build and stage the image (half a day) - -Roll 2 image = code PR merge SHA. It carries, relative to `519f4914`: - -| Change | PR | Effect | -|---|---|---| -| Per-cell inventory locks, delta counters | #18722 | Removes the global `relay_cells FOR UPDATE` behind the phone accept hang | -| Relay pool `statement_timeout` 5 s | #18722 | A relay query can no longer hang a cell | -| Accept abandonment | #18565 | Cell stops finishing accepts for phones that already closed | -| Control lease 6 h ± 30 min | #18565 | Fewer, spread-out rebinds | -| `--private-ip` proxy flag support | #18720 | Code only; flag stays unset until 2.1 | - -Steps, in order (from the findings doc's post-merge dispatch plan): - -1. `gh workflow run cloud-publish-relay-production.yml --ref main -f mode=publish`. Resolve the - digest by tag, not from the log: - `gcloud artifacts docker images describe us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay:sha- --format='value(image_summary.digest)'`. -2. Staging: `cloud-deploy-relay-staging.yml` with the new digest; paired phone plus desktop smoke - (connect, background, reconnect). Confirm `orca_relay_client_accept_abandoned` appears only when - a client closes early, and that `sqlLatencyMsMax` no longer pins at the lock timeout. -3. Director: `cloud-deploy-relay-production-director.yml -f image-digest= - -f regional-placement-mode=preserve -f prune-incompatible-revisions=false - -f expected-rehome-generation=12 -f bootstrap-runtime-identity=false - -f predecessor-image-digest=`. Blue/green; prior revision stays as rollback. - Watch director `orca_relay_postgres_transaction_retry` per minute before and after. The director - goes first so the per-cell locks are live before any cell restart burst. -4. Same-cap `verify` mode against c7 with target=, rollback=`519f4914`. Read-only. - -Go/no-go for Phase 2: director serving the new image for at least 30 min, retries per minute at or -below the pre-deploy baseline, no `container die`, no auth 5xx. - -## Phase 2. Roll the cells (one US night, mostly unattended) - -Same machinery as Roll 1: `cloud-monitor-relay-production.yml` dry-run gate, then -`cloud-deploy-relay-production-same-cap.yml`. Cells roll one at a time by design (exact selector -assertions, single Terraform state, and one cell's ~1.2k-host reconnect burst per restart). Do not -add parallelism for this roll. - -Inputs: target=, rollback=`519f4914` (c7: rollback=`85bf6799`). Selector membership is -unchanged from the end of Roll 1 (gen 148; existing-only c1–c6, c11, c12; migration-only c17, c18). - -Order: - -1. **c7 canary** (`canary-apply`, protocol 1). c7 is the rehearsal cell and the only one not on - `519f4914`. -2. **c8 canary**, then **batch c9, c10, c13, c14**. -3. **c15 canary**, then **batch c16, c19, c20, c21**. -4. **c22 canary**, then **batch c23, c24, c25, c26**. -5. **Asia c27, c28, c29** as three single canaries at protocol 0 (`PROTO=0`). Batch mode cannot - take Asia cells yet and needs at least two cells. - -Each batch needs a same-commit canary authority; each wave needs a fresh 15 min gate. Use the -chain script pattern from Roll 1 (wait gate green, check trusted-path ancestry, dispatch within 5 min, -log `CANARY `) under `caffeinate -i`. Budget: 11 to 13 min per cell plus 15 min per gate, -about 6 to 7 h total. - -Per wave checks (same as Roll 1): transition verifier passes at migration-only and again at general -with assignments carried; no `container die` fleet-wide; selector generation advances by exactly 2 -per cell. After the Asia cells: image census from MIG templates; every general cell on the new digest. - -Failure handling: a failed canary re-enters through `mode=rollback` with rollback-digest = desired -image (Roll 1 c27 pattern). A gate freeze on an Asia latency probe despite the 4 000 ms bar is a -stop-and-investigate, not a retry. Monitor-side freezes (freshness, continuity deadline) re-gate -after a 2 min back-off; the chain does this on its own. - -Record every gate and wave in the findings doc as in Roll 1. - -## Phase 3. After the roll (spread over the following week) - -- **4.4 Recalibrate the retries bar.** After one week of `orca_relay_postgres_transaction_retry` - on the new image, re-derive the `postgres_retries` monitor threshold from the new baseline - (PR against `cloud/apps/relay-ops/src/incident-monitor.ts` thresholds). About 2 h. -- **1.2 Pruner budget.** Raise `auth_token_pruner_max_rows_per_run` to the default 200k after a - clean day; watch Cloud SQL write MB/s and the checkpoint alert. Then **1.5** log metric plus - policy on `stopReason != complete`. -- **1.3 Reclaim.** Once pruner runs delete ~0 rows: `pg_repack -t refresh_tokens` off-peak (check - `pg_available_extensions` first; not `VACUUM FULL`). Confirm table, index, and `disk/utilization` - dropped. -- **Monitor residuals** already in the checklist: `probeEndpointHealth` retry decision still uses the - flat 2 000 ms bar; operator protocol unbound for Asia; `probe-relay-rehome-trust` regex. -- **Same-cap job residuals found in Roll 2** (three of eleven mutating runs needed the resume path): - the post-apply `admin_post target-runtime` read has no transient-5xx tolerance and failed twice on a - one-request 503 `unconditional drop overload` from the edge ~80 s after readiness (c26, c21); and - `probe-relay-rehome-trust` prints only the status on a 409, so the transient c13 failure left no - reason on record. Retry both once and print the error body. -- Update the checklist status header; tick 2.3, 4.1, 4.3 relay-side as deployed. - -## Deferred, owner decision required - -- **2.1 Private IP** (orca-cloud #477). One-way door with a Cloud SQL restart. When chosen: apply the - foundation off-peak, then a template-only change that sets the `--private-ip` proxy flag. That is - another cell roll unless bundled with a future image. -- **5.2 Paging channel** for auth alerts: needs a destination. -- **Parallel cell rolls** (2 or 3 at a time): about 1.5 days (relax exact-selector assertions to - "exact except in-flight", single coordinator Terraform apply, parallel job shape, tests). Only - worth building if more image rolls are planned after Roll 2, and only once the per-cell locks are - live so a multi-cell reconnect burst is safe. -- **2.2 Database split**: deferred to ~2026-11-01. - -## Not in this plan - -Desktop and mobile changes already merged (#18719 desktop early-window jitter and no same-token -refresh retry; #18565 mobile fail-fast once merged) ship with the next desktop and mobile releases -on their own schedules. No relay action needed. diff --git a/cloud/docs/relay-terraform-fencing-plan.md b/cloud/docs/relay-terraform-fencing-plan.md deleted file mode 100644 index 7d8664f19f5..00000000000 --- a/cloud/docs/relay-terraform-fencing-plan.md +++ /dev/null @@ -1,149 +0,0 @@ -# Relay Terraform Fencing Recovery Plan - -Status: private broker implementation in review; no production mutation performed - -## Current status - -- `2d7dc31` commits Terraform-owned per-cell zero/one desired size and removes - the lifecycle ignore. -- The follow-up slice implements private exact saved plans, SHA-256 binding, - durable attempt evidence, lost-response inspection, resume, proven - pre-apply abort, and exact-incarnation attestation. -- Focused Terraform-fence, multi-target, relay database, assignment-store, and - black-box tests pass locally. -- Production remains untouched. A rollout still requires review, CI, the - director/schema deployment, an initial output-state refresh, and a separate - reviewed fence-set commit for any cell selected during an incident. -- The private Cloud Run broker owns the exact Terraform checkout, saved plans, - state access, Compute mutation, and a durable GCS mutation lease. The - workflow requester can invoke the broker and read aggregate evidence but - cannot mutate Terraform state, Compute, or director fence routes. - -## Goal - -Make production relay-cell fencing a Terraform-managed, resumable operation. -The workflow must never attest a cell as fenced until Terraform state, GCE, -runtime identity, and retained routing all prove the same exact cell -incarnation is offline. - -## Safety boundary - -- Do not deploy, mutate GCP, push, or create a pull request during implementation. -- Keep the cell route and backend while its MIG is fenced at size zero. -- Treat any Terraform apply whose start cannot be disproved as recover-forward. -- Never restore a fenced cell automatically after an apply may have started. -- Store plans in a private temporary directory and remove them on every exit. -- Never print, upload, or commit complete plan JSON. -- Fence modes are fail-closed until a private mutation broker owns the narrow - state/plan and exact-cell Compute permissions. The exact-workflow GHA fence - account must never receive those permissions or director mutation routes - directly; it has only aggregate reads and fence-attempt status. -- The monitor identity has no Terraform state access. It can call only the - aggregate selector, cell, evacuation, and runtime status routes. -- The broker accepts only its configured source, failed target, and - replacement target. Its image commit must equal the reviewed fence commit; - callers cannot override topology, Terraform paths, commands, or identities. - -## State model - -1. Add `relay_gce_fenced_cells` as the committed set of cell IDs whose desired - MIG size is zero. -2. Derive every cell's desired size from that set: fenced is zero, otherwise - one. -3. Reject unknown fenced IDs and any topology outside the zero-or-one - invariant. -4. Remove the MIG `target_size` lifecycle ignore so Terraform owns the fence. -5. Persist one durable attempt record keyed by an unguessable attempt ID. It - binds environment, cell ID, exact cell incarnation, MIG/generation - identity, fence commit, saved-plan digest, GCE operation, creation time, - expiry, and terminal status. - -## Operation sequence - -### 1. Prepare and guard - -1. Require the explicit fence confirmation and a clean checkout at the exact - fence commit. -2. Confirm the committed production fence set contains the requested cell. -3. Read Terraform state and live topology, then bind the attempt to the exact - MIG, instance group, backend, origin, and cell incarnation. -4. Run the existing admission, assignment, lease, migration, connection, - heartbeat, backend, and route guards before planning. - -### 2. Create and validate the exact plan - -1. Create a private temporary directory with mode `0700` under `umask 077`. -2. Write the saved plan there and compute its SHA-256 digest. -3. Inspect only narrow fields from `terraform show -json`. -4. Require exactly one relevant in-place MIG update from target size one to - zero. -5. Reject create, delete, replace, unrelated update, route/backend removal, or - any different cell action. -6. Persist the pending attempt evidence before apply. - -### 3. Apply or recover forward - -1. Recheck the pre-apply guards and saved-plan digest immediately before apply. -2. Apply the exact saved plan with Terraform locking. -3. If apply fails or its response is lost, inspect Terraform state, live MIG - size, remaining instances, and the relevant GCE operation. -4. If apply may have started, retain the committed fence set and keep polling - forward until the MIG converges to zero or a bounded, diagnosable failure is - recorded. -5. Resume idempotently from durable evidence after workflow or runner loss. - -### 4. Abort before apply - -1. Allow abort cleanup only when evidence proves apply never began. -2. Require Terraform state and live MIG to remain at one with the original - identity and topology. -3. Mark the attempt aborted, remove the cell from the committed fence set - through a separately reviewed commit, and delete local plan artifacts. -4. If apply start is ambiguous, refuse abort and recover forward. - -### 5. Attest - -1. Require Terraform state target size zero. -2. Require live MIG target size zero and no managed instances. -3. Require the exact MIG/generation identity and retained route/backend - topology to match the attempt. -4. Require admission disabled and the exact runtime heartbeat stale. -5. Require unexpired durable attempt evidence and the same saved-plan digest, - fence commit, and GCE operation. -6. Record the exact-incarnation cell fence and complete the attempt in one - database transaction. - -## Implementation slices - -- Terraform: variable, per-cell desired size, validation, outputs, tfvars, IAM. -- Relay contract: durable attempt schema, store methods, admin endpoints, exact - attestation binding, retention/expiry. -- Deployment tooling: private plan lifecycle, digest and selector validation, - apply/recovery/abort state machine, GCE operation inspection. -- Workflow: explicit prepare/apply/resume/abort modes and no direct MIG resize. -- Runbooks: committed fence-set, exact-plan, recover-forward, and reviewed - abort procedures. - -## Required tests - -- Normal fence plan and apply. -- Lost or ambiguous apply response recovers forward. -- Resume when Terraform state and live MIG are already zero. -- Pre-apply guard failure performs no mutation. -- Proven pre-apply abort permits committed fence-set cleanup. -- Ambiguous apply refuses abort cleanup. -- No attestation before every Terraform, GCE, route/backend, heartbeat, and - exact-incarnation condition passes. -- Saved-plan validation rejects unrelated, replacement, create, and destroy - actions. -- Attempt evidence rejects wrong environment, cell, incarnation, MIG, - generation, commit, digest, operation, expiry, and terminal state. - -## Validation - -- Focused Node and relay contract tests. -- `pnpm test`, `pnpm typecheck`, and `pnpm lint`. -- Sequential relay contract and relay builds after schema/API changes. -- Terraform 1.15.8 `fmt -check -recursive`, `init -backend=false`, and - `validate`. -- `git diff --check`, local commit, and clean worktree. diff --git a/cloud/docs/relay-workflows.md b/cloud/docs/relay-workflows.md deleted file mode 100644 index 267d38e06fb..00000000000 --- a/cloud/docs/relay-workflows.md +++ /dev/null @@ -1,927 +0,0 @@ -# Relay GitHub Actions Configuration - -The `cloud-*` workflows in `.github/workflows/` are the Relay deploy and -operate surface. Every one of them is gated on the repository variable -`ORCA_CLOUD_OPERATIONS_ENABLED == 'true'` and does nothing until the repository -owner sets it. The app and auth deploy workflows this document once also -covered stay in the private `stablyai/orca-cloud` repository. - -Set these staging environment variables before running the staging deploy workflow: - -```text -STAGING_GCP_REGION -STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER -STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT -STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER -STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT -STAGING_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER -STAGING_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT -STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER -STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT -``` - -`STAGING_GCP_REGION` exists today only as a repository variable. Create it as a -staging **environment** variable before deleting any repository-level variable; -`Deploy Relay Staging` gates its whole job on it being non-empty, so a -delete-before-create silently skips it. - -The Relay deploy, capacity, and Asia values come from the matching staging -Terraform outputs after the targeted identity bootstrap: - -```sh -terraform -chdir=infra/terraform output -raw github_staging_relay_deploy_workload_identity_provider -terraform -chdir=infra/terraform output -raw github_staging_relay_deploy_service_account - -gh variable set STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER --env staging --body '' -gh variable set STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT --env staging --body '' - -terraform -chdir=infra/terraform output -raw github_staging_relay_capacity_workload_identity_provider -terraform -chdir=infra/terraform output -raw github_staging_relay_capacity_service_account - -gh variable set STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER --env staging --body '' -gh variable set STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT --env staging --body '' -terraform -chdir=infra/terraform output -raw github_relay_asia_proof_workload_identity_provider -terraform -chdir=infra/terraform output -raw github_relay_asia_proof_service_account -gh variable set STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER --env staging --body '' -gh variable set STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT --env staging --body '' -``` - -The capacity provider accepts only this repository's capacity proof and -bootstrap workflows on `main` with the `staging` environment. It does not fall -back to the shared deploy identity. - -The Relay deploy provider accepts exactly five workflows on `main` with the -`staging` environment: Bootstrap Relay Staging Capacity, Deploy Relay Staging, -Deploy Relay Staging GCE Candidate, Operate Relay Asia Admission, and Power -Relay Staging. Set both `STAGING_GCP_RELAY_DEPLOY_*` variables before merging -the workflow repoint; the job gates read them and skip while they are unset. - -Set these separately before enabling production deploys: - -```text -PRODUCTION_GCP_REGION -PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER -PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT -PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER -PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT -PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER -PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT -PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER -PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT -PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER -PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT -PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT -PRODUCTION_GCP_RELAY_RUNTIME_SERVICE_ACCOUNT -``` - -The Relay operations values come from matching Terraform outputs. Set them as -production GitHub environment variables, not repository fallbacks. Every one of -them is relay-owned in `infra/terraform`: - -```sh -terraform -chdir=infra/terraform output -raw github_workload_identity_provider -terraform -chdir=infra/terraform output -raw github_deploy_service_account -terraform -chdir=infra/terraform output -raw github_relay_monitor_workload_identity_provider -terraform -chdir=infra/terraform output -raw github_relay_monitor_service_account -terraform -chdir=infra/terraform output -raw github_relay_fence_workload_identity_provider -terraform -chdir=infra/terraform output -raw github_relay_fence_service_account -terraform -chdir=infra/terraform output -raw github_production_relay_capacity_workload_identity_provider -terraform -chdir=infra/terraform output -raw github_production_relay_capacity_service_account -terraform -chdir=infra/terraform output -raw relay_director_runtime_service_account -terraform -chdir=infra/terraform output -raw relay_runtime_service_account - -gh variable set PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER --env production --body '' -gh variable set PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT --env production --body '' -gh variable set PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER --env production --body '' -gh variable set PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT --env production --body '' -gh variable set PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER --env production --body '' -gh variable set PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT --env production --body '' -gh variable set PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER --env production --body '' -gh variable set PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT --env production --body '' -gh variable set PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER --env production --body '' -gh variable set PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT --env production --body '' -gh variable set PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT --env production --body '' -gh variable set PRODUCTION_GCP_RELAY_RUNTIME_SERVICE_ACCOUNT --env production --body '' -``` - -Run those commands only from the audited operator session after the targeted -identity bootstrap apply. The providers require their exact workflows on -`refs/heads/main` with the `production` environment. Missing values fail -closed; no dedicated operations identity falls back to the shared deploy identity. - -The shared production identity is restricted to nine named direct Relay callers plus the exact -regional-rehome and same-cap reusable wrapper/job pairs on `main` in the `production` environment. -Its Artifact Registry and Cloud Run mutation permissions are scoped to the Orca repository, Relay -director, and Relay fence broker; it cannot mutate the API or auth services. - -Bootstrap the production capacity identity only after its reviewed commit is on -`main`. Reinitialize the production backend explicitly, save the exact targeted -plan, require **9 additions, 0 changes, and 0 deletions**, then apply that saved -plan. `manage_artifact_dns=false` keeps the unimported Cloudflare records out of -this GCP-only operation. - -```sh -export GOOGLE_OAUTH_ACCESS_TOKEN="$(gcloud auth print-access-token)" -terraform -chdir=infra/terraform init -reconfigure \ - -backend-config=backend/production.hcl -input=false -terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \ - -var-file=environments/production.tfvars -var manage_artifact_dns=false \ - -target=google_iam_workload_identity_pool_provider.github_production_relay_capacity \ - -target=google_service_account.github_production_relay_capacity \ - -target=google_service_account_iam_member.github_production_relay_capacity_workload_identity_user \ - -target=google_project_iam_custom_role.github_production_relay_capacity_mutation \ - -target=google_project_iam_member.github_production_relay_capacity_mutation \ - -target=google_project_iam_member.github_production_relay_capacity_viewer \ - -target=google_project_iam_member.github_production_relay_capacity_artifact_reader \ - -target=google_storage_bucket_iam_member.github_production_relay_capacity_state \ - -target=google_service_account_iam_member.github_production_relay_capacity_runtime_user \ - -out=/tmp/orca-relay-production-capacity-identity.tfplan -terraform -chdir=infra/terraform show /tmp/orca-relay-production-capacity-identity.tfplan -terraform -chdir=infra/terraform apply /tmp/orca-relay-production-capacity-identity.tfplan -unlink /tmp/orca-relay-production-capacity-identity.tfplan -``` - -Confirm a second targeted plan is empty before setting the two production -environment variables from reviewed Terraform outputs. - -`Deploy Relay Asia Topology` is the only workflow allowed to add the reviewed -`asia-east2` network and fixed-one cell topology. Its dedicated identity is -bound to that exact workflow, `main`, `workflow_dispatch`, and the selected -GitHub environment. The workflow always saves a targeted plan, rejects any -delete, replacement, US-resource, SQL, DNS, certificate, or unrelated change, -and applies only the exact validated plan. It always passes -`manage_artifact_dns=false`; observability and IAM are separate targeted -operations. - -Before the first admission operation, publish and deploy a compatible director image while the -topology remains unchanged. Verify the exact serving digest, health, readiness, and rollback tag; -older directors reject the generation-zero membership fingerprint. After a topology apply, use -`Operate Relay Asia Admission` in `inspect` mode to read the exact live selector generation. If and -only if it is generation 0, run -the explicit `initialize` mode with the exact membership SHA-256 printed by -`inspect` and `INITIALIZE_ADMISSION_SELECTOR`; the director checks both under its -database lock, so this freezes the existing membership without adding, removing, -or moving a cell and rejects intervening drift. Then -atomically register the new cells as migration-only, binding every mutation to -the exact live selector generation and a durable attempt ID. Deploy and verify -the director configuration only after registration, then promote C27 alone before C28/C29. - -The production Asia set is C27-C31. C27-C29 launched as one wave; C30 and C31 are each an additive -wave of their own at the same shape, and C31 takes `asia-east2-b` so the five cells spread 2/2/1 -across the zones. The C30 steps below apply to C31 unchanged, with C31 in place of C30. C30's plan -names its template, MIG, and backend plus the shared URL map, and the URL map pulls every existing -cell's backend, MIG, and template into the plan. Committed images lag what same-cap rolls serve, -so the workflow first reads each non-target cell's served image out of its live template in state -and plans that cell at it. It reads the committed cell map from a -no-refresh, unlocked plan over the same targets, not `terraform console`. Console evaluates every -output against state, so `relay_gce_cell_deployments` wraps each per-cell resource lookup in -`try`: until C30's topology apply, console succeeds and that output shows C30 with null MIG, -backend, and template fields. The validator then rejects any change to a -cell outside the wave, so the plan must read as C30's three creations plus the URL map update. -Before the apply dispatch, run the plan mode and read its `Plan:` line; any other drift, such as a -cell whose startup script changed since its last roll, fails the plan and must be rolled first. -Register C30 alone as migration-only, configure the director with `cell-ids` set to C30 while -regional rehome is paused, then promote it alone. Promotion requires C27-C29 to be general and takes -no input evidence. It runs the same five-minute production control and splice canary C27 ran, on -C30: the evidence must show the canary control was placed on C30, read C30's own runtime metrics, -and bind the selector generation, and any failure returns C30 to migration-only. The SQL-failure and -database-pool rules read C30's own metrics only. Director values are recorded under -`director`-prefixed names but do not fail the canary, because directors show a steady baseline of -`relay_cells` lock refusals and pool waits unrelated to C30. Director region fallbacks are keyed by -the host's target region, and the canary fails on any Asia-targeted one. US-targeted fallbacks are -recorded but not gated: they are placement-lane requests from unhinted or US-preferring hosts, which -an Asia cell cannot cause. Staging still requires exactly its one -intentional fallback. C30 was promoted to general on -2026-09-23, so the same-cap job now rolls it as a general cell and the shadow gate's fleet pool list -reads it beside C27-C29. A later Asia cell stays in the same-cap migration-only list and out of the -fleet pool list until its own promotion, then moves to both together, as its own reviewed wave. -C31 followed that path and was promoted to general on 2026-10-01, so it is now a same-cap general -cell and in the fleet pool list beside C27-C30. - -C32 and C33 are US cells at that same 3,000-connection shape, in `us-central1-a` and -`us-central1-b`, and use the same two workflows and the C30 steps. They are declared together, so -they share one topology wave: the live-image step refuses a declared non-target cell with no -template, so a lone C32 plan would fail on C33. Registration, director configuration, and -promotion still take one cell at a time. Each wave's region comes from its reviewed zone. A US wave plans no additional-region network, and its template -carries no region label or region line. Its pool stays at the US default of 10 and emits no pool -line, because 16 exists only for the asia-east2 round trip. Registration and the runtime check -expect `us-central1`. Promotion skips the Asia launch-order gates, which bind Asia cells only. -The canary aims its load at `us-central1`, reads the cell's own `us-central1` metrics, and -requires a US selection. It gates neither region's fallbacks: US-targeted fallbacks have a -nonzero baseline while the US fleet is full, and Asia-targeted ones are not the cell's to cause. -Both are recorded. The US selection gate is a fleet-level check that director metrics flowed; the -placement check is what proves the cell. Placement breaks a load-ratio tie by cell ID, so do not -promote while a same-cap restore has just returned an empty general US cell: the canary control -would land there and the canary would roll the new cell back. Both cells are declared rehome -sources and sit in the same-cap migration-only list until each one's canary promotes it, then move -to the general list. The shadow gate's fleet pool list tracks the 16-connection Asia pools, so -whether a US cell belongs there is decided at promotion, not assumed. Both were promoted to general -on 2026-10-01, so the same-cap job now rolls them as general cells. They stay out of the fleet pool -list because their pool is the US default of 10. - -C34 is a sixth Asia cell at the C31 shape in `asia-east2-c`, so the six Asia cells spread 2/2/2. It -was its own topology wave, registered alone as migration-only, and the director was configured with -`cell-ids` set to C34, all on 2026-10-05. It launched as a migration-only spare and has a promotion -wave of its own, with the same five-minute canary C30 and C31 ran. Promotion compares the -director's serving digest and C34's runtime digest with the one `image-digest` input, so C34 was -first rolled to the director's image as a migration-only same-cap wave. Once its canary promoted -it, it moved to the same-cap general list and the fleet pool list together, so the same-cap job now -rolls it as a general cell and a rollback restores it general. It is a declared rehome source. - -C35 is a seventh Asia cell at the C34 shape in `asia-east2-a`, starting a third zone round. It was -its own topology wave on image F (`sha256:41cba995…`), registered alone as migration-only, and then -the director was configured with `cell-ids` set to C35. It launched as a migration-only spare and has -a promotion wave of its own, with the same five-minute canary C30, C31 and C34 ran. Promotion -compares the director's serving digest and C35's runtime digest with the one `image-digest` input, -so C35 was first rolled to the director's image as a migration-only same-cap wave. Once its canary -promoted it, it moved to the same-cap general list and the fleet pool list together, so the same-cap -job now rolls it as a general cell and a rollback restores it general. It is a declared rehome source. -Its pool of 16 leaves 3 connections under the usable Cloud SQL ceiling -(`relay-cloud-sql-connection-budget.mjs`). -Rollback returns -Asia cells to migration-only; it does not destroy the network or use -existing-only. The production topology dispatch remains unavailable until the -published compatible image is committed for C27-C29. - -`Prove Relay Asia Staging` runs from a dedicated ephemeral repository runner in -`asia-east2` with the `relay-asia-east2-load` label. It promotes only staging C4, runs four bounded -load shards at the exact 3,000/6,000 shape, validates continuous cell/director/Cloud SQL evidence, -and always returns C4 to migration-only before publishing evidence. Register the runner with -`--ephemeral` immediately before dispatch so it accepts one proof job and then removes itself. Each -shard exchanges its -exact workflow OIDC identity for a ten-minute in-memory staging token; no load -credential, signing key, or raw load output is stored or uploaded. - -Production promotion evidence must prove the exact production manifest, not an independent rebuild. -Before refreshing C4, target and apply only -`google_artifact_registry_repository_iam_member.github_production_relay_staging_mirror_writer` -from the staging state with `manage_artifact_dns=false`. Run `Publish Relay Production Image` in -`mirror-staging` mode with the exact digest and typed confirmation, then run `Deploy Relay Staging` -with that digest. The mirror validates identical source and target manifest digests; the staging -deploy binds the request to C4's checked-in image and deploys the director by that same digest. -Only then refresh empty migration-only C4 and run the proof. The proof and production promotion both -reject a serving director whose runtime digest differs from the cell/evidence digest. - -Expected project IDs: - -```text -staging: onorca-cloud-staging -production: onorca-cloud -``` - -Production relay delivery keeps the stable director separate from GCE cell rollout. `Publish Relay -Production Image` builds and prints an immutable digest. `Deploy Relay Production Director` accepts -only that digest, performs a health-gated Cloud Run director update, and never deploys data-plane -cell stamps. Its explicitly confirmed prune option retains only the serving and cold rollback pair, -and runs only after both compatible revisions pass the capacity-protocol health gate. Use that gate -before adding Asia cells so an incompatible dormant revision cannot be routed later. A reviewed -Terraform candidate pins the same digest on a distinct disabled GCE cell; -`Deploy Relay Production Candidate` then runs read-only preflight or an explicitly confirmed -target-first evacuation. Staging uses the same GCE data-plane shape as production; `Deploy Relay -Staging GCE Candidate` exercises the reviewed GCE preflight and evacuation state machine before -production use. - -`Prove Relay Staging Capacity` is the only cap-transition path. Apply mode -reversibly moves `staging-gce-c3` to migration-only, drains it, validates the -saved director and C3 Terraform plans, updates the director first, and requires -stale telemetry before replacing the exact C3 template and MIG. A fresh -matching heartbeat is required before C3 becomes the sole general placement -cell; C2 remains recoverable in migration-only. Restore mode does not depend on -Terraform or image agreement: it restores C2 first, then restores C3 only after -a fresh, healthy, non-draining capacity check. The same transition order -restores 600 before an older director image can be used. - -Its bounded C4 refresh mode keeps Asia admission migration-only, accepts only an exact predecessor -or already-applied target digest, validates a saved two-resource image-only plan, fences and proves -C4 empty before replacement, and requires an empty targeted readback afterward. It cannot change -C4 capacity, routing, trust configuration, or any production resource. - -`Recover Relay Staging C4 Image` runs independently after a failed, timed-out, or cancelled C4 -refresh and can also be dispatched with `RECOVER_STAGING_ASIA_C4_IMAGE`. It verifies the triggering -job and both exact Terraform end states, preserves a fully converged ready target or predecessor, -and fences partial state before restoring the pinned predecessor through an exact saved two-resource -plan. A separate no-credential supervisor requeues a recovery cancelled while waiting for the shared -staging mutation lane. Admin credentials are refreshed around Terraform. Before the first refresh, -target only -`google_iam_workload_identity_pool_provider.github_staging_relay_capacity`, require exactly one -in-place condition update, apply the saved plan, and require an empty targeted readback. - -This identity cannot bootstrap its own Relay authorization. Before the first -capacity dispatch, use the existing audited staging blue/green deploy path to -roll the compatible image and verified `ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT` -onto both director revisions. Roll C2/C3 through saved, validated cell plans -with `Bootstrap Relay Staging Capacity` while they remain at 600/60. That -workflow keeps the deploy identity only for Relay admin calls and uses the -capacity identity for Terraform and GCP mutations. It isolates, drains, rolls, -verifies, and restores one cell at a time, with the other cell as its failure -fallback. Commit the matching -C2/C3 image and capacity pairs in staging tfvars, then require the capacity workflow's read-only 600/60 -verification to pass. Only a later reviewed configuration commit may select -1,000/0 or 1,000/60. The capacity workflow carries the reviewed director -topology through blue/green; it never targets the drifted director or its Cloud -SQL dependencies with Terraform. - -The one-time bootstrap recognizes only the exact pre-capacity C2/C3 image and -its five-field runtime-status response. It first proves C2 as the general fallback, -then isolates and drains C3 and requires zero durable activity plus two fresh, -instance-bound zero runtime metrics. After restarting the fixed-one MIG, it -requires two new zero samples and a new director heartbeat incarnation started -after the restart before restoring C3. This clears the legacy process's -unreported drain flag without treating missing runtime fields as proof. Any -other image, response shape, activity, or stale evidence fails closed with C2 -preserved as the general fallback. Reruns classify partial C2/C3 progress. A -legacy target may carry only no capacity record or the exact stale 600/60 record -before the idempotent director update; afterward the exact stale record is -required until that cell is replaced. - -`Power Relay Staging` lowers the staging bill when no internal testing is underway. It runs a -guarded sleep attempt at 09:00 UTC every day and also supports manual `status`, `wake`, and `sleep` -dispatches. Manual mutations require the exact `WAKE_STAGING` or `SLEEP_STAGING` confirmation. -Sleep refuses to stop a cell with active Relay work, disables admission and checks again, then -scales the three GCE MIGs to zero and stops the shared staging Cloud SQL instance. Wake starts SQL, -waits for healthy workers and authenticated heartbeats, then restores only the admission state -declared in Terraform. The default wake starts c1/c2; choose `all` before a Terraform apply or GCE -candidate operation so the complete Terraform-owned topology is running. - -`Deploy Relay Production Multi-Target` handles a source that cannot fit on one -candidate. It serializes deterministic per-target quotas, enforces each -target's reviewed 600- or 1,000-connection gate and the ten-minute lease gate, -and treats a drain attempt as the -rollback point of no return. Fence and fence-abort remain fail-closed. -It also registers one additive migration cell and retires exactly one -migration-only cell through explicit, generation-bound selector operations. -Registered-target supersession invokes the IAM-only private broker, which owns -the durable mutation lease, exact Terraform checkout, saved plans, state, and -narrow Compute mutation. The workflow requester has read and broker-invocation -authority only; it never receives those mutation permissions directly. -`Deploy Relay Fence Broker` updates only that service's immutable image and -requires the digest to carry the exact `sha-${GITHUB_SHA}` tag. Terraform -continues to own its identity, scaling, IAM, environment, and deletion -protection. -Its `add-migration-cells` mode is the selector-safe path for newly provisioned -empty targets after generation 1. It requires `ADD_MIGRATION_CELLS` and a -stable selector attempt ID, but no pre-drain artifact because it moves no -assignments. Run the fresh 15-minute gate only after the new cells are -registered and healthy. - -## Cloud SQL rollout lease - -Every workflow that mints a Cloud Run revision or applies a relay instance template against a shared -Cloud SQL instance takes the compare-and-swap lease in `.github/actions/cloud-sql-rollout-lease` -immediately after `google-github-actions/setup-gcloud`. The per-repository `concurrency` groups -(`production-cloud-sql-rollout`, `relay-staging-mutation`) only serialize runs inside one repository; -once the relay workflows live in `stablyai/orca` there are two queues pointed at one instance, and -`relay-cloud-sql-connection-budget.mjs` computes `rolloutOverlap` as a `Math.max` that is only sound -with one rollout in flight. Keep both the groups and the lease. - -| Environment | Bucket | Object | -| ----------- | -------------------------------------- | --------------------------------------------------- | -| production | `onorca-cloud-terraform-state` | `terraform/state/cloud-sql-rollout/production.lock` | -| staging | `onorca-cloud-staging-terraform-state` | `terraform/state/cloud-sql-rollout/staging.lock` | - -`Deploy Relay Asia Topology` and `Operate Relay Asia Admission` pick the pair from -`inputs.environment`. `Deploy Relay Production Capacity` and `Deploy Relay Production Same-Cap` call -their reusable job several times per run, so every wave job acquires with `release: 'false'` under -the run-scoped default holder key and a single `if: always()` `release_lease` job frees it once every -wave has finished. - -`Monitor Relay Production` stays off the lease. It is read-only, holds only viewer roles, and putting -it on a durable lease would let monitoring block a rollout and a rollout block monitoring. -`dev/scripts/production-cloud-sql-rollout-lock.test.mjs` enforces the group, the lease wiring, and a -content-derived census of every rollout candidate against -`dev/scripts/cloud-sql-rollout-lock-census.mjs`. - -`Monitor Relay Production` is manual and read-only. Its `dry-run` mode enforces the 15-minute -pre-drain gate; `monitor` records a 90-minute incident watch. Both require the -operator to enter the exact selector generation and tri-state membership. The -workflow must use a dedicated identity for aggregate monitoring and -exact-audience read-only Relay-admin calls. Do not dispatch it until that -monitor identity, exact workflow-bound WIF trust, and read-only admin-route -authorization have been bootstrapped. -Capacity-transition monitoring binds the evidence to one exact general cell. It -still blocks all migration failures and any inactive registered migration from -that cell or another serving cell; it permits only inactive rows -from unrelated existing-only cells because a capacity restart neither creates -nor advances assignment migrations. -Reruns restore hash-verified private state from the prior attempt. Production -candidate and multi-target mutations require a fresh dry-run artifact and -recheck its exact selector and every live safety signal before any mutation -command. All three workflows share the production deployment lock, and each -passing dry-run artifact is marked consumed before the mutation starts. -The dry-run lineage fails closed after 25 total minutes, so continuity resets cannot extend the -15-minute gate indefinitely. -Missing or stale telemetry fails closed, and the workflow uploads only private aggregate -Markdown/JSON evidence. - -`Deploy Relay Production Capacity` is the only production cap-transition path. It runs only -from `main` and accepts exactly the current general rollout set: C7-C10, C13-C16, and C19-C26. -C17/C18 and every existing-only, draining, fenced, or disabled cell are excluded in code. Its -Terraform/GCE phase uses the dedicated exact-workflow capacity identity. Read-only checks, selector -isolation, drain, and the audited director blue/green update use the existing shared production -deploy identity; the production environment and common deployment lock still gate those steps. -Apply mode consumes a fresh 15-minute monitor gate bound to the selected cell, moves only that cell -from general to migration-only, drains it, updates only its director capacity entry, and applies a -saved validated plan for only its template and MIG. Previously completed 1,000 cells remain -unchanged while later 600 cells roll. The selected cell returns to general only after a fresh -matching 1,000/60 heartbeat. The restart gate waits up to 15 minutes for genuine activity to finish -while preserving every zero-work check. Rollback performs the same isolated sequence to 600/60 -without waiting on a cell that may already be unhealthy. If the selected cell cannot answer the -drain call, rollback instead requires two stale-heartbeat snapshots with zero durable activity -before replacing it. Its typed confirmation includes the exact selected cell so a form-selection -mistake cannot downgrade another cell. Interrupted Terraform applies resume only when the planned -current template has the exact reviewed image, capacity, and identity and the remaining change is -that selected MIG update or obsolete-template deletion. Production configuration pins only the -approved serving set to the compatible image and 1,000/60; the transition classifier accepts only -the reviewed mixed 600/1,000 envelope until every selected cell converges. Every GCP-only Terraform -command disables artifact DNS. Any failed mutation leaves only the selected cell migration-only and -never changes another cell's selector state. - -After multiple production cells pass the canary path, `wave-apply` may raise two to four reviewed -600/60 serving cells under one fresh 15-minute capacity-transition gate. The first cell is bound to -the sealed evidence; every later cell derives the exact expected selector generation and reruns the -complete live preflight before mutation. After the first cell, continuation preflights retry only -missing or stale signal evidence for at most one minute; health, threshold, selector, and migration -failures stop immediately. Cells still drain, restart, and verify sequentially. A -failed cell stays isolated and prevents every later wave job from starting; earlier completed cells -remain general at 1,000/60. The workflow lock, single-use evidence marker, exact predecessor check, -targeted Terraform plan, and per-cell heartbeat/admission oracle are unchanged. - -`Deploy Relay Production Same-Cap` rolls only the reviewed US 1,000/60 and Asia 3,000/60 serving -sets and the two migration-only US 600/60 cells, C17 and C18, without changing a cell's connection -shape. Use `canary-apply` for exactly one cell. A successful canary -seals its commit, target and rollback digests, selector generation, durable rehome generation, and -drain pace window; `batch-apply` accepts only that same authority and rolls two to ten cells sequentially. Both apply -modes and `rollback` first refuse a cell whose hosts (controls) exceed 80% of the free slots on the -other fresh general cells, since drained hosts with nowhere to go keep redialling and pin the cell. -`verify` runs the same read-only check, so it reports the headroom answer before an apply is -dispatched; a rollback that resumes after its restart drains nothing and skips it. A cell's free -slots are its normal admission pause minus the larger of observed connections and enforced units, -minus outstanding control reservations; each moved host also brings its splices, which the 20% -margin covers. Each cell is isolated, drained until restart-safe, replaced -from a targeted saved plan, and restored only after a new incarnation reports the exact digest, cap, -heartbeat, and rehome protocol. Restart-safe means the cell runtime itself carries nothing live (no -controls, in-flight connections, reserved connection units, splices, or queued bytes) and no -migration is open, on two consecutive samples the second of which is taken at or after the end -of the drain pace window. Every live restart-safe call supplies that window (`--pace-window-ms`). -Pre-auth and total connections are printed but do not restart the count: they include -unauthenticated redials that lose nothing on a restart. Director activity leases left by hosts that already went or cannot -be placed do not hold the restart; every `relay_capacity_transition_restart_progress` sample and -the final verified line report them under `stranded`. The durable -worker must remain disabled throughout. The post-restart trust check is application-mediated by the -director; the workflow never receives or mints a director or stamped-cell runtime token. A failure -keeps only the selected cell migration-only, while the exact rollback digest remains dispatchable via -the same workflow's `rollback` mode. - -A roll holds the whole startup script identical before and after except the image, so a -template stale enough to predate a pinned line fails closed rather than absorbing the drift. -The one exception is the capacity identity: a cell that predates it gains it on its next roll, -and the plan validator pins the exact reviewed identity instead of comparing that line, so a -roll can never drop or rewrite it. Any other stale line still fails closed and needs a -convergence apply first. - -A drained cell is refused before a roll, because draining means something is already -shedding its connections. A migration-only cell has none to shed, so the flag decides nothing -there and is accepted on entry; the replacement VM is still required not to be draining, and -the incarnation check still proves it was replaced. That also unwedges the state a failed -canary leaves behind, where the wave's own drain set the flag and no restart followed. - -C17 and C18 hold no hosts and are not general, so rolling one displaces nobody: they are the -zero-displacement canary for a new image. Their wave enters and leaves migration-only, so its -isolate and its restore are both no-ops and the selector generation does not move; a general -cell's wave still advances it by two. One wave may not mix the two classes, because every cell -after the first offsets from a single per-wave delta. Neither cell is a declared regional-rehome -source, so its template carries no rehome trust lines and it may roll only at rehome protocol `0`; -the job refuses a trusted protocol for it before it plans anything. - -### Recovering a wave that died after its drain - -A cell's drain flag is a one-way latch on the running process. Only a restart clears it, and -the failsafe that isolates a failed cell does not restart anything. So a wave that stopped -any time after its drain step leaves the cell migration-only and draining, and it stays that -way until the cell is rolled. - -Read the failed run before dispatching anything. If its log has a -`"event":"relay_production_capacity_canary","mode":"drain"` line for the cell, the cell is -drained. Then read the cell's live runtime image from -`POST https://.relay.onorca.dev/v1/admin/runtime-status`. - -1. **Do not re-dispatch `apply`.** It requires the cell general and not draining, and a - drained cell is neither. It will fail closed at the predecessor check. -2. **Dispatch `rollback`,** with the same `target-image-digest` and `rollback-image-digest` - the failed wave used, the live selector generation, and the live tri-state membership - with the failed cell listed under migration-only. The confirmation is - `ROLL_BACK_RELAY_SAME_CAP ` at the default drain pace (see - below for any other). -3. The job classifies the cell itself and needs no extra input: - - serving the **rollback** image and draining, it is `stranded`. The wave stopped before - or during its template apply. The job re-isolates, re-drains, applies the reviewed - template, and, if that template was already in place, recreates the cell's one instance - with `recreate-instances`. The cell comes back on a new instance, so the drain clears, and - it is restored to its entry class. The recovery does not use a rolling action: that - rewrites the MIG's version name outside Terraform. The plan validator does accept a MIG - moving back to the version name and update policy `relay-gce-cells.tf` declares, so a cell - an older rolling action left relabelled reconciles on its next apply, roll, or stranded - rollback. The recreate refuses a MIG that does not hold exactly one instance, such as a - fenced cell; that failure is the guard, not a fault, so unfence before dispatching. - - serving the **target** image, it is `roll`, the ordinary rollback. The template applied - and the instance was replaced. - - serving the **rollback** image and not draining, it is `resume`: a rollback that failed - after its own template apply. Nothing is applied and nothing restarts. -4. Rollback takes exactly one cell per dispatch. Recover the cells one at a time. -5. If the run died inside `wait-until stable`, the MIG is still rolling on its own. Wait for - it to settle and re-read the runtime before dispatching, or the stage will be read off a - state that is about to change. -6. A `stranded` dispatch that fails at plan review means the template already carries the - target image while the old instance is still up. Wait for the MIG to finish replacing it, - then dispatch again; it will classify as `roll`. - -### Drain pace ladder - -The `drain-pace-window-ms` input sets the window the cell spreads its drain sends over. Each -drained host re-dials the director as soon as it reads `drain`, so the window sets the re-placement -arrival rate: hosts / window, about 1.8-2.7 hosts/s for a US cell at the default. The window also -sets two waits: restart-safe needs an empty runtime on two consecutive 5-second samples, the -second at or after the window's end, and the drain step's overall timeout is the 15-minute -migration lease plus the window (20 minutes at the default, 35 at `1200000`). The runtime empties -as the window closes, so a drain step takes about the window plus a few seconds to half a minute. - -- Allowed values are `300000` (the default, and every wave before this input), `60000`, - `30000`, `900000`, and `1200000`. The wave validator refuses anything else, and each cell job - checks again. -- Below `300000` is for US general cells only (C7-C10, C13-C16, C19-C26, C32, C33). Asia drains - are bound by the target cells' own accept rate (about 4-6 hosts/s per Asia cell), not by the - window. Migration-only cells hold no hosts. Both stay at `300000` or slower. -- `900000` and `1200000` are for any cell. They exist because c28's 1,901-host drain at `300000` - (about 6.3 hosts/s, 2026-10-07) held database lock time above its bar for about 10 minutes. - At `1200000` a 2,000-host Asia cell re-places about 1.7 hosts/s. -- A non-default window must be named at the end of the confirmation, for example - `ROLL_RELAY_SAME_CAP drain-pace-window-ms=60000`. A confirmation that - names no window confirms `300000`, so a form left at another value fails closed. -- A canary's authority records its window and its pace verdict (below). A batch may use that - window or a slower one, never a faster one. A batch below `300000` also needs the canary's pace - verdict to be PASS. Stepping back to `300000` or slower mid-ladder needs no new canary. -- A cell on an image without paced drains rejects the window, and the job falls back to an - unpaced drain. The job records what the cell accepted, and a canary that did not drain at its own - window seals `UNVERIFIED`. -- A window above `300000` needs the cell being drained to already run an image that accepts it - (the cap rose with this ladder's slow rungs). Before the isolate, the job reads the cap the cell - advertises on `/health` (`drainPaceWindowMaxMs`; an image without it has `300000`) and stops - with the cell untouched if the window is above it. Re-dispatch at `300000`, or after the cell's - next roll. If a cell still rejects the window at the drain, the job fails rather than drain - unpaced, and the cell stays isolated for an operator, as any failed cell does. - -**What judges a paced drain.** The shadow health gate (report only, after each cell) judges two -checks the pace can move. Together they are the report's `paceVerdict`, which the canary seals: - -- **Director 503s** come from Cloud Run's own request counter (`run.googleapis.com/request_count`, - code 503), aligned per minute by Cloud Monitoring. A log read of them stops at its entry limit - in exactly the minutes that matter: it read only 20k of 10-01 c29's ~31k. -- **Scheduled 503s are taken out**, using the director's `orca_relay_runtime_metrics` counters: - - drain-return deferrals; - - sticky and placement answers to a host's own early retry (`host-rate-limited`, - `host-in-flight`). - - Each tells one host when to come back. What is left is lanes, capacity, or the database - refusing work. On 10-02 the early-retry answers were about three quarters of all 503s, and - they doubled with placement volume whether or not a drain was running. -- **The background** is the median minute of the 10 same-day minutes before the drain. A busy - morning raises it with the window, and one incident minute inside it does not. - -| check | rule | -|---|---| -| `nonDrain503Budget` | Two consecutive minutes above max(1.5x background, background + 20) warn; above max(2x, background + 40) would-block. One minute above max(10x, 200) would-block on its own; any other single minute is a transient | -| `drainDeferrals` | Warn if the largest Retry-After exceeds 30 s; would-block above 60 s. Reports deferrals and re-placements | - -A read that fails, hits its limit, or returns fewer director-metric samples than one instance -emits (one per 30 s) makes both checks `unverified`. An empty answer is not a calm director. - -Replayed read-only against past rolls: -- Every brownout and herd replayed is `would-block`: 10-01 c29, 09-23 c27, 09-24 c30, both 10-01 - c28 windows, and the 09-28 and 09-30 herds. Each peaked at 5,999 non-drain 503s a minute or - more. 10-01 c29, for example, ran 9 minutes in a row over a 41.5/min line. -- Clean rolls: all nine 10-02 cells, 10-01 c25, and 11 other US and Asia rolls have a pace - verdict of PASS. Their largest minute was 87, and none held two minutes over the warn line. -- Two daytime Asia c29 rolls on 10-01 read WARN and WOULD_BLOCK on sustained non-drain 503s at the - default pace. Their largest minute was 112, still under the 200 single-minute line. -- These 30 verdicts are unchanged from before the per-minute peak check was folded into this - one. - -The other checks (`cellServing`, `cellPool`, `cloudSqlFatal`, `fleetPool:*`) stay in the overall -verdict as context. They read the new boot and fleet-wide pools, so a clean roll at any pace can -still WARN on them, and every 10-02 roll did on `cloudSqlFatal`. - -The report records: -- `background`: the pre-drain minutes; -- `nonDrain503Budget.perMinute`: the per-minute series; -- `drain`: the window, the window the cell applied, the host count, and the seconds from - isolation to restart-safe. - -Two other 503 rules exist, and neither needs this split. The pre-drain sample's 500/min rule reads -the 10 minutes *before* a drain, and a previous cell's drain has ended by then. The incident -monitor already excludes 503s from its director 5xx rule. - -**Procedure.** One rung at a time, on routine US same-cap rolls: - -1. Before the first rung, run a fresh canary. This input is evidence code, so merging it - invalidates any sealed monitor or canary authority. Staging has no paced-drain path today: its - capacity proof drains unpaced. A rung's first use is therefore one production canary. -2. Roll a `canary-apply` at the next rung. Use `60000` after a clean `300000` roll. Use `30000` - only after a clean `60000` roll, and only once the director reports the drain-return lane's - service time (`drainReturnServiceMs`, #25645). At 30 s the lane holds only if that time stays - under about 190 ms. -3. A rung is clean when: - - the canary job succeeded; - - its `paceVerdict` is PASS, sealed only from a report on that cell that drained at least 400 - hosts at that pace; - - time to empty is within window + hosts / 50 s + 10 s. Take `settledAfterSeconds` minus the - one 5-second confirming sample; this is an upper bound, since it includes the isolate. - - The batch check enforces the PASS. A canary that sealed anything else authorizes only - `300000` batches. The 400-host floor exists because a pace is an arrival rate (hosts / - window). A canary that small would test less than half the rate a 692-782-host US cell - (10-02) reaches at the same window. Pick a canary cell above it. -4. Record each rung, and keep the shadow gate JSON artifact with the row: - - | Field | Source | - |---|---| - | Cell, hosts, window | The report's `drain` block | - | Measured drain rate | `drainDeferrals.replacementsPeakPerMinute`, and hosts / time to empty | - | Non-drain 503s | `nonDrain503Budget.perMinute` against `background.medianPerMinute` | - | Lane deferrals | `drainDeferrals.deferralsTotal` and `retryAfterSecondsMax` | - | Roll time | The cell job's duration | - - This job does not report four of the rung's bars: drain-to-reconnected p95 (20 s), the - drain-return lane's service time, over-cap cells (0), and selector compare-and-swap retries - (at most 2). Read them by hand where a source exists, or record them as unmeasured until the - director observability work lands. -5. If any bar fails, set the input back to `300000` for the rest of the wave and stop the ladder. - The `300000` batch needs no new canary. - -### Pre-drain fleet-health sample - -A same-cap dispatch does not need a separate monitor run. Each `apply` wave samples fleet health -itself, inside its own job, as the last step before it isolates its cell: - -1. The live preflight takes one sample against the monitor's thresholds, with the expected - selector taken from the dispatch inputs (offset for the wave) and the migration policy pinned - to `strict`. The membership is canonicalised the way the monitor canonicalises its own, so it - must name every configured cell exactly once and its order does not matter. -2. The headroom check reads how many hosts the cell carries. -3. The pre-drain sample (`pnpm incident:relay-pre-drain-sample`) then samples once a minute for a - window sized to that count: up to 500 hosts, 3 minutes; up to 1,500, 5 minutes; above that, - 8 minutes. Every sample is judged by the monitor's own evaluator and thresholds, with the same - tolerance for a flaky cell probe, a director instance replacement, or an unread signal (two - consecutive samples). Every sample also applies three lookback rules no single reading can - see: no container exit (`orca_relay_cell_process_exit`) in the last 10 minutes on a cell that - takes placements (general or migration-only in the dispatch membership) other than the cell - being rolled, no minute in the last 10 with more than 500 director 503s (a disconnect - pulse), and director concurrency p99 at most the monitor's 64 over the last 4 minutes. The window does not end on a - sample that still carries a tolerated failure, and trips if three samples past the window - still have not come back clean. - -The exit metric names only an instance, so each exiting instance is matched to a cell by that -instance's own newest `orca_relay_runtime_metrics` line from the last two hours. The target's own -exits are ignored, because the roll exists to fix them, and so are existing-only legacy cells, -which take no placements. An exit whose instance cannot be matched to a configured cell trips the -rule; a failed lookup counts as a failed read. A newly booted instance can exit several times in -its first seconds while its Cloud SQL proxy sidecar starts (c25's replacement did on -2026-09-28); if that lands within 10 minutes of the next wave's sample, that wave trips and the -remaining cells need a new dispatch. - -Any trip fails the wave before isolation, so nothing has changed; dispatch again once the fleet is -quiet. Every later cell in a batch runs all three again, so a batch never drains on health read -before the previous cell rolled. `rollback` runs the live preflight but not the window, because -getting off a crash-looping image must not wait for the crashes to stop; `verify` runs neither. - -The monitor workflow itself is unchanged and still gates the rehome enable path and incident -watches. What a same-cap wave no longer has is the 15-minute history before the dispatch; the -sized window plus the 10-minute lookbacks replace it, and the dispatch no longer has to land -within minutes of a monitor run finishing. - -The first compatible director rollout uses `bootstrap-runtime-identity=true` with -`BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY`. That one-time path requires the exact stamped-cell -predecessor identity, creates both the cold rollback and candidate on the distinct director identity, -and proves the disabled durable control through those compatible revisions before moving traffic. -Later director deploys reject the predecessor identity and verify the disabled control on the serving, -rollback, and candidate revisions. - -`Operate Relay Production Rehome` is the only durable worker control. `inspect` is read-only; -`enable` is selector-, director-digest-, rollback-digest-, and control-generation-bound, starts at -exactly 10 hosts per minute, consumes the fresh 15-minute safety monitor, and seals 24 hourly buckets -of aggregate requested-region, selected-region, fallback, and unavailable-region evidence with -positive Asia requests and selections. `pause` and `disable` apply their generation CAS immediately -after checkout and authentication, before package installation, revision checks, or log diagnostics. -Their typed confirmations are `PAUSE_REGIONAL_REHOMING` and `DISABLE_REGIONAL_REHOMING`. Keep the -default 3,600,000 ms drain grace so existing splices can finish. The job summary contains only fresh -aggregate active, receipt, registration, completion, and abort counts. - -### Director deploy driver - -`dev/scripts/drive-relay-director-deploy.mjs` runs a whole director deploy from an operator machine -with `gh` and `gcloud` logged in. It only dispatches the workflows above and reads their results; it -holds no credentials and changes no workflow. It never fills in a workflow's typed confirmation: the -operator types each one when the driver reaches it. - -```bash -cd cloud -node dev/scripts/drive-relay-director-deploy.mjs --commit --dry-run -node dev/scripts/drive-relay-director-deploy.mjs --commit \ - [--configure production-gce-c34=sha256:] -``` - -It keeps no state between runs. Every decision comes from live state read at the start of each run: - -- the serving director's digest and configured cells, from `gcloud`; -- the admission selector, from an `Operate Relay Asia Admission` `inspect`; -- the rehome control, from a rehome `inspect` at the generation the newest rehome run printed, or - at `--rehome-generation`. - -Steps already done are skipped: a serving digest that matches is not deployed again, and cells -already configured are not configured again. It always reads rehome, even when nothing is left to -do, so it never reports success over a pause it cannot explain. - -The sequence: - -1. **Publish.** No `cloud-*` workflow is queued or running (all pages; the hourly clock-skew - monitor and `cloud-verify` excepted), and `main` is the reviewed commit. The publish workflow - builds whatever `main` is when it is dispatched, so the driver dispatches it straight after that - check, before the inspects and the typed phrase. It changes nothing serving, so a bad build needs - no cleanup. The digest is the registry digest of `relay:sha-`, and the run's own push - line must name the same digest. If `main` still moved in those seconds, the driver stops and - names the `--commit --publish-run ` that deploys that build once it is reviewed. -2. **Preflight, read-only**, then the operator types `DEPLOY `. -3. **Pause**, only if rehome is enabled, after the operator types `PAUSE_REGIONAL_REHOMING`. -4. **Deploy** with that digest, the paused generation, `preserve` for both regional inputs, no - prune, and the old serving digest as predecessor. -5. **Soak**, with `--configure` only, while no wave is configured yet. It watches 5 minutes of - director 5xx and stops if they exceed twice the 5 minutes before the new revision existed, plus - 25. The window starts at the traffic switch (a minute before the deploy run completed) when this - run deployed, otherwise at the time of the run, so a re-run judges fresh traffic. It is read a - minute late, to allow for log lag. Then the operator types `CONFIGURE_ASIA_DIRECTOR` and each - pending wave is configured. -6. **Digest check.** A rehome `inspect` bound to the serving and rollback digests that `gcloud` - reports now. A wrong digest fails here, read-only, before 15 minutes of monitor evidence is - spent on it. -7. **Monitor.** The operator types `ENABLE_REGIONAL_REHOMING`. The prompt says this arms an - automatic enable, sent about 17 minutes later, and only if the monitor is green and its evidence - is at most 150 s old. The monitor dry-run then starts. Its artifact passes the same - `relay-monitor-evidence.mjs verify-authority` check the enable job runs. -8. **Enable** with the verified digests, within 150 s of the monitor completing. - -Steps 6 to 8 run only for a pause this driver owns. - -**Ownership.** The driver owns a pause only if it can name the run that made it, and the live -control is still at that run's generation. Two kinds of line in a run's log prove it paused -rehome: - -- `pause`; -- `recover-enable` with `recovered: true`, meaning a failed enable that disabled rehome again itself. - -The run must be a rehome-control run by the same GitHub user. A `recover-enable` with -`recovered: false` found rehome already disabled, for example by a director safety pause, and is -never adopted. A failed enable run is never counted as an enable, whatever it printed last. A fresh run that finds rehome paused stops. It goes ahead only -with: - -- `--pause-run `, which an earlier run of this driver printed; or -- `--leave-rehome-paused`, which deploys and leaves rehome paused. It refuses an enabled switch. - -A pause made by anything else is never lifted. - -**Stops.** On any failure, Ctrl-C, SIGTERM or SIGHUP, the driver prints what changed: - -- `REHOME IS CHANGING` when a pause or enable run is in flight and will apply on its own; -- `PAUSE UNCONFIRMED` or `ENABLE UNCONFIRMED` when such a run printed no usable result; -- `REHOME IS PAUSED by this driver` with the owning run; -- the serving director, re-read; -- the published digest; -- the rollback point, with the `gh workflow run` command that redeploys it. - -It ends with the single command that finishes from where it stopped. That command carries -`--publish-run` and `--pause-run`, and the driver re-verifies both against the runs' logs and live -state. Each run writes a timestamped log under `~/.orca/relay-director-deploy/` -(`--log-directory` overrides it). - -## Reserve watch - -`Operate Relay Production Reserve Watch` watches one cell already switched to reserve mode and, on -the first step 5 stop rule it trips, flips it back to db through the flag tool's own write -(`operate-relay-cell-flags.mjs`: CAS, read-back, re-registration wait, Postgres record). It is -manual only and inert until dispatched; `dry-run` logs the flag tool's plan instead of writing. -`write` takes the flag workflow's confirmation, `WRITE_RELAY_CELL_FLAGS `. - -It samples once a minute for up to 300 minutes, re-authenticating every 55-minute segment because -an ID token lasts an hour, and logs one JSON line per sample. The 503 background must predate the -flip: give `flipped-at` (when the reserve write applied; the background is the median of the 10 -minutes before it) or `background-503-per-minute`, which overrides it. With neither, the watch is -refused. A watch dispatched before the reserve write applied waits up to 10 minutes for the cell to -report reserve, judging nothing meanwhile; a cell that never does ends the watch refused, without a -write. It also refuses to start while any run holds the Cloud SQL rollout lease (same-cap and -capacity rolls hold it for their whole run, and it names no cell), or when that lease cannot be -read. It only reads the lease: holding it for hours would block every rollout. Pausing on a roll -that starts mid-watch was not built, since a pause blinds the stop rules for the roll's length and -a flip back mid-roll races its drain, so do not start a roll on a watched cell. It trips on, in -this order: - -| Rule | Signal | Exact or proxy | -|---|---|---| -| Dead-man tripped | runtime-status `admitModeEffective=db` under a reserve switch file | Exact | -| Overrun | runtime-status `enforcedConnectionUnits` over `connectionCapacity.hardCap` | Invariant check: the ledger already refuses past its cap, so this trips only if that refusal broke. It counts seats, bookings, in-flight and phone units | -| 503s | Director non-drain 503s (the roll ladder's measure: `request_count` 503s less the drain-return deferrals and the `host-rate-limited` and `host-in-flight` re-dials the director's `orca_relay_runtime_metrics` count) above max(1.5x background, background + 20) for 3 complete minutes, with at least 2 director instances each above an even share of that line (divided by the fewest instances that logged runtime metrics in any of those minutes, at least 2) in the same 3 minutes; or above max(10x background, 200) for 3 minutes with no instance split | Proxy that can miss: fleet-wide, since no signal attributes a 503 to a cell, so a problem on one cell can stay under the fleet's line. The instance split keeps one director instance whose database calls stall alone from flipping every watched cell back (10-21 07:55Z: 45, 164, 40 a minute, 54, 155, 5 of them on one instance) | -| Duplicate past grace | The ledger reconcile's `orca_relay_reserve_ledger_seat_behind_row` for the cell, reported again more than 90 s (60 s demotion close + 30 s map lag) after it was first seen, keyed by director instance, cell, epoch and row epoch | Proxy that can only over-count: the line names no host, so two hosts sharing that key read as one seat | -| Host drop | runtime-status `controls` fell more than 5% (and at least 10 hosts) within 5 minutes with no `draining` sample | Exact | -| Ledger drift | Queued plus dropped rows in each director's newest `orca_relay_reserve_ledger_summary`, over the cell's controls, above 0.5% for 2 minutes | Proxy that can only over-count: the summaries are fleet-wide, so exact only while this is the only reserve cell | - -Any rule it cannot read in a sample makes that sample unverifiable; `unverifiable-limit` such -samples in a row is a trip of its own. The 503 read takes every response code in one query, and the -newest minute any code has a point for is how far the series is published: a 503 minute at or -before it with no point had no 503s, so a quiet fleet reads zero. Minutes past it are left out, not -counted; fewer than 3 published minutes in the read, no request series at all, or an unparseable -timestamp is unverifiable. This assumes the codes publish together; one code lagging the others -delays a trip by its lag but cannot hide one, because a later read sees its points. A host's own -early or duplicate re-dial says nothing about a cell and grows with traffic: on 2026-10-18 it was -~98% of 503s and tripped c7 back twice with no fault. The subtraction is only trusted where the -director logged: a judged minute no runtime-metrics line covers is unverifiable, a 10-minute read -with fewer lines than a single instance logs (20) or at its entry limit is unverifiable, and the -measured background leaves such minutes out (and is not measured with fewer than half of its 10 -metered). Coverage is any instance, not every one: an instance whose lines are missing while others -log under-subtracts its re-dials, which can only push toward a trip, never hide one. The instance -split counts each instance's own `/v1/assign` 503s from its runtime-metrics lines -(`assign503sByCauseDelta`, by `labels.instanceId`; the metric names no instance), less that -instance's own scheduled 503s. The fleet's non-drain 503s no instance's lines account for (Cloud -Run's own, a route the director does not count, or an instance that stopped logging) are -`unattributed`: summed over the 3 judged minutes, since an instance's 30 s lines skew against the -fleet's minutes, and over the whole fleet line by themselves they trip (no one instance's stall -explains them; the largest such sum in the 24 hours to 10-21 was 38, against 90). A stalled instance that also stops logging its lines therefore trips, on purpose: the 10-21 07:55Z instance kept logging through its stall. On the recorded 10-21 07:55Z stall this reads 40, 169, 5 -on the stalled instance, where Cloud Run's request log read 54, 155, 5. With a single director the -share is half the line, so that one instance alone never trips between the lines. A judged minute -with no instance count is unverifiable. A single instance id must hold its share for all 3 -minutes, so a fault that keeps replacing instances can pass under the broad line. - -Each sample makes one Cloud Logging read: the director's runtime metrics, its ledger summaries, -the cell's seat-behind-row lines and, while the phone path is watched, the cell's own runtime -metrics, as one OR filter split back client-side. The project allows 60 reads a minute shared by -every watch and tool; on 2026-10-21 18:39Z twelve watches at about five reads a sample drew 429s -until c21 and c22 tripped unverifiable. A read is ~200 lines (1,000 is one page and the limit); one -truncated or failed after its retries makes every log-read rule unverifiable for that sample, -which counts once toward the streak as any unreadable rule did. The 503 background and the phone -background are each read once, on their own. The same measure is the -background and `background-503-per-minute`. Until the -minutes before `flipped-at` are published (3 minutes past it) the 503 rule is pending, which does -not add to the streak. The duplicate read also takes `orca_relay_reserve_ledger_summary`, which -every placing director logs each minute, in the same query and window: with no summary from the -last 150 s in it, the read is blind and the sample unverifiable, not zero duplicates. The duplicate -rule keys a seat by director instance, so a director restart (a new instance id) starts its -first-seen clock over. A cell whose switch file no longer says reserve ends the watch without a -write. - -In `write` mode a watch that fails, times out or is cancelled before a verdict flips the cell back -as a trip would; the flip-back job and each of its steps run on `always()`, so a cancel during the -lease wait does not skip the switch-file write. That cancel can still kill the process before -Postgres records db, leaving the file at db and Postgres at reserve; a force-cancel can stop it -outright. The workflow's only group is -its own per-cell one, which a running run never yields. It stays out of the flag workflow's group -because GitHub keeps one pending run per group and a newer one cancels it, so a flip back queued -there could be dropped by any later flag dispatch, and a watch held there would block manual writes -for hours. The flag tool's generation check (CAS) orders it against other writes instead; on a -conflict it re-reads and retries, up to 5 times, while the switch file still says reserve, and stops -if it already says db. A cell that still does not answer runtime-status after 4 tries with backoff -cannot take the ordinary flip back: the run fails and prints the exact break-glass inputs for -`Operate Relay Production Cell Flags` (`set=admitMode=default`, `break-glass=BREAK GLASS `). - -An operator watches the run. A refused watch and a flip back that did not land (no ID token, a -failed login, a cell that does not answer) can each leave the cell in reserve with nothing watching -it, so both raise an `::error::` annotation naming the cell, as the clock-skew monitor does. In -`write` mode both also log `orca_relay_reserve_watch_alarm` (log `orca-relay-reserve-watch`, -resource `global`) with the deploy identity, and the `Orca Relay: reserve cell may be unwatched` -alert pages on any such line. A failed login cannot page: the runner then has no credentials for -`gcloud logging write`, so only the `::error::` and the operator watching the run report it. Apply -the root Terraform that adds the workflow to the shared deploy identity's provider condition and -grants that identity `roles/logging.logWriter` before the first `write` run; until then the -workflow cannot log in and no alarm line can be written. - -## Mobile push gateway - -`Deploy Push Gateway Production` (`.github/workflows/cloud-push-deploy.yml`) is the deploy path -for `orca-cloud-push`, the mobile push gateway. It is the one `cloud-*` workflow that is not a -relay operation, and it is here because it shares the Artifact Registry repository and rollout -lease. Push uses a dedicated Cloud SQL instance. - -It authenticates through `PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER` and -`PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT`. The dedicated identity has Artifact Registry writer, -Cloud Run developer on the push service, and impersonation of only the push runtime account. -Its provider pins the repository, production environment, main branch, and exact dispatch workflow; -its distinct principal attribute cannot assume the shared Relay deploy account. - -Foundation grants the dedicated account access to the rollout-lock prefix and bucket metadata. -Apply that companion grant and publish the identity outputs before running the workflow. See -[push gateway deployment setup](./push-gateway.md#deploying) for the activation steps. - -The run builds the exact reviewed image digest before taking the lease and rejects images -without validation-mode support using a network-isolated container. Under the lease it boots -an inert, read-only validation revision, checks readiness, mode, scaling and FCM credentials, -then deletes it before deliberately activating the same digest in a new revision. Activation -starts schema writes, pruners and queue consumers before HTTP promotion. Rollback requires -restoring traffic, deleting the rejected active revision, and restoring the service template to -the known-good image in normal mode. The untagged template-recovery revision can run known-good -workers and is retired before lease release; see the -[deployment and rollback contract](./push-gateway.md#deploying). diff --git a/cloud/docs/staging-relay-deploy-identity-rollout.md b/cloud/docs/staging-relay-deploy-identity-rollout.md deleted file mode 100644 index 81fd3efe44a..00000000000 --- a/cloud/docs/staging-relay-deploy-identity-rollout.md +++ /dev/null @@ -1,177 +0,0 @@ -# Staging Relay deploy identity rollout - -Moves the five staging Relay workflows off the apps-owned `github_deploy` -account and onto the relay-owned `github_staging_relay_deploy` account -(`orca-cloud-staging-gha-relay`). This is a **rollout**, not state surgery, so -it lives beside [`terraform-root-split-runbook.md`](./terraform-root-split-runbook.md) -rather than inside it: that runbook is state-only and runs no apply, and every -step below applies. - -Production is untouched. `local.relay_github_deploy_service_account_email` -still renders `orca-cloud-gha-deploy` there, and the production relay plan slice -is byte-identical to the pre-split single-root baseline. - -## What moves, and what it costs - -The staging Relay runtime allowlists exactly one deploy account -(`ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT`, `apps/relay/src/admin-token-verifier.ts`), -so the account, the director env, the four cell startup scripts, and the -workflow variables have to change together. Plan on a staging window in which -no Relay workflow runs. - -| Change | Cost | -| --- | --- | -| 10 new identity resources | Create only. No compute. | -| 6 relay bindings repoint to the new account | Delete + create. The old account loses them. | -| `ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT` on the director | New Cloud Run revision. | -| Cells c1–c4 startup metadata | Four instance-template replacements, one MIG repoint each. | - -## Preconditions - -- [ ] PR 13 is merged, so both accounts are declared and the census test passes. -- [ ] No staging Relay workflow is running or queued. All five share the - `relay-staging-mutation` concurrency group; `prove-relay-staging-capacity` - and `recover-relay-staging-c4-image` are in it too. -- [ ] Staging is awake, or you accept waking it as part of step (e). - -## (a) Compute-free identity apply - -Targeted apply on the staging relay root. Verified against a post-surgery state -copy: **10 creates, nothing else**. - -```sh -terraform -chdir=infra/terraform apply \ - -var-file=environments/staging.tfvars \ - -target='google_service_account.github_staging_relay_deploy[0]' \ - -target='google_iam_workload_identity_pool_provider.github_staging_relay_deploy[0]' \ - -target='google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user[0]' \ - -target='google_storage_bucket_iam_member.github_staging_relay_deploy_state_list[0]' \ - -target='google_storage_bucket_iam_member.github_staging_relay_deploy_state[0]' \ - -target='google_artifact_registry_repository_iam_member.github_staging_relay_deploy_artifact_reader[0]' \ - -target='google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer[0]' \ - -target='google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_auth_developer[0]' \ - -target='google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user[0]' \ - -target='google_project_iam_member.github_staging_relay_deploy_compute_viewer[0]' -``` - -Reject the plan if it shows anything but those ten creates. - -## (b) Publish the two variables - -```sh -terraform -chdir=infra/terraform output -raw github_staging_relay_deploy_workload_identity_provider -terraform -chdir=infra/terraform output -raw github_staging_relay_deploy_service_account - -gh variable set STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER --env staging --body '' -gh variable set STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT --env staging --body '' -``` - -Nothing reads them until (c) merges, so this step is reversible on its own. - -## (c) Repoint the workflows and flip the relay bindings - -The workflow repoint ships in PR 13. Merging it and applying the six repointed -bindings is one step, because the new account cannot operate without them and -the old account must not keep them: - -```sh -terraform -chdir=infra/terraform apply \ - -var-file=environments/staging.tfvars \ - -target='google_project_iam_member.github_staging_relay_power[0]' \ - -target='google_service_account_iam_member.github_relay_runtime_service_account_user[0]' \ - -target='google_service_account_iam_member.github_relay_director_runtime_service_account_user[0]' \ - -target='google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor[0]' \ - -target='google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder[0]' \ - -target='google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer[0]' -``` - -Expect **six replacements plus one create**: the target on -`github_relay_director_runtime_service_account_user` drags -`google_service_account.relay_director_runtime`, which staging has never -created. That create is additive and does not move the director onto the new -identity; only applying `google_cloud_run_v2_service.relay` does that. Drop -that one `-target` if you would rather leave it to the staging drift -remediation, and accept that the new account then has no `serviceAccountUser` -on the director runtime account. - -The director's `ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT` changes only through -`google_cloud_run_v2_service.relay`, and applying that address in staging also -carries the whole staging director backlog: the identity swap to -`orca-cloud-staging-relay-dir`, `timeout` 3600s to 30s, concurrency 1000 to 80, -the four-cell topology, and roughly twenty new env entries. Do it as part of -the staging identity remediation in the drift plan, in this same window, with -that plan reviewed on its own terms. - -## (d) Roll the cells, one at a time - -Use **Prove Relay Staging Capacity**. It authenticates only as -`STAGING_GCP_RELAY_CAPACITY_*`, that is `github_staging_relay_capacity`, never -the new deploy account, and the capacity identity's admin routes -(`RELAY_CAPACITY_ADMIN_ROUTES`) cover every call the roll makes. It is -therefore unaffected by this cutover in either direction. - -Per cell the targeted apply is: - -```text --target='google_compute_instance_template.relay_gce_cell["staging-gce-c"]' --target='google_compute_instance_group_manager.relay_gce_cell["staging-gce-c"]' -``` - -That plan is one template replacement plus one MIG update, and it also drags an -in-place update of `google_compute_health_check.relay_gce_liveness[0]` from the -standing staging log_config drift. Confirm it is in place, not a replacement. - -Two gaps to plan around: - -- `prove-relay-staging-capacity` has arms for **c3 and c4** only. -- `bootstrap-relay-staging-capacity` rolls **c2 and c3**, but it mints its admin - token as the deploy account. Running it across the email change fails: it - verifies a rolled cell with a token that cell no longer allowlists. -- **c1 has no workflow roller.** Roll c1, and c2 if you do not use bootstrap, - by the same two-target apply under human review inside the window. - -Wait for each MIG to report stable before starting the next cell. - -## (e) Verify - -Dispatch **Power Relay Staging** in `status` mode. It exercises the new -credential end to end: Workload Identity exchange on the new provider, the -state read, `gcloud sql instances describe` and the MIG reads through -`orcaRelayStagingPower`, `run services describe` on both the director and the -shared staging auth service, and an admin `cell-status` call that only succeeds -if the director allowlists the new account. Then run a `sleep` and a `wake` to -exercise the mutation paths. - -## (f) Retire the staging Relay grants on the shared account - -Follow-up PR against `infra/terraform-apps`. Once (e) passes, the staging -`github_deploy` account no longer needs the Relay-only reach it has today. -Narrow, in the apps root: - -- `google_project_iam_member.github_compute_viewer` — kept only for the Relay - candidate preflight; no staging app workflow reads GCE topology. -- `google_project_iam_member.github_cloud_run_developer` — project-wide today; - the Relay services are now covered by the two service-scoped grants above, so - the apps root can scope it to the API and auth services. -- `google_project_iam_member.github_artifact_writer` — still needed by the app - deploys; verify before touching. - -Leave alone: the AR mirror writer -(`google_artifact_registry_repository_iam_member.github_production_relay_staging_mirror_writer`) -names the **production** deploy account by literal and is unrelated to this -change; `github_runtime_service_account_user` and -`github_auth_runtime_service_account_user` are still used by the staging app -deploys. - -## Rollback - -Before (c): revert the two variables, or unset them. The job gates skip while -they are empty, and the old account still holds every grant. - -After (c) but before the cells are rolled: re-apply the six bindings from the -previous commit, which points them back at `orca-cloud-staging-gha-deploy`, and -revert the workflow repoint. The new account and its provider can stay; they -grant nothing the old path needs. - -After the cells are rolled: roll forward. Rolling four templates back costs the -same as rolling them forward and leaves the same window. diff --git a/cloud/infra/terraform/.terraform.lock.hcl b/cloud/infra/terraform/.terraform.lock.hcl deleted file mode 100644 index 2ca6063889e..00000000000 --- a/cloud/infra/terraform/.terraform.lock.hcl +++ /dev/null @@ -1,76 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/external" { - version = "2.4.2" - constraints = "~> 2.3" - hashes = [ - "h1:4UInMFuK4GNw4uf2vkUwwDtc0CajvJ88BkAE6xLKOa4=", - "h1:ERhVaFFS4/WRonirXUJV1DWN+cSTyEKEfs2ZnoP1BgY=", - "h1:O6uC9yKr7swQtc/kHVyaV9yETT20A39oUi5X+k/b290=", - "h1:QP724n6VWM/iitpILCwO079UOlzx6I0Ylh7g8Gwv1Y0=", - "h1:rwlUbh50HZYdRQWKW12nG4+3Giu2rp+mQXjqF0NzmVg=", - "zh:0b51793be4f66934a3666339e44c01fd56e1c6a56256dfc66d1cb391584b4c2f", - "zh:31cdd9b30e4ec63d130befc89471757ef3937b99a8f6cc006769d215365c5ba8", - "zh:61f86de4a3166cfa5da6800eeba8e6a2e4ab6403fc3d7b396508260b45d3de7d", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:817e8d5946aed6ca692e0bb2f6463c28774ef8fb2fdd3922543a495a249229ea", - "zh:b35f1bd1be09ed1a1620b43ab8cb43fe93407cf419586d53fe965691cc1b4a8e", - "zh:bcb170063ec8b5728bc2a4568bc48f539a1991cdaaf31667aa35568aebc34725", - "zh:c0d2c824cc7c047f26ce793bb0cbb6746f9745d1fc6e630d34a0afb5b92850d1", - "zh:cde68f51089b02db50e2c5a17f6e132dc1ead2fc08d3dd267b8dd54ec58133fa", - "zh:d1f3c497aa41f17e8d61067122f6d94e51ef942ab08be5465489864d900854ab", - "zh:e62568bfc0934b63e14f3547c823b01ed60d7475ff16bf12c0e55d5ac8d98ba7", - "zh:ed8890c29dba2b0ac27afcefa75e7395e27253b7025aaaa4258345fc59dad23e", - "zh:f220c56c7e487f01fd158126066f6525178bbd82805b27205354bc523cc7c413", - ] -} - -provider "registry.terraform.io/hashicorp/google" { - version = "8.8.0" - constraints = "~> 8.8" - hashes = [ - "h1:PyTXDNqrLJDsxiCm16XQD3oxoZjbSeD3ENQUZmO0rQg=", - "h1:hDDa+YGmy+34+mtPvpVQeNAHNU2mK3oBvegOpq03JLQ=", - "h1:hjnTDx39rMQnZQ9l9cpGYIllP9kIYH6/eFCLCdDTF4w=", - "h1:iYEtU+gkrIIyjMnkKEGAi0pn/XObXCZh+R0ybKdixXY=", - "h1:yzldJAMvM2XwkoGtE9mEclSBpwzxKr2q94ltPqr7h18=", - "zh:14040d76f45124f5f6c0ef5da49f8a1be0154569824f952d4743ded5acf3b5bc", - "zh:1ad6d26da010b1a2372489b9674dfbde3c35f4309e5d7653ff329d2dce9a2c3f", - "zh:277ab10bedd60a4021e0126b577b0e970761051d7ae9296249d8cbb30ca9045d", - "zh:2ae1cd5b4f42ceca33d9b78ee2e1476377b121d730477ac6cf2431315ba6d37f", - "zh:3a0ec05144511ddac12a8ce5df885468257bb6b9665b854934efcb9af8851064", - "zh:5798f24b9765f238a8139ab9b1e31b0269703804fd3273974cf2420a8be155bb", - "zh:598358108411fda3454c53d4c1d92a3fc2ff49292b7b57f48c5522a84c472247", - "zh:6c4f10729f32ac9bf0020da7d5c1ec4bfb6e26da646dcb4cbb058d05c7ab3e94", - "zh:78e79f9018ad089a0c7e3e38effa4e6ce692fdfe509044039377ea7bfb946505", - "zh:92f41ef74c86cf62162f79cebbae017fcaf44faf78528edce426be397d87000f", - "zh:d13ad2079ff34253edf6ec2ec3535993574ce0f2c8d1c4f93ca947e4166990bf", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.9.1" - constraints = "~> 3.6" - hashes = [ - "h1:7uiStw0Rl9KOdX5UNMG/sp9nyadoD4LZekQTiYlYPhE=", - "h1:PYbnOqRuGn4c0/Ae1f7yOS/0zvmXNHFJRZjuF4KECnM=", - "h1:PlW+UZ4EElQF3NQwf41KQwavFujab3Czc51zu9dyVM8=", - "h1:g40qr7yDmIpaur4SsK5BcOda3HSo1RJ6zHVMqN4EJ+0=", - "h1:nozfr4CZq73d4HjubKJundX/8A+Mj282oS/hyNfjUPU=", - "zh:05f4734c1f0be840b711b3eff259ebc5fca436784c728955b1678078466f48d7", - "zh:0b91bf19371d012434eba1deeb6aab77158def9b39601dcbd94450b3974a2a26", - "zh:0ee6eacd47ec00183d55d726a4b6c4ce951a199f944bf22f1aa58392ebdfa7a2", - "zh:19388a4074b76a89a43a6c8328d7ae8ee2e7de3d346af51e80d3e6d3d12925f1", - "zh:23e74d48c5e2ac2e823fd527f49fee9db37d32a1990c9e3bf126ead697b843eb", - "zh:3cabf7fbd096c520064aae3aba61aba670af83ab91291a71fa1b1332929c2b7f", - "zh:5c0a3b8af0be60be4eca12ddee385cfa8babc1ec8e98cdf9de2f2274c73eabfa", - "zh:60b4f8a8ef18f52bf8e19215229dae408bee732825964092db7c989fd2de4097", - "zh:7359015acfedcbd6366f2329c854cf8d3c8ca5cd0faa89d2d37db358d6eba6c5", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7b38758402f0e13a1071162da28994023cd2ac676e54af350c9ffd8dfa73fa7b", - "zh:7c7fbb8895eb75bb4de1f933e98553bd99c8d048c89a925ddba490aa5a67f7dc", - "zh:8c2b8c6a7ccdec16b73e2fb9f3700ea097f58c592571e4c5de60c93d2301732c", - ] -} diff --git a/cloud/infra/terraform/README.md b/cloud/infra/terraform/README.md deleted file mode 100644 index eebe07af809..00000000000 --- a/cloud/infra/terraform/README.md +++ /dev/null @@ -1,409 +0,0 @@ -# Terraform - -This root manages the Orca Cloud relay and nothing else. It requires Terraform >= 1.7 -(`removed` blocks); OpenTofu at that floor works too. - -## Three roots - -Orca Cloud is three Terraform roots sharing one project and one state bucket per environment, -with a different prefix each. They are separate so the relay can be extracted into a public -repository without carrying the app plane, its database passwords, or its Cloudflare credential -with it. - -| Root | Directory | State prefix | Owns | -| --- | --- | --- | --- | -| foundation | `infra/terraform-foundation` | `terraform/foundation` | Project service enablement, the Artifact Registry repository, the API runtime service account, the Cloud SQL instance, the GitHub Workload Identity pool, the Cloud SQL rollout lease grant | -| apps | `infra/terraform-apps` | `terraform/apps` | The API and auth services, the artifact and skill-package buckets, the skill plane and its observability, auth and artifact DNS, the app deploy identities | -| relay | `infra/terraform` | `terraform/state` | Everything relay: the director, GCE cells, the fence broker, relay observability, and the relay operator identities | - -Apply order on a greenfield project is **foundation first**, then relay and apps in either order. -The other two roots reach foundation only by literal or by `data` lookup, never through -`terraform_remote_state`: foundation state holds the Cloud SQL instance and apps state holds -generated database passwords in cleartext, and the relay root must not acquire a read path into -either once it is public. Each root's substitution for a foundation value is pinned by -`dev/scripts/terraform-root-partition.test.mjs`, which asserts every declared resource family is -owned by exactly one root per environment. - -Three families are owned per environment rather than outright: the shared deploy service account, -its Workload Identity provider, and its WIF binding live in the relay root for production and in -the apps root for staging, with complementary counts. Every IAM binding on that account follows -it. `dev/fixtures/terraform-root-partition/families.json` is the authority. - -### The carve is complete - -Both state surgeries have run (`docs/terraform-root-split-runbook.md`), so this root's state holds -only relay families and the `removed` guard blocks from the window are gone. The shared deploy -identity (`google_service_account.github_deploy`, its provider, and its bindings) is declared here -with production-only counts; staging's copies are declared by `infra/terraform-apps`. An untargeted -plan is orderable again; the `Plan:` line still reflects the standing cell-template drift backlog. - -### Workload Identity trusts the public repository - -The cutover closed on 2026-09-03. Every relay Workload Identity provider now accepts exactly one -repository, `stablyai/orca` (`1183888342`, owner `127256420`), and every workflow ref it names is -built from `github_workflow_file_prefix` (`cloud-`), which is the rename the public repo applies to -the workflow files it carries. `github_repo`, `github_repo_id`, and that prefix are set in both -`environments/*.tfvars` as well as defaulted here, and `github_accepted_repositories` is empty. -Nothing in this root trusts `stablyai/orca-cloud` any more; the apps and foundation roots still do, -because the app workflows still live there. - -`github_accepted_repositories` stays available for the next repository move. Each entry renders its -own parenthesised OR arm in `relay-github-workflow-trust.tf`, carrying that repository's own -`repository`, `repository_id`, and `repository_owner_id` claims plus its exact workflow refs, while -`ref`, `environment`, and `event_name` stay outside the OR. An empty list renders byte-identically -to the single-repository form, so adding and removing a repository is a tfvars edit with no provider -block change. The rendered strings are pinned by -`dev/scripts/workload-identity-attribute-conditions.test.mjs`. - -Repointing the primary and emptying the list must land in the same apply: dropping the accepted -entry before repointing the primary would revoke the surviving repository mid-flight. - -### `ORCA_RELAY_IMAGE_DIGEST` is not Terraform-owned - -`deploy-relay-blue-green.mjs` sets `ORCA_RELAY_IMAGE_DIGEST` on the director container at deploy -time, but `relay.tf` does not declare it and the director's `ignore_changes` cannot name a single -list element. A director apply from this root therefore strips that variable. Terraform is not the -owner today: deploy through the director workflow, and treat any direct -`google_cloud_run_v2_service.relay` apply as something that needs the next deploy to restore the -digest. Giving Terraform the variable (a declared input the deploy script writes through) is -tracked as follow-up work in the split checklist, not in this change. - -Select a root with `--root`; omitting it keeps the relay root, so existing callers are unchanged. - -```sh -pnpm infra:init --env staging --root foundation -pnpm infra:plan --env staging --root apps -``` - -## Bootstrap Remote State - -The GCS backend bucket must exist before `init`. - -Staging: - -```sh -gcloud storage buckets create gs://onorca-cloud-staging-terraform-state --project onorca-cloud-staging --location us -gcloud storage buckets update gs://onorca-cloud-staging-terraform-state --versioning -``` - -Production: - -```sh -gcloud storage buckets create gs://onorca-cloud-terraform-state --project onorca-cloud --location us -gcloud storage buckets update gs://onorca-cloud-terraform-state --versioning -``` - -One bucket per environment holds all three roots' state under separate prefixes, so this is a -one-time step for the whole project. - -Do not commit `.tfstate`, `.tfplan`, or `.terraform` files. - -## Production app deploy identity: moved - -The production app deploy identity, the skill alert channel guard, and every -other app-plane resource now live in `infra/terraform-apps`. Their bootstrap -procedure moved with them; run it with `-chdir=infra/terraform-apps`. This root -no longer declares the API service, the auth service, the artifact or -skill-package buckets, the Cloud SQL databases, or the app DNS records, and it -no longer needs a Cloudflare or 1Password credential. - -## Staging Relay capacity identity bootstrap - -Before the capacity workflow can mutate staging, create a saved targeted plan -containing only `github_staging_relay_capacity` providers, accounts, roles, -bindings, and outputs. Apply it with backend locking, then require a targeted -refresh/no-op plan. The identity is bound to the exact workflow on `main` and -the `staging` environment. Its state write access is limited to the default -staging state and lock object prefix. - -Copy these outputs into same-named staging GitHub environment variables: - -1. `github_staging_relay_capacity_workload_identity_provider` -2. `github_staging_relay_capacity_service_account` - -Before dispatch, prove it can read the saved state and reviewed Relay -resources, but cannot change unrelated Cloud Run services, templates, managed -instance groups, databases, DNS, or secrets. - -The identity bootstrap alone does not authorize Relay admin routes. Publish the -compatible image, then use the existing staging blue/green deploy path to carry -and verify the capacity service account on both director revisions. Use saved, -validated cell plans through `Bootstrap Relay Staging Capacity` to roll C2/C3 -at 600/60. The reviewed staging tfvars must pin the same image and capacity. -Require a read-only 600/60 capacity-workflow -run before reviewing either 1,000-policy configuration. Do not target the -director with Terraform: its dependency closure includes unrelated live drift. - -## Production Relay incident identity bootstrap - -Before running the production monitor, create a saved targeted plan containing -only the two dedicated service accounts, their exact-workflow -providers/bindings, and monitor read roles. Reject any Cloud Run, GCE, -database, network, runtime-service-account, or unrelated IAM change. Apply -that plan with backend locking, then run a targeted refresh/no-op plan. - -Copy these outputs, in order, into same-named production GitHub environment -variables documented in `.github/workflows/README.md`: - -1. `github_relay_monitor_workload_identity_provider` -2. `github_relay_monitor_service_account` -3. `github_relay_fence_workload_identity_provider` -4. `github_relay_fence_service_account` - -Use an audited operator session for the GitHub variable writes. Before -dispatch, prove the monitor account can read required aggregate telemetry but -cannot mutate Relay or state. Fence modes remain disabled until a separate -private broker owns and validates the exact state, plan, cell, and durable -attempt boundary; never grant direct Compute update or Terraform-state write -access or director mutations to the GHA fence account. - -## Relay Asia topology identity bootstrap - -Bootstrap each environment's Asia topology identity with operator credentials -before dispatching its workflow. IAM cannot bootstrap itself. Reinitialize the -exact backend and save a targeted plan that -contains only these twelve additive resources: - -1. `google_iam_workload_identity_pool_provider.github_relay_asia_topology` -2. `google_service_account.github_relay_asia_topology` -3. `google_service_account_iam_member.github_relay_asia_topology_workload_identity_user` -4. `google_project_iam_custom_role.github_relay_asia_topology_mutation` -5. `google_project_iam_member.github_relay_asia_topology_mutation` -6. `google_project_iam_custom_role.github_relay_asia_topology_read` -7. `google_project_iam_member.github_relay_asia_topology_read` -8. `google_artifact_registry_repository_iam_member.github_relay_asia_topology_artifact_reader` -9. `google_storage_bucket_iam_member.github_relay_asia_topology_state` -10. `google_project_iam_custom_role.github_relay_asia_topology_state_list` -11. `google_storage_bucket_iam_member.github_relay_asia_topology_state_list` -12. `google_service_account_iam_member.github_relay_asia_topology_runtime_user` - -The state-list role contains only `storage.objects.list`. Terraform's GCS backend -needs that bucket-level permission before it can access the exact state and lock -objects protected by the conditional object-admin binding. - -Production also requires one exact in-place update to -`google_iam_workload_identity_pool_provider.github[0]` so the existing deploy -identity accepts `operate-relay-asia-admission.yml`; staging's shared provider -already accepts repository workflows. Reject every other change. Apply only -that saved plan, then require the same targeted plan to be empty. Publish the two -`github_relay_asia_topology_*` outputs as the matching staging or production -GitHub environment variables documented in `.github/workflows/README.md`. - -Apply observability separately from IAM and topology. The topology identity -has no IAM, logging-metric, alert-policy, Cloud SQL, DNS, certificate, global -IP, or deletion permission. Its read role includes `serviceusage.services.list` -because the Google provider lists managed APIs while refreshing targeted plans. -Its mutation role includes `compute.networks.updatePolicy`, which Compute requires -to attach the reviewed Asia subnet and router to the existing Relay VPC. -It also includes `compute.healthChecks.useReadOnly`, which backend creation requires -to reference the existing Relay readiness health check. -Managed-group creation additionally requires `compute.instanceGroups.create`; adding -that group as a backend requires `compute.instanceGroups.use` and `compute.instances.use`. - -Bootstrap the staging Asia proof identity separately before its director roll. -Its targeted plan contains only the proof provider, service account, -workload-identity binding, logging/monitoring viewer bindings, and two outputs. The provider -accepts only `prove-relay-asia-staging.yml` on `main` in the staging environment; -the account has no Compute, Cloud SQL, Secret Manager, Terraform-state, or -Cloud Run mutation permission. Publish its provider and account outputs as -`STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER` and -`STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT`, then deploy the compatible -staging director so it accepts that exact account for bounded capacity routes. - -## Relay regional-placement switch bootstrap - -Before the first director deployment that references the regional-placement -switch, apply its Secret Manager resources with operator credentials. Save a -targeted plan containing exactly these six additions and no other changes: - -1. `google_secret_manager_secret.relay_regional_placement_enabled` -2. `google_secret_manager_secret_version.relay_regional_placement_enabled` -3. `google_secret_manager_secret_iam_member.relay_regional_placement_runtime_accessor` -4. `google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor[0]` -5. `google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder[0]` -6. `google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer[0]` - -Pass the exact environment tfvars, apply only -the saved plan, then require the same targeted plan to be empty. Verify the -runtime and deploy identities can access the secret without printing its value, and the deploy -identity can read version metadata without gaining broader mutation rights. -Only then deploy a director revision. Every director revision pins one exact -numeric secret version; the audited director workflow preserves the serving -version by default and creates a new boolean version only for an explicit -enable or disable. The traffic move is therefore the switch commit, and a -failed candidate cannot change the value used by serving instances. -Terraform reads and preserves the currently served director's exact numeric -version, falling back to the bootstrap version only before the setting exists. -It still owns the secret name and every environment field; an unrelated apply -therefore cannot revert a later audited switch version. - -## Relay director runtime identity bootstrap - -Before the regional-rehome director rollout, create the distinct director -runtime identity with operator credentials. Reinitialize the exact environment -backend, export a fresh `GOOGLE_OAUTH_ACCESS_TOKEN` without printing it, pass -the environment tfvars, and save a targeted -plan containing only the applicable resources below: - -1. `google_service_account.relay_director_runtime` -2. `google_project_iam_member.relay_director_runtime_cloudsql_client` -3. `google_secret_manager_secret_iam_member.relay_assignment_signing_key_director_accessor` -4. `google_secret_manager_secret_iam_member.relay_regional_placement_director_accessor` -5. `google_secret_manager_secret_iam_member.relay_database_url_director_accessor` -6. `google_service_account_iam_member.github_relay_director_runtime_service_account_user[0]` -7. `google_iam_workload_identity_pool_provider.github[0]` in production when its - condition adds the exact regional-rehome and same-cap workflow/job pairs -8. `google_iam_workload_identity_pool_provider.github_production_relay_capacity[0]` - in production when its condition adds the exact same-cap workflow/job pair - -Reject Cloud Run, GCE template, database, network, DNS, or any other change. -Apply only the reviewed saved plan, then require the same targeted plan to be -empty. Publish `relay_director_runtime_service_account` and -`relay_runtime_service_account` as the matching GitHub environment variables -documented in `.github/workflows/README.md`. The identity bootstrap does not -authorize a rollout by itself; use the one-time director workflow mode so both -candidate and rollback revisions move together while rehoming remains durably -disabled. - -## Usage - -```sh -pnpm infra:init --env staging -pnpm infra:plan --env staging -pnpm infra:apply --env staging -``` - -Add `--root foundation` or `--root apps` for the other two roots; the default is the relay root. -On a greenfield project apply foundation before either of the others. - -Run staging first. Production should only follow after staging has a successful `/health` smoke test. - -## Relay staging topology - -The stable Cloud Run service is the director. Staging uses fixed-one GCE cells so its data plane -matches production; Cloud Run stamped cells are no longer retained in the live environment. - -Staging is intentionally allowed to drift to a powered-off runtime state between internal test -windows. Use the `Power Relay Staging` GitHub Actions workflow to inspect, wake, or sleep it. A -normal `pnpm infra:apply --env staging` refuses while Cloud SQL is stopped or any staging MIG is -scaled below its Terraform-owned size of one. Dispatch `wake` with `wake-cells: all`, wait for its -health checks, and only then apply a reviewed staging plan. Do not use Terraform to wake staging: -that can mix infrastructure changes with a partial power transition. - -The staging tfvars keep both Cloud Run services at zero minimum instances. Requests wake the auth -service and director when SQL is running; the power workflow separately controls SQL and the GCE -MIGs. The staging-only GitHub service-account role can resize those MIGs and change the SQL -activation policy. Terraform does not create that role in production. - -## Relay GCE production data plane - -`relay_gce_domain` creates the shared private network/NAT, LB address, and Certificate Manager -wildcard authorization used by fixed-one GCE cell MIGs. Cells use exact hosts one label below the -domain, such as `c1.relay-staging.onorca.dev`; future cells therefore reuse one DNS-only wildcard -A record while the HTTPS URL map still admits only Terraform-configured exact hosts. - -After the foundation apply, publish both Terraform outputs and leave them in place for renewal: - -1. `relay_gce_certificate_dns_authorization`: the exact Certificate Manager CNAME. -2. `relay_gce_wildcard_dns_record`: the DNS-only wildcard A record to the reserved LB address. - -Every `relay_gce_cells` entry is one durable cell generation and must pin both its exact COS boot -image and its Artifact Registry relay image. Terraform creates one private COS instance template, one size-one zonal -MIG, and one backend service for that exact host. The MIG uses `RECREATE`, zero surge, and one -unavailable worker; `/health` alone drives autoheal while SQL/JWKS-backed `/ready` controls LB -admission. The backend timeout is 86,400 seconds and connection draining is 60 seconds, which -covers only a host still mid-handshake because the rollout drains a cell before Terraform runs. -The URL map aborts unknown wildcard hosts before they reach a worker. The startup script obtains short-lived metadata -credentials, fetches the two relay secrets without logging them, and runs a digest-pinned Cloud SQL -Auth Proxy beside the digest-pinned relay image. - -The primary `us-central1` subnet, router, and NAT retain their original -Terraform addresses. `relay_gce_additional_region_subnetwork_cidrs` creates -only additive regional resources; cells select the subnet from their declared -region. Every cell also declares an explicit database pool maximum in startup -metadata and deployment outputs. The initial Asia shape is `e2-standard-4`, -3,000 physical connections, 60 unobserved connections, 6,000 request units, -and a database pool maximum of 10. Production Asia pools now run at 16 and -staging C4 stays at 10; the topology and director validators pin both. - -Provision the complete identical Asia wave in one `Deploy Relay Asia Topology` -saved plan. Its validator permits only the additive subnet/router/NAT, reviewed -cell templates/MIGs/backends, and exact shared URL-map host additions. It -rejects deletes, replacements, loss of an existing host route, US-resource -changes, and unrelated drift. Do not add production C27-C29 until the -compatible image has been published and each entry can pin its immutable -digest. A later cell, such as C30, C31, C34 or the C35 spare, is its own reviewed wave. The shared URL map -pulls every live cell into its plan, so the workflow plans each live cell at the -image its state template already serves, and the validator rejects any change -to a cell outside the wave. US C32 and C33 use the same workflow at the same -3,000-connection shape, as one wave because they are declared together: each -wave's region comes from its reviewed zone, a -`us-central1` wave targets no additional-region network, and its template -carries no region label or line and the default pool of 10. - -Topology creation intentionally does not apply the director resource. Once all -MIGs and backends are healthy, register every new cell atomically as -migration-only through `Operate Relay Asia Admission` with the exact live -selector generation and a durable attempt ID. Only then may a director -deployment list the new cells. Verify that configuration and fresh heartbeats -before using the same workflow to promote the canary. A failed canary returns to -migration-only; do not delete the Asia network during rollout recovery. - -`relay_gce_cells` takes precedence in the director's configured-cell list. Adding or replacing a -generation requires a new map key and hostname; do not change an active cell's image in place. Run -`terraform fmt -check -recursive` and `terraform validate` locally; the same non-credentialed -checks run on every pull request. - -GCE deployments must add a distinct cell ID, host, backend, and MIG for every candidate generation; -never update an existing generation's image behind its origin. - -For a post-launch worker replacement, first publish an immutable image with the production image -workflow. Add that digest as a distinct `relay_gce_cells` entry with -`initially_enabled = false`, review/apply the Terraform change, and deploy the compatible director. -The production candidate workflow then reads the remote-state topology and defaults to a read-only -preflight. It verifies the exact TLS origin, `/health`, dependency-backed `/ready`, authenticated -heartbeat, served digest, private fixed-one MIG, runtime identity, dedicated backend, 86,400-second -timeout, and authoritative request-unit headroom. `execute` requires the literal `EVACUATE` -confirmation, disables the source only after preflight, enables the candidate, performs bounded -target-first evacuation, drains the exact source origin, and verifies aggregate completion. Keep -both source and candidate Terraform routes until a later reviewed removal proves the old origin has -no assignments, activity leases, or migrations. - -After any failure following target registration, use `audit` before `recover-forward`; never retry -`execute` or reverse admission. Forward recovery retries only bounded idempotent status operations. -If every remaining migration belongs to a registered target whose desktop is currently offline, it -emits `candidate_forward_pending` and stops without retiring those rows. Keep both origins intact -and rerun recovery only after a fresh audit shows target controls have returned. - -The production multi-target workflow is the reviewed path for evacuations that -need more than one candidate. It enforces deterministic serialized quotas, -target connection ceilings, and the oldest-migration lease gate before drain. -After selector generation 1, add new disabled targets without changing the -director resource in the targeted apply. Deploy the selector-version-2 -director, apply only the new cell templates, MIGs, backends, and URL-map -routes, then use `add-migration-cells` to register their exact configs as -migration-only in one selector generation. A single additive target is valid; -ordinary evacuation and supersession retain their multi-target requirements. -Use `retire-migration-cell` with an exact attempt ID to move one -migration-only cell to existing-only before its reviewed fence. -The director does not depend on the GCE forwarding-rule graph; keep director -configuration plans scoped away from immutable cell generations. -Its guarded `fence-source` mode applies an exact private Terraform saved plan -for a fully quiescent cell already listed in `relay_gce_fenced_cells`. The plan -must contain only that MIG's in-place target-size change from one to zero, so -the origin, backend, and generation remain retained. An interrupted apply is -always recovered forward unless Terraform state, live GCE state, and operation -history prove it never began. `abort-fence-source` records that proven -pre-apply abort; remove the cell from the fence set only in a later reviewed -commit. Never resize a production relay MIG directly. -Before the first fencing workflow rollout, apply this schema with an empty -fence set so remote-state topology contains `generation_identity`, -`fenced`, and `desired_target_size`. Only then commit a cell ID into the -production fence set. -The same workflow's `supersede-target` mode is the only supported path for a -failed registered target: it proves the failed MIG is zero with no instances -before recording an exact-incarnation fence and publishing newer epochs. -It invokes an IAM-authenticated max-one Cloud Run broker. The broker runtime -alone can access the exact state/saved-plan/lease object prefixes and update a -Relay MIG; the GitHub requester can read aggregate safety evidence and invoke -that service but cannot perform either mutation directly. diff --git a/cloud/infra/terraform/backend/production.hcl b/cloud/infra/terraform/backend/production.hcl deleted file mode 100644 index e482b091273..00000000000 --- a/cloud/infra/terraform/backend/production.hcl +++ /dev/null @@ -1,3 +0,0 @@ -bucket = "onorca-cloud-terraform-state" -prefix = "terraform/state" - diff --git a/cloud/infra/terraform/backend/staging.hcl b/cloud/infra/terraform/backend/staging.hcl deleted file mode 100644 index fbd0f0c17b2..00000000000 --- a/cloud/infra/terraform/backend/staging.hcl +++ /dev/null @@ -1,3 +0,0 @@ -bucket = "onorca-cloud-staging-terraform-state" -prefix = "terraform/state" - diff --git a/cloud/infra/terraform/environments/production.tfvars b/cloud/infra/terraform/environments/production.tfvars deleted file mode 100644 index b2d5a27a22a..00000000000 --- a/cloud/infra/terraform/environments/production.tfvars +++ /dev/null @@ -1,533 +0,0 @@ -project_id = "onorca-cloud" -environment = "production" -name_prefix = "orca-cloud" -region = "us-central1" - -artifact_repository_id = "orca-cloud" - -# The relay source lives in the public stablyai/orca repository, where the workflows carry a -# `cloud-` file prefix. github_owner and github_owner_id keep their defaults. -github_repo = "orca" -github_repo_id = "1183888342" -github_workflow_file_prefix = "cloud-" - -# Our first-party auth service. auth.onorca.dev is PropelAuth's prod domain, so -# our service lives at login.onorca.dev (desktop points ORCA_CLOUD_API_URL here). -auth_base_url = "https://login.onorca.dev" - -relay_cloud_run_service_name = "orca-cloud-relay" -relay_base_url = "https://relay.onorca.dev" -# Why: public admission is a per-instance semaphore, so fleet assignment capacity is -# concurrency x instances. Scaling to 2 instances took placement failures 35% -> 70%. -relay_min_instances = 5 -relay_max_instances = 5 -# Proven on staging (2026-10-11, image F) before production turns them on. -relay_director_cloud_sql_proxy_sidecar = true -relay_event_loop_stall_alert = true -# Production cells run only on fixed-one GCE MIGs; Cloud Run remains the director. -relay_cells = {} -manage_relay_domain_mapping = true - -# Production GCE cells use exact hosts such as c1.relay.onorca.dev. -# The wildcard only handles DNS/TLS; the load balancer rejects unknown hosts. -relay_gce_domain = "relay.onorca.dev" -relay_gce_subnetwork_cidr = "10.42.0.0/24" -relay_gce_additional_region_subnetwork_cidrs = { - "asia-east2" = "10.42.1.0/24" -} -# Fenced cells are retired existing-only capacity: the selector can never place on them again, -# so their MIGs run at zero rather than holding a VM and 10 Postgres connections each. -relay_gce_fenced_cells = [ - "production-gce-c1", - "production-gce-c2", - "production-gce-c3", - "production-gce-c6", - "production-gce-c11", - "production-gce-c12" -] -# Initial cells stay admission-disabled until production preflight and go-live approval. -relay_gce_cells = { - "production-gce-c1" = { - hostname = "c1" - zone = "us-central1-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:36a56b106c9e6d5897135c6af829085ab8fd53c85466406d9e887a7a5cfe9a02" - initially_enabled = false - } - "production-gce-c2" = { - hostname = "c2" - zone = "us-central1-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:36a56b106c9e6d5897135c6af829085ab8fd53c85466406d9e887a7a5cfe9a02" - initially_enabled = false - } - "production-gce-c3" = { - hostname = "c3" - zone = "us-central1-c" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:19ef4e6e4a043f63d78011d1c29a395a9002ec077d03ee6931f25479fe66f349" - initially_enabled = false - } - # Distinct origins let each existing cell drain without an in-place image swap. - "production-gce-c4" = { - hostname = "c4" - zone = "us-central1-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:36a56b106c9e6d5897135c6af829085ab8fd53c85466406d9e887a7a5cfe9a02" - initially_enabled = false - } - "production-gce-c5" = { - hostname = "c5" - zone = "us-central1-c" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d" - initially_enabled = false - } - "production-gce-c6" = { - hostname = "c6" - zone = "us-central1-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:36a56b106c9e6d5897135c6af829085ab8fd53c85466406d9e887a7a5cfe9a02" - initially_enabled = false - } - "production-gce-c7" = { - hostname = "c7" - zone = "us-central1-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c8" = { - hostname = "c8" - zone = "us-central1-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c9" = { - hostname = "c9" - zone = "us-central1-c" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - # Canary for the control-activation fence (PR #207, main b253fcd). - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c10" = { - hostname = "c10" - zone = "us-central1-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c11" = { - hostname = "c11" - zone = "us-central1-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:e592371013188b8297e395979c70a8b42c39f4bb5f90b01190f0778279cbaef5" - initially_enabled = false - connection_hard_cap = 600 - connection_unobserved_bound = 60 - } - "production-gce-c12" = { - hostname = "c12" - zone = "us-central1-c" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:3d8b388dcbf190be20491ce9c14eeafa0dccd0afbb2725712f6f9d9a754838dc" - initially_enabled = false - connection_hard_cap = 600 - connection_unobserved_bound = 60 - } - "production-gce-c13" = { - hostname = "c13" - zone = "us-central1-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c14" = { - hostname = "c14" - zone = "us-central1-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c15" = { - hostname = "c15" - zone = "us-central1-c" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c16" = { - hostname = "c16" - zone = "us-central1-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c17" = { - hostname = "c17" - zone = "us-central1-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d" - initially_enabled = false - connection_hard_cap = 600 - connection_unobserved_bound = 60 - } - # Canary for halved control lease renewal (PR #253, main 11256b5). Chosen as the - # smallest live cell: ~9 connections, so a replace costs 9 reconnects, not ~400. - "production-gce-c18" = { - hostname = "c18" - zone = "us-central1-c" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d" - initially_enabled = false - connection_hard_cap = 600 - connection_unobserved_bound = 60 - } - "production-gce-c19" = { - hostname = "c19" - zone = "us-central1-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c20" = { - hostname = "c20" - zone = "us-central1-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - # First full-size cell on the halved lease renewal, after the C18 canary measured - # 9.15 -> 5.32 queries per connection with zero failures. C21 also carries the - # control-close churn we still need to attribute to a client build. - "production-gce-c21" = { - hostname = "c21" - zone = "us-central1-c" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c22" = { - hostname = "c22" - zone = "us-central1-c" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c23" = { - hostname = "c23" - zone = "us-central1-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c24" = { - hostname = "c24" - zone = "us-central1-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c25" = { - hostname = "c25" - zone = "us-central1-c" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c26" = { - hostname = "c26" - zone = "us-central1-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "production-gce-c27" = { - hostname = "c27" - region = "asia-east2" - zone = "asia-east2-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 6000 - database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters). - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 3000 - connection_unobserved_bound = 60 - } - "production-gce-c28" = { - hostname = "c28" - region = "asia-east2" - zone = "asia-east2-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 6000 - database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters). - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 3000 - connection_unobserved_bound = 60 - } - "production-gce-c29" = { - hostname = "c29" - region = "asia-east2" - zone = "asia-east2-c" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 6000 - database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters). - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 3000 - connection_unobserved_bound = 60 - } - "production-gce-c30" = { - hostname = "c30" - region = "asia-east2" - zone = "asia-east2-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 6000 - database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters). - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:4158d8a2e18e9caec439d257f0c1e45d92ffea8c0262f057b2f08c76a134bcf0" - initially_enabled = false - connection_hard_cap = 3000 - connection_unobserved_bound = 60 - } - "production-gce-c31" = { - hostname = "c31" - region = "asia-east2" - zone = "asia-east2-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 6000 - database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters). - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269" - initially_enabled = false - connection_hard_cap = 3000 - connection_unobserved_bound = 60 - } - # US cells at the Asia 3,000-host shape, one topology wave: a was lightest, b ties c. - "production-gce-c32" = { - hostname = "c32" - region = "us-central1" - zone = "us-central1-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 6000 - database_pool_max = 10 # The US default; 16 exists only for the asia-east2 round trip. - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269" - initially_enabled = false - connection_hard_cap = 3000 - connection_unobserved_bound = 60 - } - "production-gce-c33" = { - hostname = "c33" - region = "us-central1" - zone = "us-central1-b" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 6000 - database_pool_max = 10 # The US default; 16 exists only for the asia-east2 round trip. - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269" - initially_enabled = false - connection_hard_cap = 3000 - connection_unobserved_bound = 60 - } - # Sixth Asia cell: launched as a migration-only spare, now general; c completes the 2/2/2 zone spread. - "production-gce-c34" = { - hostname = "c34" - region = "asia-east2" - zone = "asia-east2-c" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 6000 - database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters). - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269" - initially_enabled = false - connection_hard_cap = 3000 - connection_unobserved_bound = 60 - } - # Seventh Asia cell: launched as a migration-only spare, now general; a starts the third zone round. - "production-gce-c35" = { - hostname = "c35" - region = "asia-east2" - zone = "asia-east2-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 6000 - database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters). - image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe" - initially_enabled = false - connection_hard_cap = 3000 - connection_unobserved_bound = 60 - } -} - -relay_region_rehome_source_cell_ids = [ - "production-gce-c7", - "production-gce-c8", - "production-gce-c9", - "production-gce-c10", - "production-gce-c13", - "production-gce-c14", - "production-gce-c15", - "production-gce-c16", - "production-gce-c19", - "production-gce-c20", - "production-gce-c21", - "production-gce-c22", - "production-gce-c23", - "production-gce-c24", - "production-gce-c25", - "production-gce-c26", - # Asia cells carry the same trust so mis-homed hosts can be drained back off them. - "production-gce-c27", - "production-gce-c28", - "production-gce-c29", - "production-gce-c30", - "production-gce-c31", - "production-gce-c32", - "production-gce-c33", - "production-gce-c34", - "production-gce-c35" -] - -# Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply -# was otherwise going to strip it from every policy, leaving the alerts firing at nobody. -relay_alert_notification_channels = ["projects/onorca-cloud/notificationChannels/4879431412695417284"] - -# Cells below this RELAY_FIX_LEVEL page after 6 hours. Raise it with a targeted apply of the -# outdated-image alert once a wave has rolled every serving cell, never mid-wave. -relay_cell_min_fix_level = 1 - -# Mobile push gateway. Production is the only environment that runs one; the runtime account, -# the three Apple secrets, and their accessor bindings already exist and are imported once -# (see docs/push-gateway.md). -push_gateway_enabled = true -push_base_url = "https://push.onorca.dev" -# Dedicated push pools allow three revision resources during validation and recovery. -push_max_instances = 2 -push_database_pool_max = 6 -manage_push_domain_mapping = true diff --git a/cloud/infra/terraform/environments/staging.tfvars b/cloud/infra/terraform/environments/staging.tfvars deleted file mode 100644 index 9d3fac2c3c4..00000000000 --- a/cloud/infra/terraform/environments/staging.tfvars +++ /dev/null @@ -1,139 +0,0 @@ -project_id = "onorca-cloud-staging" -environment = "staging" -name_prefix = "orca-cloud-staging" -region = "us-central1" - -artifact_repository_id = "orca-cloud" - -# The relay source lives in the public stablyai/orca repository, where the workflows carry a -# `cloud-` file prefix. github_owner and github_owner_id keep their defaults. -github_repo = "orca" -github_repo_id = "1183888342" -github_workflow_file_prefix = "cloud-" - -auth_base_url = "https://auth-staging.onorca.dev" - -relay_cloud_run_service_name = "orca-cloud-relay-staging" -relay_staging_power_auth_service_name = "orca-cloud-auth-staging" -relay_base_url = "https://relay-staging.onorca.dev" -relay_min_instances = 0 -relay_max_instances = 2 - -# Staging proves these before production turns them on. -relay_director_cloud_sql_proxy_sidecar = true -relay_event_loop_stall_alert = true -relay_dead_cell_verdict_alert = true -# What the serving staging director runs (revision 00171, 2026-10-11). Pinned so the sidecar apply, -# whose revision the next deploy adopts as its shape, does not also drop it to the 80 / 30 s defaults. -relay_director_concurrency = 1000 -relay_director_request_timeout_seconds = 3600 -# Staging now exercises the production-shaped GCE data plane exclusively. -relay_cells = {} -# Keep the stable director mapping Terraform-owned while removing cell mappings. -manage_relay_domain_mapping = true - -# GCE cells use exact hosts below this wildcard, for example -# c1.relay-staging.onorca.dev. Cloudflare records remain out-of-band. -relay_gce_domain = "relay-staging.onorca.dev" -relay_gce_subnetwork_cidr = "10.42.0.0/24" -relay_gce_additional_region_subnetwork_cidrs = { - "asia-east2" = "10.42.1.0/24" -} -relay_gce_fenced_cells = [] -relay_gce_cells = { - "staging-gce-c1" = { - hostname = "c1" - zone = "us-central1-b" - machine_type = "e2-standard-2" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - # Step-6 image J (main 30f5bef9, phonePath). Never 2d0f6e6: it cannot report capped telemetry. - image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:21f1fead4d3b9900d3b1179fd72569dcba548dc0ee29c76c8389fec94f74a7b4" - # Step-5 staging e2e: c1 is the second US reserve cell, so it needs a cap and rehome trust. - connection_hard_cap = 600 - connection_unobserved_bound = 60 - } - "staging-gce-c2" = { - hostname = "c2" - zone = "us-central1-c" - machine_type = "e2-standard-2" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe" - connection_hard_cap = 600 - connection_unobserved_bound = 60 - } - "staging-gce-c3" = { - hostname = "c3" - zone = "us-central1-a" - machine_type = "e2-standard-2" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" - initially_enabled = false - connection_hard_cap = 1000 - connection_unobserved_bound = 60 - } - "staging-gce-c4" = { - hostname = "c4" - region = "asia-east2" - zone = "asia-east2-a" - machine_type = "e2-standard-4" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 6000 - database_pool_max = 10 - image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:21f1fead4d3b9900d3b1179fd72569dcba548dc0ee29c76c8389fec94f74a7b4" - initially_enabled = false - connection_hard_cap = 3000 - connection_unobserved_bound = 60 - } - # Dead-cell guard e2e: six feed cells, five in the US, so one dead US cell is 1/6 of the fleet and - # 1/5 of its region (cell-poll-reachability.ts). c2's shape. - "staging-gce-c5" = { - hostname = "c5" - zone = "us-central1-a" - machine_type = "e2-standard-2" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d" - connection_hard_cap = 600 - connection_unobserved_bound = 60 - } - "staging-gce-c6" = { - hostname = "c6" - zone = "us-central1-b" - machine_type = "e2-standard-2" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d" - connection_hard_cap = 600 - connection_unobserved_bound = 60 - } - "staging-gce-c7" = { - hostname = "c7" - zone = "us-central1-f" - machine_type = "e2-standard-2" - boot_disk_gb = 30 - boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13" - capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d" - connection_hard_cap = 600 - connection_unobserved_bound = 60 - } -} - -# c1, c4 and c5-c7 hold rehome trust for the step-5 staging e2e: the seat feed, reserve and demote -# routes need it. -relay_region_rehome_source_cell_ids = [ - "staging-gce-c1", "staging-gce-c2", "staging-gce-c3", "staging-gce-c4", - "staging-gce-c5", "staging-gce-c6", "staging-gce-c7" -] - -# Push is currently provisioned only in production. -push_gateway_enabled = false diff --git a/cloud/infra/terraform/outputs.tf b/cloud/infra/terraform/outputs.tf deleted file mode 100644 index f886b137fce..00000000000 --- a/cloud/infra/terraform/outputs.tf +++ /dev/null @@ -1,216 +0,0 @@ -output "github_deploy_service_account" { - value = try(google_service_account.github_deploy[0].email, null) - description = "Service account email to use in the GitHub Actions deploy workflow." -} - -output "github_workload_identity_provider" { - value = try(google_iam_workload_identity_pool_provider.github[0].name, null) - description = "Workload Identity provider resource name for GitHub Actions." -} - -output "github_relay_monitor_workload_identity_provider" { - value = try(google_iam_workload_identity_pool_provider.github_monitor[0].name, null) - description = "Exact-workflow provider for PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER." -} - -output "github_relay_monitor_service_account" { - value = try(google_service_account.github_monitor[0].email, null) - description = "Read-only identity for PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT." -} - -output "github_relay_fence_workload_identity_provider" { - value = try(google_iam_workload_identity_pool_provider.github_fence[0].name, null) - description = "Exact-workflow provider for PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER." -} - -output "github_relay_fence_service_account" { - value = try(google_service_account.github_fence[0].email, null) - description = "Narrow fencing identity for PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT." -} - -output "github_staging_relay_capacity_workload_identity_provider" { - value = try(google_iam_workload_identity_pool_provider.github_staging_relay_capacity[0].name, null) - description = "Exact-workflow provider for STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER." -} - -output "github_staging_relay_capacity_service_account" { - value = try(google_service_account.github_staging_relay_capacity[0].email, null) - description = "Narrow transition identity for STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT." -} - -output "github_staging_relay_deploy_workload_identity_provider" { - value = try(google_iam_workload_identity_pool_provider.github_staging_relay_deploy[0].name, null) - description = "Exact-workflow provider for STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER." -} - -output "github_staging_relay_deploy_service_account" { - value = try(google_service_account.github_staging_relay_deploy[0].email, null) - description = "Account for STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT." -} - -output "github_production_relay_capacity_workload_identity_provider" { - value = try(google_iam_workload_identity_pool_provider.github_production_relay_capacity[0].name, null) - description = "Exact-workflow provider for PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER." -} - -output "github_production_relay_capacity_service_account" { - value = try(google_service_account.github_production_relay_capacity[0].email, null) - description = "Narrow transition identity for PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT." -} - -output "github_relay_asia_topology_workload_identity_provider" { - value = try(google_iam_workload_identity_pool_provider.github_relay_asia_topology[0].name, null) - description = "Workflow-bound provider for validated additive Relay Asia topology plans." -} - -output "github_relay_asia_topology_service_account" { - value = try(google_service_account.github_relay_asia_topology[0].email, null) - description = "Dedicated identity for validated additive Relay Asia topology plans." -} - -output "github_relay_asia_proof_workload_identity_provider" { - value = try(google_iam_workload_identity_pool_provider.github_relay_asia_proof[0].name, null) - description = "Workflow-bound staging Relay Asia proof Workload Identity provider." -} - -output "github_relay_asia_proof_service_account" { - value = try(google_service_account.github_relay_asia_proof[0].email, null) - description = "Least-privilege staging Relay Asia proof service account." -} - -output "relay_fence_broker_service_uri" { - value = try(google_cloud_run_v2_service.relay_fence_broker[0].uri, null) - description = "IAM-authenticated private Relay fence broker URI." -} - -output "relay_fence_broker_service_account" { - value = try(google_service_account.relay_fence_broker[0].email, null) - description = "Runtime identity that owns exact Relay fence mutations." -} - - -output "relay_cloud_run_service_uri" { - value = google_cloud_run_v2_service.relay.uri - description = "Default relay service URI for pre-domain smoke tests." -} - -output "relay_runtime_service_account" { - value = google_service_account.relay_runtime.email - description = "Runtime identity for stamped Relay cells." -} - -output "relay_director_runtime_service_account" { - value = google_service_account.relay_director_runtime.email - description = "Runtime and regional rehoming caller identity for the Relay director." -} - -output "relay_cell_cloud_run_service_uris" { - value = { - for cell_id, service in google_cloud_run_v2_service.relay_cell : - cell_id => service.uri - } - description = "Native Cloud Run URIs for stamped relay cells." -} - -output "relay_database_name" { - value = google_sql_database.relay.name - description = "Database isolated for durable relay state." -} - -output "relay_gce_load_balancer_ip" { - value = try(google_compute_global_address.relay_gce[0].address, null) - description = "Reserved IPv4 address for the shared GCE relay HTTPS load balancer." -} - -output "relay_gce_wildcard_dns_record" { - value = var.relay_gce_domain == "" ? null : { - name = "*.${var.relay_gce_domain}" - type = "A" - data = try(google_compute_global_address.relay_gce[0].address, null) - } - description = "DNS-only wildcard record that routes future cell hosts to the shared LB." -} - -output "relay_gce_certificate_dns_authorization" { - value = try(google_certificate_manager_dns_authorization.relay_gce[0].dns_resource_record[0], null) - description = "Certificate Manager DNS record that must remain published for renewal." -} - -output "relay_gce_cell_origins" { - value = local.relay_gce_cell_urls - description = "Exact public origins admitted by the shared GCE relay load balancer." -} - -output "relay_gce_cell_instance_groups" { - value = { - for cell_id, manager in google_compute_instance_group_manager.relay_gce_cell : - cell_id => manager.instance_group - } - description = "Terraform-sized managed instance groups backing each GCE relay cell." -} - -output "relay_gce_cell_backend_services" { - value = { - for cell_id, backend in google_compute_backend_service.relay_gce_cell : - cell_id => backend.id - } - description = "Non-overlapping backend service for each exact relay cell host." -} - -output "relay_gce_cell_deployments" { - # try: a cell declared before its topology apply has no resources, and console evaluates this output. - value = { - for cell_id, cell in var.relay_gce_cells : cell_id => { - origin = local.relay_gce_cell_urls[cell_id] - region = cell.region - zone = cell.zone - mig_name = try(google_compute_instance_group_manager.relay_gce_cell[cell_id].name, null) - instance_group = try(google_compute_instance_group_manager.relay_gce_cell[cell_id].instance_group, null) - backend_name = try(google_compute_backend_service.relay_gce_cell[cell_id].name, null) - backend_id = try(google_compute_backend_service.relay_gce_cell[cell_id].id, null) - url_map_name = google_compute_url_map.relay_gce[0].name - generation_identity = try(google_compute_instance_template.relay_gce_cell[cell_id].self_link, null) - image = cell.image - capacity_requests = cell.capacity_requests - database_pool_max = cell.database_pool_max - connection_hard_cap = cell.connection_hard_cap - connection_unobserved_bound = cell.connection_unobserved_bound - initially_enabled = cell.initially_enabled - fenced = contains(var.relay_gce_fenced_cells, cell_id) - desired_target_size = local.relay_gce_cell_target_sizes[cell_id] - target_size = try(google_compute_instance_group_manager.relay_gce_cell[cell_id].target_size, null) - } - } - description = "Non-secret candidate deployment topology consumed by the GCE preflight workflow." - - precondition { - condition = alltrue([ - for cell_id in var.relay_gce_fenced_cells : contains(keys(var.relay_gce_cells), cell_id) - ]) - error_message = "relay_gce_fenced_cells may contain only configured relay_gce_cells keys." - } -} - -output "push_cloud_run_service_uri" { - value = try(google_cloud_run_v2_service.push[0].uri, null) - description = "Default push gateway service URI for pre-domain smoke tests." -} - -output "push_runtime_service_account" { - value = try(google_service_account.push_runtime[0].email, null) - description = "Runtime identity that holds the APNs key and sends through FCM." -} - -output "push_database_name" { - value = try(google_sql_database.push_dedicated[0].name, null) - description = "Database isolated for durable push gateway state." -} - -output "push_dns_record" { - value = var.push_gateway_enabled ? { - name = local.push_fqdn - type = "CNAME" - data = "ghs.googlehosted.com." - } : null - description = "Record the stablyai/orca-cloud apps root must publish in the onorca.dev zone." -} diff --git a/cloud/infra/terraform/push-dedicated-database.tf b/cloud/infra/terraform/push-dedicated-database.tf deleted file mode 100644 index 9a4b9e4c69f..00000000000 --- a/cloud/infra/terraform/push-dedicated-database.tf +++ /dev/null @@ -1,111 +0,0 @@ -resource "google_sql_database_instance" "push_dedicated" { - count = local.push_gateway_count - - project = var.project_id - name = "${var.name_prefix}-push-db" - region = var.region - database_version = "POSTGRES_17" - deletion_protection = true - - settings { - tier = "db-custom-2-7680" - availability_type = "REGIONAL" - edition = "ENTERPRISE" - disk_type = "PD_SSD" - disk_size = 50 - disk_autoresize = true - user_labels = local.relay_shared_labels - - backup_configuration { - enabled = true - point_in_time_recovery_enabled = true - transaction_log_retention_days = 7 - start_time = "05:00" - backup_retention_settings { - retained_backups = 7 - } - } - - ip_configuration { - ipv4_enabled = true - ssl_mode = "ENCRYPTED_ONLY" - } - - maintenance_window { - day = 7 - hour = 6 - update_track = "stable" - } - - deletion_protection_enabled = true - } - - lifecycle { - prevent_destroy = true - } -} - -resource "google_sql_database" "push_dedicated" { - count = local.push_gateway_count - - project = var.project_id - name = "orca_push" - instance = google_sql_database_instance.push_dedicated[0].name - - lifecycle { - prevent_destroy = true - } -} - -resource "random_password" "push_dedicated_database" { - count = local.push_gateway_count - length = 32 - special = false -} - -resource "google_sql_user" "push_dedicated" { - count = local.push_gateway_count - - project = var.project_id - name = "orca_push" - instance = google_sql_database_instance.push_dedicated[0].name - password = random_password.push_dedicated_database[0].result -} - -resource "google_secret_manager_secret" "push_dedicated_database_url" { - count = local.push_gateway_count - - project = var.project_id - secret_id = "${var.name_prefix}-push-dedicated-database-url" - labels = local.relay_shared_labels - - replication { - auto {} - } - - lifecycle { - prevent_destroy = true - } -} - -resource "google_secret_manager_secret_version" "push_dedicated_database_url" { - count = local.push_gateway_count - - secret = google_secret_manager_secret.push_dedicated_database_url[0].id - secret_data = format( - "postgresql://%s:%s@/%s?host=/cloudsql/%s", - google_sql_user.push_dedicated[0].name, - random_password.push_dedicated_database[0].result, - google_sql_database.push_dedicated[0].name, - google_sql_database_instance.push_dedicated[0].connection_name - ) -} - -resource "google_secret_manager_secret_iam_member" "push_dedicated_database_url_accessor" { - count = local.push_gateway_count - - project = var.project_id - secret_id = google_secret_manager_secret.push_dedicated_database_url[0].secret_id - role = "roles/secretmanager.secretAccessor" - member = google_service_account.push_runtime[0].member -} diff --git a/cloud/infra/terraform/push-deploy-identity.tf b/cloud/infra/terraform/push-deploy-identity.tf deleted file mode 100644 index 057c8fb8bee..00000000000 --- a/cloud/infra/terraform/push-deploy-identity.tf +++ /dev/null @@ -1,82 +0,0 @@ -locals { - # Deployment uses its own production-only identity. - push_gateway_deploy_count = ( - var.push_gateway_enabled && local.relay_create_production_ops_identity ? 1 : 0 - ) - github_push_workflow_clauses = [ - for prefix in local.relay_github_workflow_ref_prefixes : - "assertion.workflow_ref == '${prefix}push-deploy.yml@refs/heads/main' && assertion.job_workflow_ref == '${prefix}push-deploy.yml@refs/heads/main'" - ] - push_deploy_member = one(google_service_account.github_push_deploy[*].member) -} - -resource "google_service_account" "github_push_deploy" { - count = local.push_gateway_deploy_count - account_id = "${var.name_prefix}-gha-push" - display_name = "Orca push production deploy" -} - -resource "google_iam_workload_identity_pool_provider" "github_push" { - count = local.push_gateway_deploy_count - - project = var.project_id - workload_identity_pool_id = local.relay_workload_identity_pool_id - workload_identity_pool_provider_id = "github-push-deploy" - display_name = "GitHub push production deploy" - # Do not map attribute.repository: that principal set can assume the shared deploy account. - attribute_mapping = { - "google.subject" = "assertion.sub" - "attribute.push_deploy" = "'production'" - } - attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ - "assertion.ref == 'refs/heads/main'", - "assertion.environment == 'production'", - "assertion.event_name == 'workflow_dispatch'", - local.relay_github_workflow_conditions["github_push"] - ])) - - oidc { - issuer_uri = "https://token.actions.githubusercontent.com" - } -} - -resource "google_service_account_iam_member" "github_push_workload_identity_user" { - count = local.push_gateway_deploy_count - - service_account_id = google_service_account.github_push_deploy[0].name - role = "roles/iam.workloadIdentityUser" - member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.push_deploy/production" -} - -resource "google_artifact_registry_repository_iam_member" "github_push_artifact_writer" { - count = local.push_gateway_deploy_count - - project = var.project_id - location = var.region - repository = var.artifact_repository_id - role = "roles/artifactregistry.writer" - member = local.push_deploy_member -} - -output "github_push_workload_identity_provider" { - value = try(google_iam_workload_identity_pool_provider.github_push[0].name, null) -} - -output "github_push_deploy_service_account" { - value = try(google_service_account.github_push_deploy[0].email, null) -} - - -resource "google_storage_bucket_iam_member" "github_push_rollout_lease" { - count = local.push_gateway_deploy_count - - bucket = "${var.project_id}-terraform-state" - role = "roles/storage.objectAdmin" - member = local.push_deploy_member - - condition { - title = "push_rollout_lease" - description = "Limits push deployment coordination to its own lease object." - expression = "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/push-rollout/production.lock'" - } -} diff --git a/cloud/infra/terraform/push-gateway.tf b/cloud/infra/terraform/push-gateway.tf deleted file mode 100644 index 10164d1911e..00000000000 --- a/cloud/infra/terraform/push-gateway.tf +++ /dev/null @@ -1,305 +0,0 @@ -# Orca mobile push gateway (`cloud/apps/push`). -# -# One public Cloud Run service that holds the APNs key and sends through APNs and FCM V1 on -# behalf of paired phones. Operations: `docs/push-gateway.md`. -# -# There is no staging push gateway by decision, so every resource here is behind -# `var.push_gateway_enabled`, which only `environments/production.tfvars` sets true. The file -# still reads every environment-shaped value from a variable, like the rest of this root, so a -# future staging gateway is a tfvars edit rather than a rewrite. -# -# Several resources below already exist in `onorca-cloud`; they are declared so a plan is clean -# and imported once. `docs/push-gateway.md` carries the exact `terraform import` commands. - -locals { - push_gateway_count = var.push_gateway_enabled ? 1 : 0 - - # The runtime account, the three provider secrets, and their accessor bindings already exist in - # production and were created out of band with the Apple credentials. - push_runtime_service_account_id = "${var.name_prefix}-push" - - # Secret Manager holds the Apple credentials. Terraform owns the secret names, labels, and - # replication; it never owns a version. The `.p8` is issued by the Apple developer portal and - # rotated by `docs/push-gateway.md`, so a Terraform-managed version would either put the key in - # state or fight the rotation. `ignore_changes` on the whole resource is not available, so the - # versions are simply not declared and every consumer reads `latest`. - push_provider_secret_ids = var.push_gateway_enabled ? toset([ - "${var.name_prefix}-push-apns-key", - "${var.name_prefix}-push-apns-key-id", - "${var.name_prefix}-push-apple-team-id" - ]) : toset([]) - - push_provider_secret_env = { - "${var.name_prefix}-push-apns-key" = "ORCA_PUSH_APNS_KEY" - "${var.name_prefix}-push-apns-key-id" = "ORCA_PUSH_APNS_KEY_ID" - "${var.name_prefix}-push-apple-team-id" = "ORCA_PUSH_APPLE_TEAM_ID" - } - - push_fqdn = replace(replace(var.push_base_url, "https://", ""), "http://", "") - - -} - -# --- Runtime identity --------------------------------------------------------------------- - -resource "google_service_account" "push_runtime" { - count = local.push_gateway_count - - project = var.project_id - account_id = local.push_runtime_service_account_id - display_name = "Orca mobile push gateway" - description = "Runtime identity for the Orca mobile push gateway; sends through FCM V1." -} - -# FCM V1 sends are authorized by the runtime account's own metadata-server token. -resource "google_project_iam_member" "push_runtime_fcm_admin" { - count = local.push_gateway_count - - project = var.project_id - role = "roles/firebasecloudmessaging.admin" - member = google_service_account.push_runtime[0].member -} - -# The FCM V1 endpoint bills against the caller's project quota, which the caller must consume. -resource "google_project_iam_member" "push_runtime_service_usage_consumer" { - count = local.push_gateway_count - - project = var.project_id - role = "roles/serviceusage.serviceUsageConsumer" - member = google_service_account.push_runtime[0].member -} - -resource "google_project_iam_member" "push_runtime_cloudsql_client" { - count = local.push_gateway_count - - project = var.project_id - role = "roles/cloudsql.client" - member = google_service_account.push_runtime[0].member -} - -# --- Apple credentials ---------------------------------------------------------------------- - -resource "google_secret_manager_secret" "push_provider" { - for_each = local.push_provider_secret_ids - - project = var.project_id - secret_id = each.value - labels = local.relay_shared_labels - - replication { - auto {} - } - - # Why: Apple issues a `.p8` once and Secret Manager has no undelete. Turning the gateway off - # must fail the plan rather than destroy the only copy of the signing key. - lifecycle { - prevent_destroy = true - } -} - -resource "google_secret_manager_secret_iam_member" "push_provider_runtime_accessor" { - for_each = local.push_provider_secret_ids - - project = var.project_id - secret_id = google_secret_manager_secret.push_provider[each.value].secret_id - role = "roles/secretmanager.secretAccessor" - member = google_service_account.push_runtime[0].member -} - -# --- Service -------------------------------------------------------------------------------- - -resource "google_cloud_run_v2_service" "push" { - count = local.push_gateway_count - - project = var.project_id - name = var.push_cloud_run_service_name - location = var.region - ingress = "INGRESS_TRAFFIC_ALL" - # Why: the host proof in `POST /v1/host/challenge` is the authentication, not Cloud Run IAM. - # The project's domain-restricted-sharing policy refuses an `allUsers` invoker binding, so the - # service opts out of invoker IAM exactly as the relay director does. - invoker_iam_disabled = true - deletion_protection = var.environment == "production" - labels = local.relay_shared_labels - - template { - service_account = google_service_account.push_runtime[0].email - timeout = "${var.push_request_timeout_seconds}s" - max_instance_request_concurrency = var.push_concurrency - - scaling { - min_instance_count = var.push_min_instances - max_instance_count = var.push_max_instances - } - - volumes { - name = "cloudsql" - - cloud_sql_instance { - instances = [google_sql_database_instance.push_dedicated[0].connection_name] - } - } - - containers { - image = var.push_cloud_run_image - - ports { - container_port = 8080 - } - - volume_mounts { - name = "cloudsql" - mount_path = "/cloudsql" - } - - env { - name = "ORCA_PUSH_PUBLIC_URL" - value = var.push_base_url - } - - env { - name = "ORCA_PUSH_FCM_PROJECT_ID" - value = var.project_id - } - - # Bound the declared pool against the dedicated database rollout budget. - env { - name = "ORCA_PUSH_DATABASE_POOL_MAX" - value = tostring(var.push_database_pool_max) - } - - env { - name = "ORCA_PUSH_DATABASE_URL" - - value_source { - secret_key_ref { - secret = google_secret_manager_secret.push_dedicated_database_url[0].secret_id - version = google_secret_manager_secret_version.push_dedicated_database_url[0].version - } - } - } - - # Rotation adds a new version and redeploys; `latest` is what the redeploy picks up. - dynamic "env" { - for_each = local.push_provider_secret_env - - content { - name = env.value - - value_source { - secret_key_ref { - secret = google_secret_manager_secret.push_provider[env.key].secret_id - version = "latest" - } - } - } - } - - resources { - limits = { - cpu = var.push_cloud_run_cpu - memory = var.push_cloud_run_memory - } - - cpu_idle = false - } - - startup_probe { - failure_threshold = 12 - initial_delay_seconds = 0 - period_seconds = 5 - timeout_seconds = 2 - - http_get { - path = "/health" - port = 8080 - } - } - } - } - - # Deploys update the immutable image and shift traffic; Terraform owns the shape and IAM. - # - # `traffic` is ignored as well as the image. A deploy ends with traffic pinned to an exact - # revision and a rollback pins it to the previous one; an apply that reset the service to - # 100% LATEST would silently undo either, and this root carries unrelated standing drift, so - # that apply need not be a push change at all. - lifecycle { - precondition { - condition = var.push_max_instances * var.push_database_pool_max * 3 <= 64 - error_message = "Dedicated push serving, validation/rejected and successor pools must fit the 64-connection rollout budget." - } - - ignore_changes = [ - client, - client_version, - template[0].containers[0].image, - traffic - ] - } - - depends_on = [ - data.google_artifact_registry_repository.relay_images, - google_project_iam_member.push_runtime_cloudsql_client, - google_secret_manager_secret_iam_member.push_provider_runtime_accessor, - google_secret_manager_secret_version.push_dedicated_database_url, - google_secret_manager_secret_iam_member.push_dedicated_database_url_accessor - ] -} - -# Google issues and renews the certificate for the mapping. The DNS record itself is a -# hand-managed Cloudflare CNAME to ghs.googlehosted.com, like relay.onorca.dev; this root has no -# Cloudflare surface by design. `terraform output push_dns_record` prints the record. -resource "google_cloud_run_domain_mapping" "push" { - count = var.push_gateway_enabled && var.manage_push_domain_mapping ? 1 : 0 - - location = var.region - name = local.push_fqdn - - metadata { - namespace = var.project_id - } - - spec { - route_name = google_cloud_run_v2_service.push[0].name - } - - # Same reason as relay-dns.tf: a gcloud-created mapping reports an empty legacy - # certificate_mode, and replacing it would reset issuance for no behavioral change. - lifecycle { - ignore_changes = [spec[0].certificate_mode] - } -} - -# --- Deploy identity grants ------------------------------------------------------------------- -# Push deploy authority is isolated from Relay; foundation grants its rollout-lock access. - -resource "google_cloud_run_v2_service_iam_member" "github_production_push_developer" { - count = local.push_gateway_deploy_count - - project = var.project_id - location = var.region - name = google_cloud_run_v2_service.push[0].name - role = "roles/run.developer" - member = local.push_deploy_member -} - -resource "google_service_account_iam_member" "github_production_push_runtime_user" { - count = local.push_gateway_deploy_count - - service_account_id = google_service_account.push_runtime[0].name - role = "roles/iam.serviceAccountUser" - member = local.push_deploy_member -} - -# Why: the deploy workflow's validate-only FCM send has to exercise the credential the gateway -# will actually use. Impersonating the runtime account proves its firebasecloudmessaging grant; -# granting the deploy account FCM admin outright would prove nothing about the runtime account -# and would widen a project-level role on the shared identity. -resource "google_service_account_iam_member" "github_production_push_runtime_token_creator" { - count = local.push_gateway_deploy_count - - service_account_id = google_service_account.push_runtime[0].name - role = "roles/iam.serviceAccountTokenCreator" - member = local.push_deploy_member -} diff --git a/cloud/infra/terraform/relay-asia-proof-iam.tf b/cloud/infra/terraform/relay-asia-proof-iam.tf deleted file mode 100644 index e1ff687677b..00000000000 --- a/cloud/infra/terraform/relay-asia-proof-iam.tf +++ /dev/null @@ -1,79 +0,0 @@ -locals { - create_relay_asia_proof_identity = ( - local.relay_create_github_deploy_identity && var.environment == "staging" - ) - github_relay_asia_proof_workflow_file = "prove-relay-asia-staging.yml" - github_relay_asia_proof_workflow_clauses = [ - for prefix in local.relay_github_workflow_ref_prefixes : - "assertion.workflow_ref == '${prefix}${local.github_relay_asia_proof_workflow_file}@refs/heads/main'" - ] - relay_asia_proof_service_account_email = try( - google_service_account.github_relay_asia_proof[0].email, - "" - ) -} - -resource "google_iam_workload_identity_pool_provider" "github_relay_asia_proof" { - count = local.create_relay_asia_proof_identity ? 1 : 0 - - project = var.project_id - workload_identity_pool_id = local.relay_workload_identity_pool_id - workload_identity_pool_provider_id = "github-relay-asia-proof" - display_name = "GitHub Relay Asia staging proof" - - attribute_mapping = { - "google.subject" = "assertion.sub" - "attribute.environment" = "assertion.environment" - "attribute.event_name" = "assertion.event_name" - "attribute.ref" = "assertion.ref" - "attribute.repository" = "assertion.repository" - "attribute.repository_id" = "assertion.repository_id" - "attribute.repository_owner_id" = "assertion.repository_owner_id" - "attribute.workflow_ref" = "assertion.workflow_ref" - "attribute.relay_ops_identity" = "'staging-asia-proof'" - } - - attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ - "assertion.ref == 'refs/heads/main'", - "assertion.environment == 'staging'", - "assertion.event_name == 'workflow_dispatch'", - local.relay_github_workflow_conditions["github_relay_asia_proof"] - ])) - - oidc { - issuer_uri = "https://token.actions.githubusercontent.com" - } -} - -resource "google_service_account" "github_relay_asia_proof" { - count = local.create_relay_asia_proof_identity ? 1 : 0 - - project = var.project_id - account_id = "${var.name_prefix}-gha-aproof" - display_name = "Orca Relay Asia staging proof" - description = "Reads staging telemetry and performs only Relay's bounded Asia proof operations." -} - -resource "google_service_account_iam_member" "github_relay_asia_proof_workload_identity_user" { - count = local.create_relay_asia_proof_identity ? 1 : 0 - - service_account_id = google_service_account.github_relay_asia_proof[0].name - role = "roles/iam.workloadIdentityUser" - member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/staging-asia-proof" -} - -resource "google_project_iam_member" "github_relay_asia_proof_logging_viewer" { - count = local.create_relay_asia_proof_identity ? 1 : 0 - - project = var.project_id - role = "roles/logging.viewer" - member = google_service_account.github_relay_asia_proof[0].member -} - -resource "google_project_iam_member" "github_relay_asia_proof_monitoring_viewer" { - count = local.create_relay_asia_proof_identity ? 1 : 0 - - project = var.project_id - role = "roles/monitoring.viewer" - member = google_service_account.github_relay_asia_proof[0].member -} diff --git a/cloud/infra/terraform/relay-asia-topology-iam.tf b/cloud/infra/terraform/relay-asia-topology-iam.tf deleted file mode 100644 index eea36e26247..00000000000 --- a/cloud/infra/terraform/relay-asia-topology-iam.tf +++ /dev/null @@ -1,207 +0,0 @@ -locals { - create_relay_asia_topology_identity = local.relay_create_github_deploy_identity - github_relay_asia_topology_workflow_file = "deploy-relay-asia-topology.yml" - github_relay_asia_topology_workflow_clauses = [ - for prefix in local.relay_github_workflow_ref_prefixes : - "assertion.workflow_ref == '${prefix}${local.github_relay_asia_topology_workflow_file}@refs/heads/main'" - ] -} - -resource "google_iam_workload_identity_pool_provider" "github_relay_asia_topology" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - project = var.project_id - workload_identity_pool_id = local.relay_workload_identity_pool_id - workload_identity_pool_provider_id = "github-relay-asia" - display_name = "GitHub Relay Asia topology" - - attribute_mapping = { - "google.subject" = "assertion.sub" - "attribute.environment" = "assertion.environment" - "attribute.event_name" = "assertion.event_name" - "attribute.ref" = "assertion.ref" - "attribute.repository" = "assertion.repository" - "attribute.repository_id" = "assertion.repository_id" - "attribute.repository_owner_id" = "assertion.repository_owner_id" - "attribute.workflow_ref" = "assertion.workflow_ref" - "attribute.relay_ops_identity" = "'${var.environment}-asia-topology'" - } - - attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ - "assertion.ref == 'refs/heads/main'", - "assertion.environment == '${var.environment}'", - "assertion.event_name == 'workflow_dispatch'", - local.relay_github_workflow_conditions["github_relay_asia_topology"] - ])) - - oidc { - issuer_uri = "https://token.actions.githubusercontent.com" - } -} - -resource "google_service_account" "github_relay_asia_topology" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - project = var.project_id - account_id = "${var.name_prefix}-gha-asia" - display_name = "Orca Relay Asia topology" - description = "Applies only validated additive Relay Asia topology plans." -} - -resource "google_service_account_iam_member" "github_relay_asia_topology_workload_identity_user" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - service_account_id = google_service_account.github_relay_asia_topology[0].name - role = "roles/iam.workloadIdentityUser" - member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/${var.environment}-asia-topology" -} - -resource "google_project_iam_custom_role" "github_relay_asia_topology_mutation" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - project = var.project_id - role_id = "orcaRelayAsiaTopology" - title = "Orca Relay Asia topology" - description = "Creates additive Relay Asia network and cell topology and updates its shared URL map." - permissions = [ - "compute.backendServices.create", - "compute.backendServices.get", - "compute.backendServices.update", - "compute.backendServices.use", - "compute.disks.create", - "compute.globalOperations.get", - "compute.healthChecks.use", - "compute.healthChecks.useReadOnly", - "compute.images.useReadOnly", - "compute.instanceGroupManagers.create", - "compute.instanceGroupManagers.get", - "compute.instanceGroupManagers.update", - "compute.instanceGroups.create", - "compute.instanceGroups.get", - "compute.instanceGroups.use", - "compute.instances.create", - "compute.instances.setLabels", - "compute.instances.setMetadata", - "compute.instances.setTags", - "compute.instances.use", - "compute.instanceTemplates.create", - "compute.instanceTemplates.get", - "compute.instanceTemplates.useReadOnly", - "compute.networks.get", - "compute.networks.updatePolicy", - "compute.networks.use", - "compute.regionOperations.get", - "compute.routers.create", - "compute.routers.get", - "compute.routers.update", - "compute.subnetworks.create", - "compute.subnetworks.get", - "compute.subnetworks.setPrivateIpGoogleAccess", - "compute.subnetworks.use", - "compute.urlMaps.get", - "compute.urlMaps.update", - "compute.zoneOperations.get" - ] -} - -resource "google_project_iam_member" "github_relay_asia_topology_mutation" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - project = var.project_id - role = google_project_iam_custom_role.github_relay_asia_topology_mutation[0].id - member = google_service_account.github_relay_asia_topology[0].member -} - -resource "google_project_iam_custom_role" "github_relay_asia_topology_read" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - project = var.project_id - role_id = "orcaRelayAsiaTopologyRead" - title = "Orca Relay Asia topology read" - description = "Refreshes only resource types required by validated Relay Asia topology plans." - permissions = [ - "artifactregistry.repositories.get", - "cloudsql.instances.get", - "compute.backendServices.get", - "compute.healthChecks.get", - "compute.instanceGroupManagers.get", - "compute.instanceGroups.get", - "compute.instanceTemplates.get", - "compute.instances.get", - "compute.networks.get", - "compute.routers.get", - "compute.subnetworks.get", - "compute.urlMaps.get", - "iam.serviceAccounts.get", - "iam.serviceAccounts.getIamPolicy", - "resourcemanager.projects.get", - "resourcemanager.projects.getIamPolicy", - "run.revisions.get", - "run.services.get", - "secretmanager.secrets.get", - "secretmanager.secrets.getIamPolicy", - "serviceusage.services.get", - "serviceusage.services.list" - ] -} - -resource "google_project_iam_member" "github_relay_asia_topology_read" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - project = var.project_id - role = google_project_iam_custom_role.github_relay_asia_topology_read[0].id - member = google_service_account.github_relay_asia_topology[0].member -} - -resource "google_artifact_registry_repository_iam_member" "github_relay_asia_topology_artifact_reader" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - project = var.project_id - location = var.region - repository = var.artifact_repository_id - role = "roles/artifactregistry.reader" - member = google_service_account.github_relay_asia_topology[0].member -} - -resource "google_storage_bucket_iam_member" "github_relay_asia_topology_state" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - bucket = "${var.project_id}-terraform-state" - role = "roles/storage.objectAdmin" - member = google_service_account.github_relay_asia_topology[0].member - - condition { - title = "relay_asia_topology_state" - description = "Limits the Asia topology workflow to the environment Terraform state and lock." - expression = join(" || ", [ - "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tfstate'", - "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tflock'" - ]) - } -} - -resource "google_project_iam_custom_role" "github_relay_asia_topology_state_list" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - project = var.project_id - role_id = "orcaRelayAsiaStateList" - title = "Orca Relay Asia state list" - description = "Lists the environment state bucket so Terraform can initialize its backend." - permissions = ["storage.objects.list"] -} - -resource "google_storage_bucket_iam_member" "github_relay_asia_topology_state_list" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - bucket = "${var.project_id}-terraform-state" - role = google_project_iam_custom_role.github_relay_asia_topology_state_list[0].id - member = google_service_account.github_relay_asia_topology[0].member -} - -resource "google_service_account_iam_member" "github_relay_asia_topology_runtime_user" { - count = local.create_relay_asia_topology_identity ? 1 : 0 - - service_account_id = google_service_account.relay_runtime.name - role = "roles/iam.serviceAccountUser" - member = google_service_account.github_relay_asia_topology[0].member -} diff --git a/cloud/infra/terraform/relay-control-bucket.tf b/cloud/infra/terraform/relay-control-bucket.tf deleted file mode 100644 index 1d29095e8c1..00000000000 --- a/cloud/infra/terraform/relay-control-bucket.tf +++ /dev/null @@ -1,77 +0,0 @@ -# Runtime switch files: cells/.json read by each cell, written only by the one-cell -# flag workflow. The cell image derives the name from -# the metadata server's project id, so it must stay "-relay-control". -# -# Apply only with the targeted command in the PR that added this file; a root plan rolls the -# fleet. Nothing here touches a template, MIG or service. -resource "google_storage_bucket" "relay_control" { - project = var.project_id - name = "${var.project_id}-relay-control" - location = var.region - - uniform_bucket_level_access = true - public_access_prevention = "enforced" - force_destroy = false - - # Every write is a new generation, so the object history is the audit trail. - versioning { - enabled = true - } - - # The writer needs delete to overwrite, and delete also reaches old generations. Soft delete - # keeps any deleted generation recoverable for 90 days (the GCS maximum), so a mistaken or - # hostile delete cannot erase the trail inside that window. - soft_delete_policy { - retention_duration_seconds = 7776000 - } - - lifecycle_rule { - condition { - days_since_noncurrent_time = 365 - } - action { - type = "Delete" - } - } - - labels = { - environment = var.environment - service = "relay" - } -} - -# Cells share one runtime account, so IAM cannot scope a cell to its own object; the image -# reads only cells/.json, and read access grants no control. -resource "google_storage_bucket_iam_member" "relay_control_cell_reader" { - bucket = google_storage_bucket.relay_control.name - role = "roles/storage.objectViewer" - member = google_service_account.relay_runtime.member -} - -# Overwrite and nothing more: no list, no ACL or bucket permissions. GCS needs delete to -# replace a live object even with versioning on. -resource "google_project_iam_custom_role" "relay_control_writer" { - count = local.relay_create_github_deploy_identity ? 1 : 0 - - project = var.project_id - role_id = "orcaRelayControlWriter" - title = "Orca Relay control writer" - description = "Reads and overwrites relay runtime switch objects." - permissions = ["storage.objects.create", "storage.objects.delete", "storage.objects.get"] -} - -# The flag workflow's identity, the only writer. The deploy account is shared with every relay -# deploy workflow, so the grant reaches only the per-cell objects. -resource "google_storage_bucket_iam_member" "relay_control_workflow_writer" { - count = local.relay_create_github_deploy_identity ? 1 : 0 - - bucket = google_storage_bucket.relay_control.name - role = google_project_iam_custom_role.relay_control_writer[0].id - member = local.relay_github_deploy_service_account_member - - condition { - title = "relay_control_cell_objects" - description = "Limits the deploy identity to cells/.json." - expression = "resource.name.startsWith('projects/_/buckets/${google_storage_bucket.relay_control.name}/objects/cells/')" - } -} diff --git a/cloud/infra/terraform/relay-database.tf b/cloud/infra/terraform/relay-database.tf deleted file mode 100644 index 6e3e0f1a527..00000000000 --- a/cloud/infra/terraform/relay-database.tf +++ /dev/null @@ -1,88 +0,0 @@ -# Relay credentials and assignment state share the existing Cloud SQL instance -# with auth, but use an isolated database and principal. -resource "google_sql_database" "relay" { - project = var.project_id - name = "orca_relay" - instance = local.relay_database_instance_name -} - -resource "random_password" "relay_database" { - length = 32 - special = false -} - -resource "google_sql_user" "relay" { - project = var.project_id - name = "orca_relay" - instance = local.relay_database_instance_name - password = random_password.relay_database.result -} - -resource "google_secret_manager_secret" "relay_database_url" { - project = var.project_id - secret_id = "orca-cloud-relay-database-url" - labels = local.relay_shared_labels - - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "relay_database_url" { - secret = google_secret_manager_secret.relay_database_url.id - secret_data = format( - "postgresql://%s:%s@/%s?host=/cloudsql/%s", - google_sql_user.relay.name, - random_password.relay_database.result, - google_sql_database.relay.name, - local.relay_database_connection_name - ) -} - -# The director's URL when it reaches Cloud SQL through its proxy sidecar. Cells keep the socket URL. -resource "google_secret_manager_secret" "relay_director_database_url" { - count = var.relay_director_cloud_sql_proxy_sidecar ? 1 : 0 - - project = var.project_id - secret_id = "orca-cloud-relay-director-database-url" - labels = local.relay_shared_labels - - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "relay_director_database_url" { - count = var.relay_director_cloud_sql_proxy_sidecar ? 1 : 0 - - secret = google_secret_manager_secret.relay_director_database_url[0].id - secret_data = format( - "postgresql://%s:%s@127.0.0.1:5432/%s", - google_sql_user.relay.name, - random_password.relay_database.result, - google_sql_database.relay.name - ) -} - -resource "google_secret_manager_secret_iam_member" "relay_director_database_url_accessor" { - count = var.relay_director_cloud_sql_proxy_sidecar ? 1 : 0 - - project = var.project_id - secret_id = google_secret_manager_secret.relay_director_database_url[0].secret_id - role = "roles/secretmanager.secretAccessor" - member = google_service_account.relay_director_runtime.member -} - -resource "google_secret_manager_secret_iam_member" "relay_database_url_accessor" { - project = var.project_id - secret_id = google_secret_manager_secret.relay_database_url.secret_id - role = "roles/secretmanager.secretAccessor" - member = google_service_account.relay_runtime.member -} - -resource "google_secret_manager_secret_iam_member" "relay_database_url_director_accessor" { - project = var.project_id - secret_id = google_secret_manager_secret.relay_database_url.secret_id - role = "roles/secretmanager.secretAccessor" - member = google_service_account.relay_director_runtime.member -} diff --git a/cloud/infra/terraform/relay-dns.tf b/cloud/infra/terraform/relay-dns.tf deleted file mode 100644 index 6e1895da925..00000000000 --- a/cloud/infra/terraform/relay-dns.tf +++ /dev/null @@ -1,47 +0,0 @@ -# Relay custom domains. The Cloud Run domain mapping is the whole story here: Google issues and -# renews the certificate, and the DNS records that point at ghs.googlehosted.com are managed -# outside this root (the auth and artifact records moved to the apps root with their services). - -locals { - relay_fqdn = replace(replace(var.relay_base_url, "https://", ""), "http://", "") - relay_cell_fqdns = { - for cell_id, cell in var.relay_cells : - cell_id => replace(replace(cell.url, "https://", ""), "http://", "") - } -} - -resource "google_cloud_run_domain_mapping" "relay" { - count = var.manage_relay_domain_mapping ? 1 : 0 - location = var.region - name = local.relay_fqdn - - metadata { - namespace = var.project_id - } - - spec { - route_name = google_cloud_run_v2_service.relay.name - } - - # gcloud-created mappings report an empty legacy certificate_mode even - # though Google provisions the same automatic certificate; replacing it - # would reset issuance for no behavioral change. - lifecycle { - ignore_changes = [spec[0].certificate_mode] - } -} - -resource "google_cloud_run_domain_mapping" "relay_cell" { - for_each = var.manage_relay_domain_mapping ? var.relay_cells : {} - - location = var.region - name = local.relay_cell_fqdns[each.key] - - metadata { - namespace = var.project_id - } - - spec { - route_name = google_cloud_run_v2_service.relay_cell[each.key].name - } -} diff --git a/cloud/infra/terraform/relay-fence-broker.tf b/cloud/infra/terraform/relay-fence-broker.tf deleted file mode 100644 index 0bcff95af9a..00000000000 --- a/cloud/infra/terraform/relay-fence-broker.tf +++ /dev/null @@ -1,243 +0,0 @@ -locals { - create_relay_fence_broker = local.relay_create_production_ops_identity - relay_fence_state_bucket = "${var.project_id}-terraform-state" - relay_fence_state_prefix = "projects/_/buckets/${local.relay_fence_state_bucket}/objects/terraform/state" -} - -resource "google_service_account" "relay_fence_broker" { - count = local.create_relay_fence_broker ? 1 : 0 - - project = var.project_id - account_id = "${var.name_prefix}-relay-fence" - display_name = "Orca Relay fence broker" - description = "Owns exact reviewed Terraform cell fences behind an authenticated broker." -} - -resource "google_project_iam_custom_role" "relay_fence_broker_mutation" { - count = local.create_relay_fence_broker ? 1 : 0 - - project = var.project_id - role_id = "orcaRelayFenceBroker" - title = "Orca Relay fence broker" - description = "Updates only reviewed Relay MIG sizes and inspects their zone operations." - permissions = [ - "compute.instanceGroupManagers.update" - ] -} - -resource "google_project_iam_member" "relay_fence_broker_mutation" { - count = local.create_relay_fence_broker ? 1 : 0 - - project = var.project_id - role = google_project_iam_custom_role.relay_fence_broker_mutation[0].id - member = google_service_account.relay_fence_broker[0].member -} - -resource "google_project_iam_member" "relay_fence_broker_compute_viewer" { - count = local.create_relay_fence_broker ? 1 : 0 - - project = var.project_id - role = "roles/compute.viewer" - member = google_service_account.relay_fence_broker[0].member -} - -resource "google_project_iam_member" "relay_fence_broker_logging_viewer" { - count = local.create_relay_fence_broker ? 1 : 0 - - project = var.project_id - role = "roles/logging.viewer" - member = google_service_account.relay_fence_broker[0].member -} - -resource "google_project_iam_member" "relay_fence_broker_artifact_reader" { - count = local.create_relay_fence_broker ? 1 : 0 - - project = var.project_id - role = "roles/artifactregistry.reader" - member = google_service_account.relay_fence_broker[0].member -} - -resource "google_storage_bucket_iam_member" "relay_fence_broker_bucket_reader" { - count = local.create_relay_fence_broker ? 1 : 0 - - bucket = local.relay_fence_state_bucket - role = "roles/storage.legacyBucketReader" - member = google_service_account.relay_fence_broker[0].member -} - -resource "google_storage_bucket_iam_member" "relay_fence_broker_state_objects" { - count = local.create_relay_fence_broker ? 1 : 0 - - bucket = local.relay_fence_state_bucket - role = "roles/storage.objectAdmin" - member = google_service_account.relay_fence_broker[0].member - - condition { - title = "relay_fence_exact_objects" - description = "Main state, private saved plans, and the durable broker lease only." - expression = join(" || ", [ - "resource.name.startsWith('${local.relay_fence_state_prefix}/default')", - "resource.name.startsWith('${local.relay_fence_state_prefix}/relay-fence-plans/production/')", - "resource.name.startsWith('${local.relay_fence_state_prefix}/relay-fence-broker/')" - ]) - } -} - -resource "google_service_account_iam_member" "relay_fence_broker_requester_token_creator" { - count = local.create_relay_fence_broker ? 1 : 0 - - service_account_id = google_service_account.github_fence[0].name - role = "roles/iam.serviceAccountTokenCreator" - member = google_service_account.relay_fence_broker[0].member -} - -resource "google_service_account_iam_member" "github_relay_fence_broker_service_account_user" { - count = local.create_relay_fence_broker ? 1 : 0 - - service_account_id = google_service_account.relay_fence_broker[0].name - role = "roles/iam.serviceAccountUser" - member = local.relay_github_deploy_service_account_member -} - -resource "google_cloud_run_v2_service" "relay_fence_broker" { - count = local.create_relay_fence_broker ? 1 : 0 - - project = var.project_id - name = var.relay_fence_broker_service_name - location = var.region - ingress = "INGRESS_TRAFFIC_ALL" - deletion_protection = var.environment == "production" - labels = local.relay_shared_labels - - template { - service_account = google_service_account.relay_fence_broker[0].email - timeout = "1800s" - max_instance_request_concurrency = 1 - - scaling { - min_instance_count = 0 - max_instance_count = 1 - } - - containers { - image = var.relay_fence_broker_image - - ports { - container_port = 8080 - } - - env { - name = "ORCA_RELAY_FENCE_PROJECT" - value = var.project_id - } - - env { - name = "ORCA_RELAY_FENCE_STATE_BUCKET" - value = local.relay_fence_state_bucket - } - - env { - name = "ORCA_RELAY_FENCE_LEASE_OBJECT" - value = "terraform/state/relay-fence-broker/production.lock" - } - - env { - name = "ORCA_RELAY_FENCE_DIRECTOR_ORIGIN" - value = var.relay_base_url - } - - env { - name = "ORCA_RELAY_FENCE_ADMIN_AUDIENCE" - value = "${var.relay_base_url}/v1/admin/drain" - } - - env { - name = "ORCA_RELAY_FENCE_REQUESTER_SERVICE_ACCOUNT" - value = google_service_account.github_fence[0].email - } - - env { - name = "ORCA_RELAY_FENCE_RUNTIME_SERVICE_ACCOUNT" - value = google_service_account.relay_runtime.email - } - - env { - name = "ORCA_RELAY_FENCE_SOURCE_CELL_ID" - value = var.relay_fence_source_cell_id - } - - env { - name = "ORCA_RELAY_FENCE_FAILED_TARGET_CELL_ID" - value = var.relay_fence_failed_target_cell_id - } - - env { - name = "ORCA_RELAY_FENCE_REPLACEMENT_TARGET_CELL_ID" - value = var.relay_fence_replacement_target_cell_id - } - - env { - name = "ORCA_RELAY_FENCE_UNOBSERVED_CONNECTION_BOUND" - value = tostring(var.relay_fence_unobserved_connection_bound) - } - - resources { - limits = { - cpu = "1" - memory = "1Gi" - } - - cpu_idle = false - } - - startup_probe { - failure_threshold = 12 - initial_delay_seconds = 0 - period_seconds = 5 - timeout_seconds = 2 - - http_get { - path = "/healthz" - port = 8080 - } - } - } - } - - lifecycle { - ignore_changes = [ - client, - client_version, - template[0].containers[0].image - ] - } - - depends_on = [ - google_project_iam_member.relay_fence_broker_artifact_reader, - google_project_iam_member.relay_fence_broker_compute_viewer, - google_project_iam_member.relay_fence_broker_logging_viewer, - google_project_iam_member.relay_fence_broker_mutation, - google_storage_bucket_iam_member.relay_fence_broker_bucket_reader, - google_storage_bucket_iam_member.relay_fence_broker_state_objects - ] -} - -resource "google_cloud_run_v2_service_iam_member" "relay_fence_broker_invoker" { - count = local.create_relay_fence_broker ? 1 : 0 - - project = var.project_id - location = var.region - name = google_cloud_run_v2_service.relay_fence_broker[0].name - role = "roles/run.invoker" - member = google_service_account.github_fence[0].member -} - -resource "google_cloud_run_v2_service_iam_member" "relay_fence_broker_deploy_invoker" { - count = local.create_relay_fence_broker ? 1 : 0 - - project = var.project_id - location = var.region - name = google_cloud_run_v2_service.relay_fence_broker[0].name - role = "roles/run.invoker" - member = local.relay_github_deploy_service_account_member -} diff --git a/cloud/infra/terraform/relay-gce-cells.tf b/cloud/infra/terraform/relay-gce-cells.tf deleted file mode 100644 index 13a8c8ce116..00000000000 --- a/cloud/infra/terraform/relay-gce-cells.tf +++ /dev/null @@ -1,401 +0,0 @@ -locals { - relay_runtime_service_account_email = "${var.name_prefix}-relay@${var.project_id}.iam.gserviceaccount.com" - relay_director_runtime_service_account_email = "${var.name_prefix}-relay-dir@${var.project_id}.iam.gserviceaccount.com" - relay_capacity_service_account_email = var.environment == "production" ? try( - google_service_account.github_production_relay_capacity[0].email, - "" - ) : try(google_service_account.github_staging_relay_capacity[0].email, "") - relay_gce_cells_enabled = local.relay_gce_configured && length(var.relay_gce_cells) > 0 - relay_gce_subnetworks = merge( - { (var.region) = try(google_compute_subnetwork.relay_gce[0].id, null) }, - { for region, subnet in google_compute_subnetwork.relay_gce_additional : region => subnet.id } - ) - relay_gce_topology = { - max_surge = 0 - max_unavailable = 1 - backend_group_count = 1 - public_access_config_count = 0 - backend_timeout_seconds = 86400 - connection_drain_seconds = 60 - } - relay_gce_cell_urls = { - for cell_id, cell in var.relay_gce_cells : - cell_id => "https://${cell.hostname}.${var.relay_gce_domain}" - } - relay_gce_cell_target_sizes = { - for cell_id in keys(var.relay_gce_cells) : - cell_id => contains(var.relay_gce_fenced_cells, cell_id) ? 0 : 1 - } - relay_director_cells = local.relay_gce_cells_enabled ? { - for cell_id, cell in var.relay_gce_cells : cell_id => { - url = local.relay_gce_cell_urls[cell_id] - region = cell.region - capacity_requests = cell.capacity_requests - initially_enabled = cell.initially_enabled - connection_hard_cap = cell.connection_hard_cap - connection_unobserved_bound = cell.connection_unobserved_bound - } - } : { - for cell_id, cell in var.relay_cells : cell_id => { - url = cell.url - region = "us-central1" - capacity_requests = cell.capacity_requests - initially_enabled = true - connection_hard_cap = null - connection_unobserved_bound = null - } - } - relay_director_cells_json = jsonencode([ - for cell_id, cell in local.relay_director_cells : merge( - { - id = cell_id - url = cell.url - region = cell.region - capacityRequests = cell.capacity_requests - initiallyEnabled = cell.initially_enabled - }, - try(cell.connection_hard_cap, null) == null ? {} : { - connectionHardCap = cell.connection_hard_cap - connectionUnobservedBound = cell.connection_unobserved_bound - } - ) - ]) -} - -check "relay_gce_fixed_one_topology" { - assert { - condition = alltrue([ - for region in keys(var.relay_gce_additional_region_subnetwork_cidrs) : region != var.region - ]) && alltrue([ - for cell in values(var.relay_gce_cells) : - cell.region == var.region || contains(keys(var.relay_gce_additional_region_subnetwork_cidrs), cell.region) - ]) - error_message = "Additional Relay regions must differ from the primary region, and every cell region needs a configured subnetwork." - } - - assert { - condition = alltrue([ - for cell_id in var.relay_gce_fenced_cells : contains(keys(var.relay_gce_cells), cell_id) - ]) - error_message = "relay_gce_fenced_cells may contain only configured relay_gce_cells keys." - } - - assert { - condition = alltrue([ - # Region is not asserted here: the director's own rehome source and target predicates - # own eligibility, so this pins only cell shape. - for cell_id in var.relay_region_rehome_source_cell_ids : try( - var.relay_gce_cells[cell_id].connection_hard_cap != null && - !contains(var.relay_gce_fenced_cells, cell_id), - false - ) - ]) - error_message = "Regional rehome sources must be configured, unfenced GCE cells with explicit connection limits." - } - - assert { - condition = length(var.relay_gce_cells) == 0 || var.relay_gce_domain != "" - error_message = "relay_gce_domain is required when GCE cells are configured." - } - - assert { - condition = ( - alltrue([ - for cell_id, target_size in local.relay_gce_cell_target_sizes : - contains(var.relay_gce_fenced_cells, cell_id) ? target_size == 0 : target_size == 1 - ]) && - local.relay_gce_topology.max_surge == 0 && - local.relay_gce_topology.max_unavailable == 1 && - local.relay_gce_topology.backend_group_count == 1 && - local.relay_gce_topology.public_access_config_count == 0 && - local.relay_gce_topology.backend_timeout_seconds == 86400 && - # A rollout drains every host off the cell before Terraform runs, so this covers only a - # host still mid-handshake; pinned so a raise cannot re-add rollout wall clock unseen. - local.relay_gce_topology.connection_drain_seconds == 60 - ) - error_message = "Relay cells require fixed-one RECREATE MIGs, one non-public backend, the 86,400-second WebSocket timeout, and the 60-second connection drain." - } -} - -resource "google_compute_health_check" "relay_gce_liveness" { - count = local.relay_gce_cells_enabled ? 1 : 0 - - project = var.project_id - name = "${local.relay_gce_name}-health" - check_interval_sec = 10 - timeout_sec = 5 - healthy_threshold = 2 - unhealthy_threshold = 3 - - http_health_check { - port = 8080 - request_path = "/health" - } - - log_config { - enable = true - } - - # A project's first Compute API enablement can return before health-check - # creation is accepted, so keep this independent root behind the service. -} - -resource "google_compute_health_check" "relay_gce_readiness" { - count = local.relay_gce_cells_enabled ? 1 : 0 - - project = var.project_id - name = "${local.relay_gce_name}-ready" - check_interval_sec = 10 - timeout_sec = 5 - healthy_threshold = 2 - unhealthy_threshold = 2 - - http_health_check { - port = 8080 - request_path = "/ready" - } - - log_config { - enable = true - } - - # This check has no other Compute dependency to serialize initial API use. -} - -resource "google_compute_instance_template" "relay_gce_cell" { - for_each = var.relay_gce_cells - - project = var.project_id - name_prefix = "${substr("${local.relay_gce_name}-${each.value.hostname}", 0, 52)}-" - machine_type = each.value.machine_type - can_ip_forward = false - tags = ["orca-relay-cell"] - labels = merge( - local.relay_shared_labels, - { - orca-relay-role = "cell" - orca-relay-cell = each.key - }, - each.value.region == var.region ? {} : { orca-relay-region = each.value.region } - ) - - disk { - auto_delete = true - boot = true - device_name = "persistent-disk-0" - disk_size_gb = each.value.boot_disk_gb - disk_type = "pd-balanced" - type = "PERSISTENT" - mode = "READ_WRITE" - interface = "SCSI" - source_image = each.value.boot_image - } - - network_interface { - subnetwork = local.relay_gce_subnetworks[each.value.region] - - # An empty dynamic block makes the no-public-IP invariant machine-checkable. - dynamic "access_config" { - for_each = range(local.relay_gce_topology.public_access_config_count) - content {} - } - } - - service_account { - email = local.relay_runtime_service_account_email - scopes = ["cloud-platform"] - } - - scheduling { - automatic_restart = true - on_host_maintenance = "MIGRATE" - provisioning_model = "STANDARD" - } - - shielded_instance_config { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } - - metadata = { - block-project-ssh-keys = "TRUE" - enable-oslogin = "TRUE" - google-logging-enabled = "TRUE" - } - - metadata_startup_script = templatefile("${path.module}/relay-gce-startup.sh.tftpl", { - project_id = var.project_id - database_secret = google_secret_manager_secret.relay_database_url.secret_id - assignment_secret = google_secret_manager_secret.relay_assignment_signing_key.secret_id - cell_id = each.key - cell_region = each.value.region - include_cell_region = each.value.region != var.region - cell_url = local.relay_gce_cell_urls[each.key] - capacity_requests = each.value.capacity_requests - database_pool_max = each.value.database_pool_max - include_database_pool_max = each.value.region != var.region || each.value.database_pool_max != 10 - connection_hard_cap = each.value.connection_hard_cap - connection_unobserved_bound = each.value.connection_unobserved_bound - auth_issuer = var.auth_base_url - director_url = var.relay_base_url - deploy_service_account = local.relay_github_deploy_service_account_email - capacity_service_account = local.relay_capacity_service_account_email - asia_proof_service_account = local.relay_asia_proof_service_account_email - runtime_service_account = local.relay_runtime_service_account_email - rehome_source_enabled = contains(var.relay_region_rehome_source_cell_ids, each.key) - rehome_director_service_account = local.relay_director_runtime_service_account_email - rehome_audience = "${var.relay_base_url}/v1/admin/host-drain" - artifact_registry_host = "${var.region}-docker.pkg.dev" - relay_image = each.value.image - cloud_sql_proxy_image = var.relay_gce_cloud_sql_proxy_image - cloud_sql_private_ip = var.relay_cloud_sql_private_ip - # Keep cell-only plans independent from unrelated database configuration drift. - cloud_sql_connection_name = local.relay_database_connection_name - }) - - lifecycle { - create_before_destroy = true - } - - depends_on = [ - google_project_iam_member.relay_runtime_artifact_reader, - google_project_iam_member.relay_runtime_cloudsql_client, - google_project_iam_member.relay_runtime_log_writer, - google_secret_manager_secret_iam_member.relay_assignment_signing_key_accessor, - google_secret_manager_secret_iam_member.relay_database_url_accessor - ] -} - -resource "google_compute_instance_group_manager" "relay_gce_cell" { - for_each = var.relay_gce_cells - - project = var.project_id - name = "${local.relay_gce_name}-${each.value.hostname}" - zone = each.value.zone - base_instance_name = "relay-${each.value.hostname}" - target_size = local.relay_gce_cell_target_sizes[each.key] - - version { - name = "primary" - instance_template = google_compute_instance_template.relay_gce_cell[each.key].self_link - } - - named_port { - name = "relay" - port = 8080 - } - - auto_healing_policies { - health_check = google_compute_health_check.relay_gce_liveness[0].id - initial_delay_sec = 180 - } - - update_policy { - type = "PROACTIVE" - minimal_action = "REPLACE" - most_disruptive_allowed_action = "REPLACE" - replacement_method = "RECREATE" - max_surge_fixed = local.relay_gce_topology.max_surge - max_unavailable_fixed = local.relay_gce_topology.max_unavailable - } - - lifecycle { - precondition { - condition = contains([0, 1], local.relay_gce_cell_target_sizes[each.key]) - error_message = "A relay cell MIG must be fenced at zero or active at exactly one." - } - } -} - -resource "google_compute_backend_service" "relay_gce_cell" { - for_each = var.relay_gce_cells - - project = var.project_id - name = "${local.relay_gce_name}-${each.value.hostname}" - protocol = "HTTP" - port_name = "relay" - load_balancing_scheme = "EXTERNAL_MANAGED" - timeout_sec = local.relay_gce_topology.backend_timeout_seconds - connection_draining_timeout_sec = local.relay_gce_topology.connection_drain_seconds - health_checks = [google_compute_health_check.relay_gce_readiness[0].id] - session_affinity = "NONE" - - backend { - group = google_compute_instance_group_manager.relay_gce_cell[each.key].instance_group - balancing_mode = "UTILIZATION" - max_utilization = 0.8 - capacity_scaler = 1 - } - - # Per-connection client IP/status/latency for the data plane; a WebSocket logs once, at close. - log_config { - enable = true - sample_rate = var.relay_gce_cell_log_sample_rate - } - - lifecycle { - precondition { - condition = local.relay_gce_topology.backend_group_count == 1 - error_message = "Each exact relay host must route to one non-overlapping fixed-one MIG." - } - } -} - -resource "google_compute_url_map" "relay_gce" { - count = local.relay_gce_cells_enabled ? 1 : 0 - - project = var.project_id - name = local.relay_gce_name - default_service = google_compute_backend_service.relay_gce_cell[sort(keys(var.relay_gce_cells))[0]].id - - # Unknown wildcard hosts fail at the LB and can never fall through to a cell. - default_route_action { - fault_injection_policy { - abort { - http_status = 404 - percentage = 100 - } - } - } - - dynamic "host_rule" { - for_each = var.relay_gce_cells - iterator = cell - content { - hosts = ["${cell.value.hostname}.${var.relay_gce_domain}"] - path_matcher = "cell-${cell.value.hostname}" - } - } - - dynamic "path_matcher" { - for_each = var.relay_gce_cells - iterator = cell - content { - name = "cell-${cell.value.hostname}" - default_service = google_compute_backend_service.relay_gce_cell[cell.key].id - } - } -} - -resource "google_compute_target_https_proxy" "relay_gce" { - count = local.relay_gce_cells_enabled ? 1 : 0 - - project = var.project_id - name = local.relay_gce_name - url_map = google_compute_url_map.relay_gce[0].id - certificate_map = "//certificatemanager.googleapis.com/${google_certificate_manager_certificate_map.relay_gce[0].id}" - quic_override = "NONE" -} - -resource "google_compute_global_forwarding_rule" "relay_gce" { - count = local.relay_gce_cells_enabled ? 1 : 0 - - project = var.project_id - name = local.relay_gce_name - ip_address = google_compute_global_address.relay_gce[0].id - port_range = "443" - target = google_compute_target_https_proxy.relay_gce[0].id - load_balancing_scheme = "EXTERNAL_MANAGED" - network_tier = "PREMIUM" -} diff --git a/cloud/infra/terraform/relay-gce-foundation.tf b/cloud/infra/terraform/relay-gce-foundation.tf deleted file mode 100644 index a8d64b3fcea..00000000000 --- a/cloud/infra/terraform/relay-gce-foundation.tf +++ /dev/null @@ -1,212 +0,0 @@ -locals { - relay_gce_configured = var.relay_gce_domain != "" - relay_gce_name = "${var.name_prefix}-relay-gce" -} - -resource "google_compute_network" "relay_gce" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - name = local.relay_gce_name - auto_create_subnetworks = false - routing_mode = "REGIONAL" -} - -resource "google_compute_subnetwork" "relay_gce" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - name = local.relay_gce_name - region = var.region - network = google_compute_network.relay_gce[0].id - ip_cidr_range = var.relay_gce_subnetwork_cidr - private_ip_google_access = true - stack_type = "IPV4_ONLY" -} - -resource "google_compute_router" "relay_gce" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - name = local.relay_gce_name - region = var.region - network = google_compute_network.relay_gce[0].id -} - -resource "google_compute_router_nat" "relay_gce" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - name = local.relay_gce_name - region = var.region - router = google_compute_router.relay_gce[0].name - nat_ip_allocate_option = "AUTO_ONLY" - source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" - # Cells reach Cloud SQL's public IP through this NAT. The static default of 64 ports per VM - # filled during the 2026-09-04 incident and every cell's proxy dial timed out at once. - enable_dynamic_port_allocation = true - enable_endpoint_independent_mapping = false - min_ports_per_vm = 64 - max_ports_per_vm = 4096 - - subnetwork { - name = google_compute_subnetwork.relay_gce[0].id - source_ip_ranges_to_nat = ["ALL_IP_RANGES"] - } - - log_config { - enable = true - filter = "ERRORS_ONLY" - } -} - -# The primary US resources above retain their production addresses. New regions are additive. -resource "google_compute_subnetwork" "relay_gce_additional" { - for_each = local.relay_gce_configured ? var.relay_gce_additional_region_subnetwork_cidrs : {} - - project = var.project_id - name = "${local.relay_gce_name}-${each.key}" - region = each.key - network = google_compute_network.relay_gce[0].id - ip_cidr_range = each.value - private_ip_google_access = true - stack_type = "IPV4_ONLY" -} - -resource "google_compute_router" "relay_gce_additional" { - for_each = local.relay_gce_configured ? var.relay_gce_additional_region_subnetwork_cidrs : {} - - project = var.project_id - name = "${local.relay_gce_name}-${each.key}" - region = each.key - network = google_compute_network.relay_gce[0].id -} - -resource "google_compute_router_nat" "relay_gce_additional" { - for_each = local.relay_gce_configured ? var.relay_gce_additional_region_subnetwork_cidrs : {} - - project = var.project_id - name = "${local.relay_gce_name}-${each.key}" - region = each.key - router = google_compute_router.relay_gce_additional[each.key].name - nat_ip_allocate_option = "AUTO_ONLY" - source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" - # Cells reach Cloud SQL's public IP through this NAT. The static default of 64 ports per VM - # filled during the 2026-09-04 incident and every cell's proxy dial timed out at once. - enable_dynamic_port_allocation = true - enable_endpoint_independent_mapping = false - min_ports_per_vm = 64 - max_ports_per_vm = 4096 - - subnetwork { - name = google_compute_subnetwork.relay_gce_additional[each.key].id - source_ip_ranges_to_nat = ["ALL_IP_RANGES"] - } - - log_config { - enable = true - filter = "ERRORS_ONLY" - } -} - -resource "google_compute_firewall" "relay_gce_load_balancer" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - name = "${local.relay_gce_name}-lb" - network = google_compute_network.relay_gce[0].name - direction = "INGRESS" - source_ranges = ["35.191.0.0/16", "130.211.0.0/22"] - target_tags = ["orca-relay-cell"] - - allow { - protocol = "tcp" - ports = ["8080"] - } -} - -resource "google_compute_firewall" "relay_gce_iap_ssh" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - name = "${local.relay_gce_name}-iap-ssh" - network = google_compute_network.relay_gce[0].name - direction = "INGRESS" - source_ranges = ["35.235.240.0/20"] - target_tags = ["orca-relay-cell"] - - allow { - protocol = "tcp" - ports = ["22"] - } -} - -resource "google_project_iam_member" "relay_runtime_artifact_reader" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - role = "roles/artifactregistry.reader" - member = google_service_account.relay_runtime.member -} - -resource "google_project_iam_member" "relay_runtime_log_writer" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - role = "roles/logging.logWriter" - member = google_service_account.relay_runtime.member -} - -resource "google_compute_global_address" "relay_gce" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - name = local.relay_gce_name - address_type = "EXTERNAL" - ip_version = "IPV4" -} - -resource "google_certificate_manager_dns_authorization" "relay_gce" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - name = local.relay_gce_name - domain = var.relay_gce_domain - location = "global" - type = "PER_PROJECT_RECORD" - description = "DNS authorization for Orca Relay wildcard cell certificates." - labels = local.relay_shared_labels -} - -resource "google_certificate_manager_certificate" "relay_gce" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - name = local.relay_gce_name - location = "global" - description = "Wildcard certificate for exact-routed Orca Relay GCE cells." - labels = local.relay_shared_labels - - managed { - domains = ["*.${var.relay_gce_domain}"] - dns_authorizations = [google_certificate_manager_dns_authorization.relay_gce[0].id] - } -} - -resource "google_certificate_manager_certificate_map" "relay_gce" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - name = local.relay_gce_name - description = "Certificate map for the shared Orca Relay HTTPS load balancer." -} - -resource "google_certificate_manager_certificate_map_entry" "relay_gce" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - name = "${local.relay_gce_name}-wildcard" - map = google_certificate_manager_certificate_map.relay_gce[0].name - hostname = "*.${var.relay_gce_domain}" - certificates = [google_certificate_manager_certificate.relay_gce[0].id] -} diff --git a/cloud/infra/terraform/relay-gce-startup.sh.tftpl b/cloud/infra/terraform/relay-gce-startup.sh.tftpl deleted file mode 100644 index a77466f2169..00000000000 --- a/cloud/infra/terraform/relay-gce-startup.sh.tftpl +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash -set -euo pipefail - -readonly metadata_url="http://metadata.google.internal/computeMetadata/v1" -readonly state_dir="/var/lib/orca-relay" -readonly cloudsql_dir="$${state_dir}/cloudsql" -readonly docker_config_dir="$${state_dir}/docker" -readonly env_file="$${state_dir}/relay.env" - -# COS mounts /root read-only, and neither the plaintext environment nor pull credential should survive bootstrap. -trap 'rm -f "$${env_file}"; rm -rf "$${docker_config_dir}"' EXIT - -mkdir -p "$${cloudsql_dir}" "$${docker_config_dir}" -chmod 0700 "$${state_dir}" -chmod 0700 "$${docker_config_dir}" -chmod 0777 "$${cloudsql_dir}" -export DOCKER_CONFIG="$${docker_config_dir}" - -# Startup metadata can run before COS has made the Docker socket usable. -systemctl start docker -for _ in $(seq 1 60); do - if docker info >/dev/null 2>&1; then - break - fi - sleep 1 -done -docker info >/dev/null - -metadata_access_token() { - curl --fail --silent --show-error \ - --header 'Metadata-Flavor: Google' \ - "$${metadata_url}/instance/service-accounts/default/token" \ - | sed -n 's/.*"access_token":"\([^"]*\)".*/\1/p' -} - -read_secret() { - local secret_name="$1" - local token="$2" - local encoded - encoded="$(curl --fail --silent --show-error \ - --header "Authorization: Bearer $${token}" \ - "https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets/$${secret_name}/versions/latest:access" \ - | sed -n 's/.*"data":[[:space:]]*"\([^"]*\)".*/\1/p')" - test -n "$${encoded}" - printf '%s' "$${encoded}" | tr '_-' '/+' | base64 --decode -} - -access_token="$(metadata_access_token)" -test -n "$${access_token}" -database_url="$(read_secret '${database_secret}' "$${access_token}")" -assignment_key="$(read_secret '${assignment_secret}' "$${access_token}")" - -umask 077 -{ - printf 'DATABASE_URL=%s\n' "$${database_url}" - printf 'ORCA_RELAY_ASSIGNMENT_SIGNING_KEY=%s\n' "$${assignment_key}" - printf 'ORCA_RELAY_PUBLIC_URL=%s\n' '${cell_url}' - printf 'ORCA_RELAY_CELL_URL=%s\n' '${cell_url}' - printf 'ORCA_RELAY_AUTH_ISSUER=%s\n' '${auth_issuer}' - printf 'ORCA_RELAY_AUTH_AUDIENCE=orca-relay\n' - printf 'ORCA_RELAY_JWKS_URL=%s/.well-known/jwks.json\n' '${auth_issuer}' - printf 'ORCA_RELAY_ROLE=cell\n' - printf 'ORCA_RELAY_CELL_ID=%s\n' '${cell_id}' -%{ if include_cell_region ~} - printf 'ORCA_RELAY_REGION=%s\n' '${cell_region}' -%{ endif ~} - printf 'ORCA_RELAY_CELL_CAPACITY=%s\n' '${capacity_requests}' -%{ if include_database_pool_max ~} - printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\n' '${database_pool_max}' -%{ endif ~} -%{ if connection_hard_cap != null ~} - printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\n' '${connection_hard_cap}' - printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\n' '${connection_unobserved_bound}' -%{ endif ~} - printf 'ORCA_RELAY_CELLS_JSON=[]\n' - printf 'ORCA_RELAY_ADMIN_AUDIENCE=%s/v1/admin/drain\n' '${director_url}' - printf 'ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT=%s\n' '${deploy_service_account}' - printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\n' '${capacity_service_account}' -%{ if asia_proof_service_account != "" ~} - printf 'ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT=%s\n' '${asia_proof_service_account}' -%{ endif ~} - printf 'ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT=%s\n' '${runtime_service_account}' -%{ if rehome_source_enabled ~} - printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\n' '${rehome_director_service_account}' - printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\n' '${rehome_audience}' -%{ endif ~} - printf 'ORCA_RELAY_DIRECTOR_URL=%s\n' '${director_url}' - printf 'ORCA_RELAY_HEARTBEAT_AUDIENCE=%s/v1/admin/cell-heartbeat\n' '${director_url}' - printf 'ORCA_RELAY_IMAGE_DIGEST=%s\n' '${trimprefix(regex("@sha256:[a-f0-9]{64}$", relay_image), "@")}' -} > "$${env_file}" -unset database_url assignment_key - -# COS has no long-lived registry credential; use a short metadata token only for the pull. -printf '%s' "$${access_token}" \ - | docker login --username oauth2accesstoken --password-stdin 'https://${artifact_registry_host}' -docker pull '${relay_image}' -docker logout '${artifact_registry_host}' >/dev/null 2>&1 || true -unset access_token -docker pull '${cloud_sql_proxy_image}' - -docker rm --force orca-relay cloud-sql-proxy >/dev/null 2>&1 || true -# The persistent COS state directory can retain a dead proxy's socket across VM reboots. -find "$${cloudsql_dir}" -type s -name '.s.PGSQL.5432' -delete -docker run --detach \ - --name cloud-sql-proxy \ - --restart always \ - --security-opt no-new-privileges \ - --cap-drop ALL \ - --user 0:0 \ - --volume "$${cloudsql_dir}:/cloudsql" \ - '${cloud_sql_proxy_image}' \ -%{ if cloud_sql_private_ip ~} - --private-ip \ -%{ endif ~} - --unix-socket=/cloudsql \ - '${cloud_sql_connection_name}' - -# Readiness depends on the proxy socket, so do not start the relay into a known SQL failure. -for _ in $(seq 1 60); do - if find "$${cloudsql_dir}" -type s -name '.s.PGSQL.5432' -print -quit | grep -q .; then - break - fi - sleep 1 -done -find "$${cloudsql_dir}" -type s -name '.s.PGSQL.5432' -print -quit | grep -q . - -docker run --detach \ - --name orca-relay \ - --restart always \ - --stop-timeout 300 \ - --security-opt no-new-privileges \ - --cap-drop ALL \ - --publish 8080:8080 \ - --volume "$${cloudsql_dir}:/cloudsql" \ - --env-file "$${env_file}" \ - '${relay_image}' - -# GCE cells feed the same privacy-safe aggregate metrics as Cloud Run without app credentials. -systemctl start logging-agent.target diff --git a/cloud/infra/terraform/relay-github-actions.tf b/cloud/infra/terraform/relay-github-actions.tf deleted file mode 100644 index f7f130263db..00000000000 --- a/cloud/infra/terraform/relay-github-actions.tf +++ /dev/null @@ -1,682 +0,0 @@ -# GitHub Actions identities the relay root owns. -# -# The shared deploy account, its Workload Identity provider, and everything bound to it are -# environment-conditional under amendment A1: production is relay-owned, staging is apps-owned. -# Both state surgeries are done, so their counts are production-only here; the staging copies -# are declared by infra/terraform-apps and live in its state. -# -# The Workload Identity pool itself is foundation-owned and reached by literal through -# relay-shared.tf, never by reference. - -locals { - github_monitor_caller_workflow_file = "monitor-relay-production.yml" - github_monitor_workflow_file = "monitor-relay-production-job.yml" - github_fence_workflow_file = "deploy-relay-production-multi-target.yml" - github_production_relay_workflow_files = [ - "deploy-relay-fence-broker.yml", - "deploy-relay-production-capacity.yml", - "deploy-relay-production-director.yml", - "deploy-relay-production-multi-target.yml", - "deploy-relay-production.yml", - "operate-relay-asia-admission.yml", - "operate-relay-production-cell-flags.yml", - "operate-relay-production-reserve-watch.yml", - "publish-relay-production.yml" - - ] - github_production_relay_capacity_workflow_file = "deploy-relay-production-capacity.yml" - github_production_relay_capacity_job_workflow_file = "deploy-relay-production-capacity-job.yml" - github_production_relay_same_cap_workflow_file = "deploy-relay-production-same-cap.yml" - github_production_relay_same_cap_job_workflow_file = "deploy-relay-production-same-cap-job.yml" - github_production_relay_rehome_workflow_file = "operate-relay-production-rehome.yml" - github_production_relay_rehome_job_workflow_file = "operate-relay-production-rehome-job.yml" - github_staging_relay_capacity_workflow_files = [ - "bootstrap-relay-staging-capacity.yml", - "prove-relay-staging-capacity.yml", - "recover-relay-staging-c4-image.yml" - ] - - # The same-cap caller admits its own jobs too: release_lease runs in the caller file and presents - # the caller as job_workflow_ref, so pinning only the reusable job would refuse it. - github_production_relay_workflow_clauses = [ - for prefix in local.relay_github_workflow_ref_prefixes : - "((${join(" || ", [for workflow_file in local.github_production_relay_workflow_files : "assertion.workflow_ref == '${prefix}${workflow_file}@refs/heads/main'"])}) || (assertion.workflow_ref == '${prefix}${local.github_production_relay_rehome_workflow_file}@refs/heads/main' && assertion.job_workflow_ref == '${prefix}${local.github_production_relay_rehome_job_workflow_file}@refs/heads/main') || (assertion.workflow_ref == '${prefix}${local.github_production_relay_same_cap_workflow_file}@refs/heads/main' && (assertion.job_workflow_ref == '${prefix}${local.github_production_relay_same_cap_job_workflow_file}@refs/heads/main' || assertion.job_workflow_ref == '${prefix}${local.github_production_relay_same_cap_workflow_file}@refs/heads/main')))" - ] - github_monitor_workflow_clauses = [ - for prefix in local.relay_github_workflow_ref_prefixes : - "assertion.workflow_ref == '${prefix}${local.github_monitor_caller_workflow_file}@refs/heads/main' && assertion.job_workflow_ref == '${prefix}${local.github_monitor_workflow_file}@refs/heads/main'" - ] - github_fence_workflow_clauses = [ - for prefix in local.relay_github_workflow_ref_prefixes : - "assertion.workflow_ref == '${prefix}${local.github_fence_workflow_file}@refs/heads/main' && assertion.job_workflow_ref == '${prefix}${local.github_fence_workflow_file}@refs/heads/main'" - ] - github_production_relay_capacity_workflow_clauses = [ - for prefix in local.relay_github_workflow_ref_prefixes : - "((assertion.workflow_ref == '${prefix}${local.github_production_relay_capacity_workflow_file}@refs/heads/main' && assertion.job_workflow_ref == '${prefix}${local.github_production_relay_capacity_job_workflow_file}@refs/heads/main') || (assertion.workflow_ref == '${prefix}${local.github_production_relay_same_cap_workflow_file}@refs/heads/main' && assertion.job_workflow_ref == '${prefix}${local.github_production_relay_same_cap_job_workflow_file}@refs/heads/main'))" - ] - github_staging_relay_capacity_workflow_clauses = [ - for prefix in local.relay_github_workflow_ref_prefixes : - "(${join(" || ", [for workflow_file in local.github_staging_relay_capacity_workflow_files : "assertion.workflow_ref == '${prefix}${workflow_file}@refs/heads/main'"])})" - ] - - create_staging_relay_power_role = ( - local.relay_create_github_deploy_identity && var.environment == "staging" - ) - create_staging_relay_capacity_identity = ( - local.relay_create_github_deploy_identity && var.environment == "staging" - ) - create_production_relay_capacity_identity = ( - local.relay_create_github_deploy_identity && var.environment == "production" - ) -} - -resource "google_iam_workload_identity_pool_provider" "github" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - workload_identity_pool_id = local.relay_workload_identity_pool_id - workload_identity_pool_provider_id = "github" - display_name = "GitHub" - - attribute_mapping = { - "google.subject" = "assertion.sub" - "attribute.actor" = "assertion.actor" - "attribute.environment" = "assertion.environment" - "attribute.ref" = "assertion.ref" - "attribute.repository" = "assertion.repository" - "attribute.repository_id" = "assertion.repository_id" - "attribute.repository_owner_id" = "assertion.repository_owner_id" - "attribute.workflow_ref" = "assertion.workflow_ref" - } - - # Production-only, so the condition is unconditional. The staging copy of this provider is - # declared by infra/terraform-apps/github-actions.tf and pinned there. - attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ - "assertion.ref == 'refs/heads/main'", - "assertion.environment == 'production'", - local.relay_github_workflow_conditions["github"] - ])) - - oidc { - issuer_uri = "https://token.actions.githubusercontent.com" - } -} - -resource "google_iam_workload_identity_pool_provider" "github_monitor" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - workload_identity_pool_id = local.relay_workload_identity_pool_id - workload_identity_pool_provider_id = "github-relay-monitor" - display_name = "GitHub Relay production monitor" - - attribute_mapping = { - "google.subject" = "assertion.sub" - "attribute.repository" = "assertion.repository" - "attribute.ref" = "assertion.ref" - "attribute.environment" = "assertion.environment" - "attribute.job_workflow_ref" = "assertion.job_workflow_ref" - "attribute.relay_ops_identity" = "'monitor'" - } - - attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ - "assertion.ref == 'refs/heads/main'", - "assertion.environment == 'production'", - local.relay_github_workflow_conditions["github_monitor"] - ])) - - oidc { - issuer_uri = "https://token.actions.githubusercontent.com" - } -} - -resource "google_iam_workload_identity_pool_provider" "github_fence" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - workload_identity_pool_id = local.relay_workload_identity_pool_id - workload_identity_pool_provider_id = "github-relay-fence" - display_name = "GitHub Relay production fence" - - attribute_mapping = { - "google.subject" = "assertion.sub" - "attribute.repository" = "assertion.repository" - "attribute.ref" = "assertion.ref" - "attribute.environment" = "assertion.environment" - "attribute.job_workflow_ref" = "assertion.job_workflow_ref" - "attribute.relay_ops_identity" = "'fence'" - } - - attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ - "assertion.ref == 'refs/heads/main'", - "assertion.environment == 'production'", - local.relay_github_workflow_conditions["github_fence"] - ])) - - oidc { - issuer_uri = "https://token.actions.githubusercontent.com" - } -} - -resource "google_iam_workload_identity_pool_provider" "github_staging_relay_capacity" { - count = local.create_staging_relay_capacity_identity ? 1 : 0 - - project = var.project_id - workload_identity_pool_id = local.relay_workload_identity_pool_id - workload_identity_pool_provider_id = "github-relay-capacity" - display_name = "GitHub Relay staging capacity" - - attribute_mapping = { - "google.subject" = "assertion.sub" - "attribute.repository" = "assertion.repository" - "attribute.ref" = "assertion.ref" - "attribute.environment" = "assertion.environment" - "attribute.workflow_ref" = "assertion.workflow_ref" - "attribute.relay_ops_identity" = "'staging-capacity'" - } - - attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ - "assertion.ref == 'refs/heads/main'", - "assertion.environment == 'staging'", - local.relay_github_workflow_conditions["github_staging_relay_capacity"] - ])) - - oidc { - issuer_uri = "https://token.actions.githubusercontent.com" - } -} - -resource "google_iam_workload_identity_pool_provider" "github_production_relay_capacity" { - count = local.create_production_relay_capacity_identity ? 1 : 0 - - project = var.project_id - workload_identity_pool_id = local.relay_workload_identity_pool_id - workload_identity_pool_provider_id = "github-relay-capacity" - display_name = "GitHub Relay production capacity" - - attribute_mapping = { - "google.subject" = "assertion.sub" - "attribute.repository" = "assertion.repository" - "attribute.ref" = "assertion.ref" - "attribute.environment" = "assertion.environment" - "attribute.workflow_ref" = "assertion.workflow_ref" - "attribute.job_workflow_ref" = "assertion.job_workflow_ref" - "attribute.relay_ops_identity" = "'production-capacity'" - } - - attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ - "assertion.ref == 'refs/heads/main'", - "assertion.environment == 'production'", - local.relay_github_workflow_conditions["github_production_relay_capacity"] - ])) - - oidc { - issuer_uri = "https://token.actions.githubusercontent.com" - } -} - -resource "google_service_account" "github_deploy" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - account_id = "${var.name_prefix}-gha-deploy" - display_name = "Orca Cloud GitHub deploy" - description = "Deploys Orca Cloud from GitHub Actions." -} - -resource "google_service_account" "github_monitor" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - account_id = "${var.name_prefix}-gha-monitor" - display_name = "Orca Relay production monitor" - description = "Reads aggregate Relay production telemetry." -} - -resource "google_service_account" "github_fence" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - account_id = "${var.name_prefix}-gha-fence" - display_name = "Orca Relay production fence requester" - description = "Requests exact reviewed Relay cell fences through the private broker." -} - -resource "google_service_account" "github_staging_relay_capacity" { - count = local.create_staging_relay_capacity_identity ? 1 : 0 - - project = var.project_id - account_id = "${var.name_prefix}-gha-cap" - display_name = "Orca Relay staging capacity transition" - description = "Runs the exact reviewed Relay staging capacity workflow." -} - -resource "google_service_account" "github_production_relay_capacity" { - count = local.create_production_relay_capacity_identity ? 1 : 0 - - project = var.project_id - account_id = "${var.name_prefix}-gha-cap" - display_name = "Orca Relay production capacity transition" - description = "Runs the exact reviewed Relay production capacity workflow." -} - -resource "google_service_account_iam_member" "github_workload_identity_user" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - service_account_id = google_service_account.github_deploy[0].name - role = "roles/iam.workloadIdentityUser" - member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.repository/${local.relay_github_repository}" -} - -# The `github` provider maps assertion.repository straight through, so the binding above admits -# only the primary repository. Each additional accepted repository needs its own principalSet -# before its workflows can mint this account; with an empty list this creates nothing. -resource "google_service_account_iam_member" "github_accepted_repository_workload_identity_user" { - for_each = toset( - local.relay_create_production_ops_identity - ? slice(local.relay_github_accepted_repository_names, 1, length(local.relay_github_accepted_repository_names)) - : [] - ) - - service_account_id = google_service_account.github_deploy[0].name - role = "roles/iam.workloadIdentityUser" - member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.repository/${each.key}" -} - -resource "google_service_account_iam_member" "github_monitor_workload_identity_user" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - service_account_id = google_service_account.github_monitor[0].name - role = "roles/iam.workloadIdentityUser" - member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/monitor" -} - -resource "google_service_account_iam_member" "github_fence_workload_identity_user" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - service_account_id = google_service_account.github_fence[0].name - role = "roles/iam.workloadIdentityUser" - member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/fence" -} - -resource "google_service_account_iam_member" "github_staging_relay_capacity_workload_identity_user" { - count = local.create_staging_relay_capacity_identity ? 1 : 0 - - service_account_id = google_service_account.github_staging_relay_capacity[0].name - role = "roles/iam.workloadIdentityUser" - member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/staging-capacity" -} - -resource "google_service_account_iam_member" "github_production_relay_capacity_workload_identity_user" { - count = local.create_production_relay_capacity_identity ? 1 : 0 - - service_account_id = google_service_account.github_production_relay_capacity[0].name - role = "roles/iam.workloadIdentityUser" - member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/production-capacity" -} - -resource "google_artifact_registry_repository_iam_member" "github_production_relay_writer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - location = var.region - repository = var.artifact_repository_id - role = "roles/artifactregistry.writer" - member = local.relay_github_deploy_service_account_member -} - -resource "google_artifact_registry_repository_iam_member" "github_production_relay_staging_mirror_writer" { - count = local.relay_create_github_deploy_identity && var.environment == "staging" ? 1 : 0 - - project = var.project_id - location = var.region - repository = var.artifact_repository_id - role = "roles/artifactregistry.writer" - member = "serviceAccount:orca-cloud-gha-deploy@onorca-cloud.iam.gserviceaccount.com" -} - -resource "google_cloud_run_v2_service_iam_member" "github_production_relay_director_developer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - location = var.region - name = var.relay_cloud_run_service_name - role = "roles/run.developer" - member = local.relay_github_deploy_service_account_member -} - -resource "google_cloud_run_v2_service_iam_member" "github_production_relay_fence_broker_developer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - location = var.region - name = var.relay_fence_broker_service_name - role = "roles/run.developer" - member = local.relay_github_deploy_service_account_member -} - -# Candidate preflight reads GCE/LB topology; Terraform fencing keeps mutation narrowly scoped. -resource "google_project_iam_member" "github_compute_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/compute.viewer" - member = google_service_account.github_deploy[0].member -} - -# Incident monitoring needs aggregate telemetry and inventory without mutation permissions. -resource "google_project_iam_member" "github_monitoring_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/monitoring.viewer" - member = google_service_account.github_deploy[0].member -} - -resource "google_project_iam_member" "github_logging_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/logging.viewer" - member = google_service_account.github_deploy[0].member -} - -# The reserve watch logs its paging alarm line (orca_relay_reserve_watch_alarm) from the runner. -# Accepted risk: Cloud Logging cannot narrow writes to one log, so this identity can write any line. -resource "google_project_iam_member" "github_log_writer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/logging.logWriter" - member = google_service_account.github_deploy[0].member -} - -resource "google_project_iam_member" "github_cloudsql_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/cloudsql.viewer" - member = google_service_account.github_deploy[0].member -} - -resource "google_project_iam_member" "github_relay_monitor_monitoring_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/monitoring.viewer" - member = google_service_account.github_monitor[0].member -} - -resource "google_project_iam_member" "github_relay_monitor_logging_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/logging.viewer" - member = google_service_account.github_monitor[0].member -} - -resource "google_project_iam_member" "github_relay_monitor_cloudsql_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/cloudsql.viewer" - member = google_service_account.github_monitor[0].member -} - -resource "google_project_iam_member" "github_monitor_compute_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/compute.viewer" - member = google_service_account.github_monitor[0].member -} - -resource "google_project_iam_member" "github_fence_monitoring_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/monitoring.viewer" - member = google_service_account.github_fence[0].member -} - -resource "google_project_iam_member" "github_fence_logging_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/logging.viewer" - member = google_service_account.github_fence[0].member -} - -resource "google_project_iam_member" "github_fence_cloudsql_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/cloudsql.viewer" - member = google_service_account.github_fence[0].member -} - -resource "google_project_iam_member" "github_fence_compute_viewer" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - project = var.project_id - role = "roles/compute.viewer" - member = google_service_account.github_fence[0].member -} - -# Staging may sleep between test windows. Keep its workflow narrower than a -# general Compute/Cloud SQL editor and never create this role in production. -resource "google_project_iam_custom_role" "github_staging_relay_power" { - count = local.create_staging_relay_power_role ? 1 : 0 - - project = var.project_id - role_id = "orcaRelayStagingPower" - title = "Orca Relay staging power operator" - description = "Scales staging Relay MIGs and starts or stops its shared staging database." - permissions = [ - "cloudsql.instances.get", - "cloudsql.instances.update", - "cloudsql.operations.get", - "compute.instanceGroupManagers.get", - "compute.instanceGroupManagers.update", - "compute.zoneOperations.get" - ] -} - -resource "google_project_iam_member" "github_staging_relay_power" { - count = local.create_staging_relay_power_role ? 1 : 0 - - project = var.project_id - role = google_project_iam_custom_role.github_staging_relay_power[0].id - member = local.relay_github_deploy_service_account_member -} - -resource "google_project_iam_custom_role" "github_staging_relay_capacity_mutation" { - count = local.create_staging_relay_capacity_identity ? 1 : 0 - - project = var.project_id - role_id = "orcaRelayStagingCapacity" - title = "Orca Relay staging capacity transition" - description = "Replaces one staging Relay template and restarts its managed instance group." - permissions = [ - "compute.disks.create", - "compute.healthChecks.use", - "compute.images.useReadOnly", - "compute.instanceGroupManagers.get", - "compute.instanceGroupManagers.update", - "compute.instances.create", - "compute.instances.setLabels", - "compute.instances.setMetadata", - "compute.instances.setTags", - "compute.instanceTemplates.create", - "compute.instanceTemplates.delete", - "compute.instanceTemplates.get", - "compute.instanceTemplates.useReadOnly", - "compute.networks.use", - "compute.subnetworks.use", - "compute.zoneOperations.get" - ] -} - -resource "google_project_iam_member" "github_staging_relay_capacity_mutation" { - count = local.create_staging_relay_capacity_identity ? 1 : 0 - - project = var.project_id - role = google_project_iam_custom_role.github_staging_relay_capacity_mutation[0].id - member = google_service_account.github_staging_relay_capacity[0].member -} - -resource "google_project_iam_member" "github_staging_relay_capacity_viewer" { - count = local.create_staging_relay_capacity_identity ? 1 : 0 - - project = var.project_id - role = "roles/viewer" - member = google_service_account.github_staging_relay_capacity[0].member -} - -resource "google_project_iam_member" "github_staging_relay_capacity_artifact_reader" { - count = local.create_staging_relay_capacity_identity ? 1 : 0 - - project = var.project_id - role = "roles/artifactregistry.reader" - member = google_service_account.github_staging_relay_capacity[0].member -} - -resource "google_cloud_run_v2_service_iam_member" "github_staging_relay_capacity_developer" { - count = local.create_staging_relay_capacity_identity ? 1 : 0 - - project = var.project_id - location = var.region - name = var.relay_cloud_run_service_name - role = "roles/run.developer" - member = google_service_account.github_staging_relay_capacity[0].member -} - -resource "google_storage_bucket_iam_member" "github_staging_relay_capacity_state" { - count = local.create_staging_relay_capacity_identity ? 1 : 0 - - bucket = "${var.project_id}-terraform-state" - role = "roles/storage.objectAdmin" - member = google_service_account.github_staging_relay_capacity[0].member - - condition { - title = "relay_capacity_state" - description = "Limits the staging capacity workflow to the default Terraform state and lock." - expression = join(" || ", [ - "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tfstate'", - "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tflock'" - ]) - } -} - -resource "google_service_account_iam_member" "github_staging_relay_capacity_runtime_user" { - count = local.create_staging_relay_capacity_identity ? 1 : 0 - - service_account_id = google_service_account.relay_runtime.name - role = "roles/iam.serviceAccountUser" - member = google_service_account.github_staging_relay_capacity[0].member -} - -resource "google_project_iam_custom_role" "github_production_relay_capacity_mutation" { - count = local.create_production_relay_capacity_identity ? 1 : 0 - - project = var.project_id - role_id = "orcaRelayProductionCapacity" - title = "Orca Relay production capacity transition" - description = "Replaces exactly one production Relay template and its managed instance." - permissions = [ - "compute.disks.create", - "compute.healthChecks.use", - "compute.images.useReadOnly", - "compute.instanceGroupManagers.get", - "compute.instanceGroupManagers.update", - "compute.instances.create", - "compute.instances.setLabels", - "compute.instances.setMetadata", - "compute.instances.setTags", - "compute.instanceTemplates.create", - "compute.instanceTemplates.delete", - "compute.instanceTemplates.get", - "compute.instanceTemplates.useReadOnly", - "compute.networks.use", - "compute.subnetworks.use", - "compute.zoneOperations.get" - ] -} - -resource "google_project_iam_member" "github_production_relay_capacity_mutation" { - count = local.create_production_relay_capacity_identity ? 1 : 0 - - project = var.project_id - role = google_project_iam_custom_role.github_production_relay_capacity_mutation[0].id - member = google_service_account.github_production_relay_capacity[0].member -} - -resource "google_project_iam_member" "github_production_relay_capacity_viewer" { - count = local.create_production_relay_capacity_identity ? 1 : 0 - - project = var.project_id - role = "roles/viewer" - member = google_service_account.github_production_relay_capacity[0].member -} - -resource "google_project_iam_member" "github_production_relay_capacity_artifact_reader" { - count = local.create_production_relay_capacity_identity ? 1 : 0 - - project = var.project_id - role = "roles/artifactregistry.reader" - member = google_service_account.github_production_relay_capacity[0].member -} - -resource "google_storage_bucket_iam_member" "github_production_relay_capacity_state" { - count = local.create_production_relay_capacity_identity ? 1 : 0 - - bucket = "${var.project_id}-terraform-state" - role = "roles/storage.objectAdmin" - member = google_service_account.github_production_relay_capacity[0].member - - condition { - title = "relay_production_capacity_state" - description = "Limits the production capacity workflow to the default Terraform state and lock." - expression = join(" || ", [ - "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tfstate'", - "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tflock'" - ]) - } -} - -resource "google_service_account_iam_member" "github_production_relay_capacity_runtime_user" { - count = local.create_production_relay_capacity_identity ? 1 : 0 - - service_account_id = google_service_account.relay_runtime.name - role = "roles/iam.serviceAccountUser" - member = google_service_account.github_production_relay_capacity[0].member -} - -# Candidate preflight reads reviewed Terraform outputs, but must not be able to mutate state. -resource "google_storage_bucket_iam_member" "github_terraform_state_reader" { - count = local.relay_create_production_ops_identity ? 1 : 0 - - bucket = "${var.project_id}-terraform-state" - role = "roles/storage.objectViewer" - member = google_service_account.github_deploy[0].member -} - - -# Relay deploys replace only the image while retaining the Terraform-owned -# runtime identity and service shape. -resource "google_service_account_iam_member" "github_relay_runtime_service_account_user" { - count = local.relay_create_github_deploy_identity ? 1 : 0 - - service_account_id = google_service_account.relay_runtime.name - role = "roles/iam.serviceAccountUser" - member = local.relay_github_deploy_service_account_member -} - -resource "google_service_account_iam_member" "github_relay_director_runtime_service_account_user" { - count = local.relay_create_github_deploy_identity ? 1 : 0 - - service_account_id = google_service_account.relay_director_runtime.name - role = "roles/iam.serviceAccountUser" - member = local.relay_github_deploy_service_account_member -} - diff --git a/cloud/infra/terraform/relay-github-workflow-trust.tf b/cloud/infra/terraform/relay-github-workflow-trust.tf deleted file mode 100644 index 431408f0d9a..00000000000 --- a/cloud/infra/terraform/relay-github-workflow-trust.tf +++ /dev/null @@ -1,39 +0,0 @@ -# The workflow half of every relay Workload Identity condition, one clause per accepted -# repository. -# -# Each provider file builds its own clause list from local.relay_github_workflow_ref_prefixes, so -# a workflow allowlist stays next to the identity it authorizes. This file only names those lists -# and renders them: with a single accepted repository the clause is spliced in unchanged, and with -# more it becomes one parenthesised OR arm per repository, each arm carrying its own repository -# claims. The repository-independent claims (ref, environment, event_name) stay outside the OR in -# the provider blocks. -locals { - relay_github_workflow_clauses = { - github = local.github_production_relay_workflow_clauses - github_push = local.github_push_workflow_clauses - github_monitor = local.github_monitor_workflow_clauses - github_fence = local.github_fence_workflow_clauses - github_production_relay_capacity = local.github_production_relay_capacity_workflow_clauses - github_staging_relay_capacity = local.github_staging_relay_capacity_workflow_clauses - github_staging_relay_deploy = local.github_staging_relay_deploy_workflow_clauses - github_relay_asia_topology = local.github_relay_asia_topology_workflow_clauses - github_relay_asia_proof = local.github_relay_asia_proof_workflow_clauses - } - - relay_github_workflow_arms = { - for name, clauses in local.relay_github_workflow_clauses : - name => [ - for index, clause in clauses : - "(${local.relay_github_accepted_repository_claims[index]} && ${clause})" - ] - } - - relay_github_workflow_conditions = { - for name, clauses in local.relay_github_workflow_clauses : - name => ( - local.relay_github_single_repository - ? clauses[0] - : "(${join(" || ", local.relay_github_workflow_arms[name])})" - ) - } -} diff --git a/cloud/infra/terraform/relay-observability.tf b/cloud/infra/terraform/relay-observability.tf deleted file mode 100644 index 5149d355040..00000000000 --- a/cloud/infra/terraform/relay-observability.tf +++ /dev/null @@ -1,2082 +0,0 @@ -locals { - relay_service_names = concat( - [var.relay_cloud_run_service_name], - [for cell in values(var.relay_cells) : cell.service_name] - ) - relay_service_log_filter = join(" OR ", [ - for name in local.relay_service_names : "resource.labels.service_name=\"${name}\"" - ]) - relay_runtime_log_filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR (resource.type=\"gce_instance\" AND jsonPayload.role=\"cell\")) AND jsonPayload.event=\"orca_relay_runtime_metrics\"" - relay_gce_connection_warning_thresholds = { - for cell_id, cell in var.relay_gce_cells : - cell_id => cell.connection_hard_cap == null ? 550 : floor( - (cell.connection_hard_cap - 100 - cell.connection_unobserved_bound) * 0.85 - ) - } - relay_gce_connection_warning_groups = { - for threshold in distinct(values(local.relay_gce_connection_warning_thresholds)) : - tostring(threshold) => sort([ - for cell_id, cell_threshold in local.relay_gce_connection_warning_thresholds : - cell_id if cell_threshold == threshold - ]) - } - relay_incident_metrics = { - assignment_5xx = { - description = "Director assignment requests returning a server error." - filter = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${var.relay_cloud_run_service_name}\" AND httpRequest.requestMethod=\"POST\" AND httpRequest.requestUrl=~\"/v1/assign$\" AND httpRequest.status>=500" - } - assignment_edge_429 = { - description = "Director assignment or resolve requests rejected before an instance was available." - filter = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${var.relay_cloud_run_service_name}\" AND httpRequest.requestMethod=\"POST\" AND httpRequest.requestUrl=~\"/v1/(assign|resolve)$\" AND httpRequest.status=429" - } - postgres_retries = { - description = "Relay PostgreSQL transactions recovered after a retryable abort." - filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_postgres_transaction_retry\"" - } - postgres_retry_exhausted = { - description = "Relay PostgreSQL transactions that exhausted bounded retry." - filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_postgres_transaction_exhausted\"" - } - cell_process_exit = { - # The docker event stream is the only per-exit line: the relay's own crash footer only - # appears for unhandled rejections, and `container start` also counts healthy first boots. - description = "Relay cell container exits, one Docker `container die` event per process exit." - filter = "resource.type=\"gce_instance\" AND logName=\"projects/${var.project_id}/logs/cos_system\" AND jsonPayload.SYSLOG_IDENTIFIER=\"docker\" AND jsonPayload.MESSAGE:\"container die\" AND jsonPayload.MESSAGE:\"name=orca-relay)\"" - } - cloud_sql_wal_checkpoint = { - description = "Cloud SQL checkpoints triggered by WAL volume instead of the timed schedule; a sustained run is the fsync loop that stalled every relay process at once on 2026-09-04." - filter = "resource.type=\"cloudsql_database\" AND resource.labels.database_id=\"${var.project_id}:${local.relay_database_instance_name}\" AND textPayload:\"checkpoint starting: wal\"" - } - # Why: `db=orca_relay,` keeps auth on the shared instance out. NOWAIT refusals ("could not - # obtain lock") are excluded: sweeps step aside by design at ~160/min even with rehome paused. - cloud_sql_lock_timeouts = { - description = "Relay statements Postgres cancelled after waiting out their lock timeout; a burst means one transaction is holding rows every other relay process needs." - filter = "resource.type=\"cloudsql_database\" AND resource.labels.database_id=\"${var.project_id}:${local.relay_database_instance_name}\" AND textPayload:\"db=orca_relay,\" AND textPayload:\"canceling statement due to lock timeout\"" - } - reservation_drift = { - description = "Cells whose reserved_requests disagreed with their lease units when reconciliation corrected them; should trend to zero." - filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_reservation_drift\"" - } - # Why the cell's line and not the load balancer's: the LB writes a WebSocket's log entry at - # close but stamps it with the stream's start, and a log-based metric counts by that stamp. Of - # the 1,046 `internal_error` streams on 2026-10-06 18:34Z, 42 were stamped within 10 minutes of - # the drop; the median was 2.4 h earlier, so an LB-log rate never shows the pulse. - # Both lines come from the unhandledRejection fence (relay-database-rejection-fence.ts). - database_rejection_fenced = { - description = "Database faults that reached unhandledRejection and were survived: a caller forgot to await a database promise. Should be zero; each one is a bug to find." - filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_database_rejection_fenced\"" - } - process_fatal = { - description = "Unhandled rejections the fence kept fatal (not a database fault); each is followed by a process exit." - filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_process_fatal\"" - } - cell_control_abnormal_closes = { - description = "Desktop control sockets a GCE cell saw end with no close frame (1006), counted when they closed. Hundreds on one cell in a minute is the load balancer ending streams." - filter = "resource.type=\"gce_instance\" AND logName=\"projects/${var.project_id}/logs/cos_containers\" AND jsonPayload.message:\"[orca-relay] control closed \" AND jsonPayload.message:\" code=1006 \"" - } - } - - relay_runtime_metrics = { - total_connections = { field = "totalConnections", description = "Open relay WebSocket requests per process." } - controls = { field = "controls", description = "Authenticated standing desktop controls per process." } - splices = { field = "splices", description = "Active phone-to-desktop ciphertext splices per process." } - pending_splices = { field = "pendingSplices", description = "Phone connections waiting for their host data leg." } - queued_bytes = { field = "queuedBytes", description = "Process-wide relay backpressure bytes." } - http_latency_ms = { field = "httpLatencyMsMax", description = "Maximum director/administrative HTTP latency in the interval." } - sql_latency_ms = { field = "sqlLatencyMsMax", description = "Maximum observed SQL operation latency in the interval." } - control_renewal_latency_ms_p50 = { field = "controlRenewalLatencyMsP50", description = "Control renewal latency p50 in the interval." } - control_renewal_latency_ms_p95 = { field = "controlRenewalLatencyMsP95", description = "Control renewal latency p95 in the interval." } - control_renewal_latency_ms_max = { field = "controlRenewalLatencyMsMax", description = "Maximum control renewal latency in the interval." } - control_renewal_flushes = { field = "controlRenewalFlushesDelta", description = "Batched control-renewal statements issued in the interval, one per cell per flush window." } - control_renewal_flush_rows_max = { field = "controlRenewalFlushRowsMax", description = "Largest number of hosts renewed by a single statement in the interval; the row ceiling is what bounds how long one flush holds its row locks." } - control_renewal_flush_ms_p95 = { field = "controlRenewalFlushLatencyMsP95", description = "Batched control-renewal statement duration p95 in the interval. Row locks live until the statement commits, so this is the lock hold." } - control_renewal_flush_ms_max = { field = "controlRenewalFlushLatencyMsMax", description = "Maximum batched control-renewal statement duration in the interval." } - control_renewals = { field = "controlRenewalsDelta", description = "Control renewal attempts in the interval." } - control_renewal_successes = { field = "controlRenewalSuccessesDelta", description = "Successful control renewals in the interval." } - control_renewal_lease_misses = { field = "controlRenewalLeaseMissesDelta", description = "Control renewals that found their activity lease missing." } - control_activity_recoveries = { field = "controlActivityRecoveriesDelta", description = "Control activity leases recovered after a renewal miss." } - control_activity_recovery_failures = { field = "controlActivityRecoveryFailuresDelta", description = "Control activity lease recovery attempts that failed." } - control_rtt_ms_p50 = { field = "controlRttMsP50", description = "Control-socket ping round trip p50 in the interval. The desktop echoes the pong on its main thread, so only the median reads as distance; the p95 and max below are dominated by desktop stalls." } - control_rtt_ms_p95 = { field = "controlRttMsP95", description = "Control-socket ping round trip p95 in the interval; a desktop-stall signal, not a distance one." } - control_rtt_ms_max = { field = "controlRttMsMax", description = "Maximum control-socket ping round trip in the interval; a desktop-stall signal, not a distance one." } - control_rtt_samples = { field = "controlRttSamplesDelta", description = "Control-socket round trips observed in the interval, one per ping answered; the percentiles above are omitted when this is zero." } - control_rtt_samples_dropped = { field = "controlRttSamplesDroppedDelta", description = "Observed round trips the bounded percentile reservoir did not keep; non-zero means the percentiles above summarise a uniform sample of the interval." } - client_accepts_completed = { field = "clientAcceptCompletedDelta", description = "Phone accepts that reached relay-hello in the interval; the percentiles below are omitted when this is zero." } - client_accept_total_ms_p50 = { field = "clientAcceptTotalMsP50", description = "Successful phone-accept duration p50, dial to relay-hello." } - client_accept_total_ms_p95 = { field = "clientAcceptTotalMsP95", description = "Successful phone-accept duration p95, dial to relay-hello." } - client_accept_total_ms_max = { field = "clientAcceptTotalMsMax", description = "Maximum successful phone-accept duration in the interval." } - client_accept_assignment_ms_p95 = { field = "clientAcceptAssignmentMsP95", description = "Accept stage p95: resume/invite lookup plus assignment resolve." } - client_accept_credential_ms_p95 = { field = "clientAcceptCredentialMsP95", description = "Accept stage p95: outer credential reservation." } - client_accept_activity_ms_p95 = { field = "clientAcceptActivityMsP95", description = "Accept stage p95: credential activity lease acquisition." } - client_accept_attach_ms_p95 = { field = "clientAcceptAttachMsP95", description = "Accept stage p95: conn-open sent until the desktop's data leg authenticated." } - client_accept_basis_ms_p95 = { field = "clientAcceptBasisMsP95", description = "Accept stage p95: splice lease and connection-basis writes between the data leg and relay-hello." } - heap_used_bytes = { field = "heapUsedBytes", description = "Node.js heap bytes used by the relay process." } - event_loop_ms_p99 = { field = "eventLoopDelayMsP99", description = "Node.js event-loop delay p99 in milliseconds." } - forwarded_bytes = { field = "forwardedBytesDelta", description = "Ciphertext bytes admitted for forwarding." } - auth_successes = { field = "authSuccessesDelta", description = "Successful outer relay authentication stages." } - auth_failures = { field = "authFailuresDelta", description = "Rejected or timed-out outer relay authentication stages." } - reconnects = { field = "reconnectsDelta", description = "Desktop control rebinds or generation replacements." } - sql_queries = { field = "sqlQueriesDelta", description = "Completed relay SQL operations." } - sql_failures = { field = "sqlFailuresDelta", description = "Failed relay SQL operations." } - db_pool_total = { field = "databasePoolTotal", description = "Open PostgreSQL connections in the process pool." } - db_pool_idle = { field = "databasePoolIdle", description = "Idle PostgreSQL connections in the process pool." } - db_pool_waiting = { field = "databasePoolWaiting", description = "Current requests queued for a PostgreSQL connection." } - db_waiters_max = { field = "databasePoolWaitersMax", description = "Maximum requests queued for a PostgreSQL connection during the interval." } - db_oldest_wait_ms = { field = "databasePoolOldestWaitMs", description = "Current oldest PostgreSQL pool waiter age." } - db_wait_ms_max = { field = "databasePoolWaitMsMax", description = "Maximum PostgreSQL pool wait during the interval." } - host_hellos_shed = { field = "hostHellosShedDelta", description = "Desktop control connections refused with a retryable 503 because the PostgreSQL pool queue was full." } - cell_inventory_hold_ms_max = { field = "cellInventoryHoldMsMax", description = "Longest cell-inventory lock hold in the interval." } - cell_inventory_hold_ms_p95 = { field = "cellInventoryHoldMsP95", description = "Cell-inventory lock hold p95 in the interval; the bound is tuned against this." } - cell_inventory_holds = { field = "cellInventoryHolds", description = "Cell-inventory locks acquired in the interval; the percentiles above summarise these." } - cell_inventory_lock_unavailable = { field = "cellInventoryLockUnavailable", description = "Fail-fast cell-inventory acquisitions that found the lock held. Includes background sweeps, which step aside by design, so this is contention pressure rather than user-visible failure." } - cell_inventory_lock_timeouts = { field = "cellInventoryLockTimeouts", description = "Bounded cell-inventory waits that expired, counted per attempt rather than per request. This is the user-visible lane." } - inventory_hold_ms_p99 = { field = "inventoryHoldMsP99", description = "Cell-inventory lock (every row, or every general row) hold p99 in the interval, this site only." } - rehome_target_row_hold_ms_p99 = { field = "rehomeTargetRowHoldMsP99", description = "Rehome target-row lock hold p99 in the interval, this site only." } - isolated_replacement_hold_ms_p99 = { field = "isolatedReplacementHoldMsP99", description = "Drain-return regional target-row lock hold p99 in the interval, this site only." } - isolated_replacement_holds = { field = "isolatedReplacementHolds", description = "Drain-return regional target-row locks acquired in the interval." } - drain_return_service_ms_p50 = { field = "drainReturnServiceMsP50", description = "Drain-return lane slot hold p50; lane capacity is concurrency / this. Omitted when no drain return ran." } - drain_return_service_ms_p95 = { field = "drainReturnServiceMsP95", description = "Drain-return lane slot hold p95 in the interval." } - sticky_service_ms_p50 = { field = "stickyServiceMsP50", description = "Sticky lane slot hold p50, verification included; herd recovery is concurrency / this." } - sticky_service_ms_p99 = { field = "stickyServiceMsP99", description = "Sticky lane slot hold p99 in the interval." } - assign_non_drain_503s = { field = "assignNonDrain503sDelta", description = "Director /v1/assign 503s other than scheduled drain-return deferrals; the per-cause split is assign503sByCauseDelta." } - db_lock_wait_samples = { field = "dbLockWaitSamplesDelta", description = "Relay-database lock-wait samples taken by this director in the interval; divide the waiter sums below by this." } - db_cell_row_lock_waiters_director = { field = "dbCellRowLockWaitersDirectorDelta", description = "Director backends waiting on a relay_cells row lock, summed over samples." } - db_cell_row_lock_waiters_cell = { field = "dbCellRowLockWaitersCellDelta", description = "Cell backends waiting on a relay_cells row lock, summed over samples." } - } - - # Regions the director can hint or select. Pinned to relay-contract's RELAY_REGIONS by - # dev/scripts/relay-region-hint-metrics.test.mjs, which also checks the flat field names below - # against the emitter. A region missing here drops out of both shares the skew alert compares. - relay_region_keys = ["us-central1", "asia-east2"] - # Flat emitter fields, not the nested `requestedRegionsDelta` map: a log-based metric would need - # a quoted field path to reach a hyphenated map key, and the relay publishes these as zeros in - # every interval so no series can drop out of the alert's inner join. Spelled out rather than - # derived, so this literal and relay-contract's RELAY_REGION_METRIC_SEGMENTS can be compared - # directly; reformatting either side cannot break the check and neither can drift alone. - relay_region_field_segments = { - "us-central1" = "UsCentral1" - "asia-east2" = "AsiaEast2" - } - relay_region_columns = { for key in local.relay_region_keys : key => replace(key, "-", "_") } - relay_region_share_metrics = merge( - { - for key in local.relay_region_keys : - "requested_regions_${local.relay_region_columns[key]}" => { - field = "requestedRegion${local.relay_region_field_segments[key]}Delta" - description = "Assignment requests that hinted ${key}." - } - }, - { - for key in local.relay_region_keys : - "selected_regions_${local.relay_region_columns[key]}" => { - field = "selectedRegion${local.relay_region_field_segments[key]}Delta" - description = "Assignments that placed a host in ${key}." - } - } - ) - relay_region_hinted_total = join(" + ", [for key in local.relay_region_keys : "req_${local.relay_region_columns[key]}"]) - relay_region_selected_total = join(" + ", [for key in local.relay_region_keys : "sel_${local.relay_region_columns[key]}"]) - # MQL, not a filter condition: every runtime metric is a DELTA DISTRIBUTION, and the only scalar - # aligners a `condition_threshold` can apply to one are percentiles. Both shares need the sum of - # the extracted values, which is `sum(value.)` in MQL and unreachable otherwise. - relay_region_hint_skew_query = join("\n", concat( - ["{"], - flatten([ - for index, entry in [ - for key in local.relay_region_keys : { metric = "requested_regions_${local.relay_region_columns[key]}", column = "req_${local.relay_region_columns[key]}" } - ] : [ - index == 0 ? "" : ";", - " fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_${entry.metric}", - " | align delta(1h) | every 1h", - " | group_by [], [${entry.column}: sum(value.orca_relay_${entry.metric})]" - ] - ]), - flatten([ - for key in local.relay_region_keys : [ - ";", - " fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_selected_regions_${local.relay_region_columns[key]}", - " | align delta(1h) | every 1h", - " | group_by [], [sel_${local.relay_region_columns[key]}: sum(value.orca_relay_selected_regions_${local.relay_region_columns[key]})]" - ] - ]), - [ - "}", - "| join", - "| value [", - " hint_share: req_asia_east2 / (${local.relay_region_hinted_total}),", - " placement_share: sel_asia_east2 / (${local.relay_region_selected_total}),", - " hinted_requests: ${local.relay_region_hinted_total}", - " ]", - # Cross-multiplied, never a plain ratio of the two shares: an hour that placed nobody in the - # region makes that ratio 0/0 or x/0, and MQL drops the row instead of yielding a number, so - # the whole series vanishes before the other clauses run. That hour is the worst skew there - # is - every desktop asking for a region the director is putting nobody in - and it happens - # whenever the region is drained, fenced, or at capacity. Both forms were run read-only - # against production surrogates with a zero denominator: the ratio returned no rows, this - # returned the series with the condition true. - "| condition hint_share > 2 * placement_share && hint_share - placement_share > 0.15 '1' && hinted_requests > 500 '1'" - ] - )) - relay_custom_alerts = { - connection_headroom = { - pages_oncall = true - metric = "total_connections" - # Live values, set by hand on 2026-08-05. The cell arm is ~85% of the 440 usable - # connection units (600 hard cap - 100 rebind reserve - 60 unobserved bound); 800 - # exceeded the 600 cap outright and could never fire. - threshold_run = 550 - threshold_gce = 374 - duration = "120s" - aligner = "ALIGN_PERCENTILE_99" - reducer = "REDUCE_MAX" - documentation = "A relay process is above its reviewed connection warning point; stop new assignment to the cell and follow the hot-cell runbook." - } - queue_pressure = { - pages_oncall = false - metric = "queued_bytes" - threshold_run = 50331648 - threshold_gce = 50331648 - duration = "120s" - aligner = "ALIGN_PERCENTILE_99" - reducer = "REDUCE_MAX" - documentation = "Process-wide queued ciphertext exceeded 75% of the 64 MiB hard budget. Investigate slow receivers before admission starts rejecting." - } - auth_failures = { - pages_oncall = false - metric = "auth_failures" - threshold_run = 20 - threshold_gce = 20 - duration = "0s" - aligner = "ALIGN_PERCENTILE_99" - reducer = "REDUCE_MAX" - documentation = "Outer authentication failures exceeded the per-process interval threshold. Check auth/JWKS health and abuse sources without logging bearer values." - } - reconnects = { - pages_oncall = false - metric = "reconnects" - threshold_run = 100 - threshold_gce = 100 - duration = "0s" - aligner = "ALIGN_PERCENTILE_99" - reducer = "REDUCE_MAX" - documentation = "Relay control reconnects exceeded the per-process interval threshold. Check revision churn, GFE terminations, and reconnect jitter." - } - sql_failures = { - pages_oncall = true - # Tuned live on 2026-08-05 to stop Slack alert noise; codified here so an apply cannot revert it. - metric = "sql_failures" - threshold_run = 0 - threshold_gce = 0 - duration = "300s" - aligner = "ALIGN_PERCENTILE_99" - reducer = "REDUCE_MAX" - documentation = "A relay SQL operation failed. Check Cloud SQL availability, connection pressure, and transaction retry outcomes." - } - sql_latency = { - pages_oncall = false - metric = "sql_latency_ms" - threshold_run = 500 - threshold_gce = 500 - duration = "120s" - aligner = "ALIGN_PERCENTILE_99" - reducer = "REDUCE_MAX" - documentation = "Relay SQL operations remained above 500 ms. Inspect lock contention and Cloud SQL health before migrations or drains." - } - database_pool_waiters = { - pages_oncall = true - # Tuned live on 2026-08-05 to stop Slack alert noise; codified here so an apply cannot revert it. - metric = "db_waiters_max" - threshold_run = 5 - threshold_gce = 5 - duration = "300s" - aligner = "ALIGN_PERCENTILE_99" - reducer = "REDUCE_MAX" - documentation = "A relay process queued work for a PostgreSQL connection. Check public-assignment admission, pool saturation, and Cloud SQL latency before scaling." - } - database_pool_wait = { - pages_oncall = true - # Tuned live on 2026-08-05 to stop Slack alert noise; codified here so an apply cannot revert it. - metric = "db_wait_ms_max" - threshold_run = 500 - threshold_gce = 500 - duration = "300s" - aligner = "ALIGN_PERCENTILE_99" - reducer = "REDUCE_MAX" - documentation = "A relay process waited over 500 ms for a PostgreSQL connection. Check long transactions and lock contention before migrations or drains." - } - http_latency = { - pages_oncall = false - metric = "http_latency_ms" - threshold_run = 2000 - threshold_gce = 2000 - duration = "120s" - aligner = "ALIGN_PERCENTILE_99" - reducer = "REDUCE_MAX" - documentation = "Relay HTTP handling remained above two seconds. Check director assignment/resolve latency and SQL contention; WebSocket lifetimes are intentionally excluded." - } - heap_pressure = { - pages_oncall = false - metric = "heap_used_bytes" - threshold_run = 419430400 - threshold_gce = 419430400 - duration = "120s" - aligner = "ALIGN_PERCENTILE_99" - reducer = "REDUCE_MAX" - documentation = "Node.js heap stayed above 400 MiB on a 512 MiB container. Drain the affected cell and investigate connection or queue retention." - } - event_loop_delay = { - pages_oncall = false - metric = "event_loop_ms_p99" - threshold_run = 250 - threshold_gce = 250 - duration = "120s" - aligner = "ALIGN_PERCENTILE_99" - reducer = "REDUCE_MAX" - documentation = "Relay event-loop p99 delay stayed above 250 ms. Check CPU, SQL callbacks, synchronized heartbeats, and queue pressure." - } - } -} - -resource "google_logging_metric" "relay_snapshot" { - # Region-request metrics ride the same event and shape; merging adds map entries only, so the - # existing metric instances are untouched (a label change, not a new key, is what recreates them). - for_each = merge(local.relay_runtime_metrics, local.relay_region_share_metrics) - - project = var.project_id - name = "orca_relay_${each.key}" - description = each.value.description - filter = local.relay_runtime_log_filter - value_extractor = "EXTRACT(jsonPayload.${each.value.field})" - label_extractors = { - role = "EXTRACT(jsonPayload.role)" - cell_id = "EXTRACT(jsonPayload.cellId)" - # No region label: adding one replaces all 42 live metrics (label change = delete+create), - # which resets history and blanks the relay alert policies during the swap. - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "DISTRIBUTION" - unit = contains(["sql_latency_ms", "control_rtt_ms_p50", "control_rtt_ms_p95", "control_rtt_ms_max", "client_accept_total_ms_p50", "client_accept_total_ms_p95", "client_accept_total_ms_max", "client_accept_assignment_ms_p95", "client_accept_credential_ms_p95", "client_accept_activity_ms_p95", "client_accept_attach_ms_p95", "client_accept_basis_ms_p95", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max", "inventory_hold_ms_p99", "rehome_target_row_hold_ms_p99", "isolated_replacement_hold_ms_p99", "drain_return_service_ms_p50", "drain_return_service_ms_p95", "sticky_service_ms_p50", "sticky_service_ms_p99"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1" - - labels { - key = "role" - value_type = "STRING" - description = "Relay process role." - } - - labels { - key = "cell_id" - value_type = "STRING" - description = "Durable relay cell identifier." - } - } - - bucket_options { - exponential_buckets { - num_finite_buckets = 24 - growth_factor = 2 - scale = 1 - } - } -} - -resource "google_logging_metric" "relay_incident" { - for_each = local.relay_incident_metrics - - project = var.project_id - name = "orca_relay_${each.key}" - description = each.value.description - filter = each.value.filter - - metric_descriptor { - metric_kind = "DELTA" - value_type = "INT64" - unit = "1" - } -} - -# Why: the hold sample is emitted only after COMMIT, so a failed acquisition never reads as a hold. -# The 1,000 ms bar lives in the filter so the alert is on the sampled milliseconds, not on the -# percentile interpolation of a wide exponential bucket. -resource "google_logging_metric" "relay_long_cell_inventory_hold" { - project = var.project_id - name = "orca_relay_long_cell_inventory_hold_ms" - description = "Cell-inventory lock holds of at least 1,000 ms, one value per 30-second runtime sample." - filter = "${local.relay_runtime_log_filter} AND jsonPayload.cellInventoryHoldMsMax>=1000" - value_extractor = "EXTRACT(jsonPayload.cellInventoryHoldMsMax)" - label_extractors = { - role = "EXTRACT(jsonPayload.role)" - cell_id = "EXTRACT(jsonPayload.cellId)" - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "DISTRIBUTION" - unit = "ms" - - labels { - key = "role" - value_type = "STRING" - description = "Relay process role." - } - - labels { - key = "cell_id" - value_type = "STRING" - description = "Durable relay cell identifier, or director." - } - } - - bucket_options { - explicit_buckets { - bounds = [1000, 2000, 3000, 4000, 6000, 10000, 30000] - } - } -} - -resource "google_monitoring_alert_policy" "relay_custom" { - for_each = local.relay_custom_alerts - - project = var.project_id - display_name = "Orca Relay: ${replace(each.key, "_", " ")}" - combiner = "OR" - enabled = true - # Why: only the reviewed critical alerts page; the rest stay in the console. Applying one - # list to every policy would have put the noisy ones back into Slack. - notification_channels = each.value.pages_oncall ? var.relay_alert_notification_channels : [] - - conditions { - display_name = each.key - - condition_threshold { - filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_${each.value.metric}\"" - comparison = "COMPARISON_GT" - threshold_value = each.value.threshold_run - duration = each.value.duration - - aggregations { - alignment_period = "300s" - per_series_aligner = each.value.aligner - cross_series_reducer = each.value.reducer - group_by_fields = ["resource.label.\"service_name\""] - } - - trigger { - count = 1 - } - } - } - - dynamic "conditions" { - for_each = each.key == "connection_headroom" ? [] : [each.value] - content { - display_name = "${each.key} (GCE cell)" - - condition_threshold { - filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_${conditions.value.metric}\"" - comparison = "COMPARISON_GT" - threshold_value = conditions.value.threshold_gce - duration = conditions.value.duration - - aggregations { - alignment_period = "300s" - per_series_aligner = conditions.value.aligner - cross_series_reducer = conditions.value.reducer - group_by_fields = ["resource.label.\"instance_id\""] - } - - trigger { - count = 1 - } - } - } - } - - documentation { - content = each.value.documentation - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_snapshot] -} - -resource "google_monitoring_alert_policy" "relay_assignment_5xx" { - project = var.project_id - display_name = "Orca Relay: assignment 5xx" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "assignment 5xx" - - condition_threshold { - filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_assignment_5xx\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["resource.label.\"service_name\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A desktop could not obtain a Relay assignment. Check PostgreSQL retry/exhaustion signals and director SQL latency; do not restart GCE cells or invalidate pairings." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_incident] -} - -resource "google_monitoring_alert_policy" "relay_gce_connection_headroom" { - for_each = local.relay_gce_connection_warning_groups - - project = var.project_id - display_name = "Orca Relay: connection headroom (GCE ${each.key})" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "connection headroom (GCE ${each.key})" - - condition_threshold { - filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_total_connections\" AND (${join(" OR ", [for cell_id in each.value : "metric.label.\"cell_id\"=\"${cell_id}\""])})" - comparison = "COMPARISON_GT" - threshold_value = tonumber(each.key) - duration = "120s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_PERCENTILE_99" - cross_series_reducer = "REDUCE_MAX" - group_by_fields = ["resource.label.\"instance_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A Relay GCE cell is above 85% of its configured ordinary placement ceiling; stop new assignment to the cell and follow the hot-cell runbook." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_snapshot] -} - -resource "google_monitoring_alert_policy" "relay_assignment_edge_429" { - project = var.project_id - display_name = "Orca Relay: assignment edge 429" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "assignment edge 429" - - condition_threshold { - filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_assignment_edge_429\"" - comparison = "COMPARISON_GT" - threshold_value = 100 - duration = "0s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["resource.label.\"service_name\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "Cloud Run rejected a sustained assignment burst before an instance was available. Check public-assignment admission, director concurrency, and database latency before scaling." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_incident] -} - -resource "google_monitoring_alert_policy" "relay_postgres_retry_exhausted" { - project = var.project_id - display_name = "Orca Relay: PostgreSQL retry exhausted" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "PostgreSQL retry exhausted (Cloud Run)" - - condition_threshold { - filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_postgres_retry_exhausted\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "180s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["resource.label.\"service_name\""] - } - - trigger { - count = 1 - } - } - } - - conditions { - display_name = "PostgreSQL retry exhausted (GCE cell)" - - condition_threshold { - filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_postgres_retry_exhausted\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "180s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["resource.label.\"instance_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A Relay database transaction remained unsuccessful after bounded whole-transaction retry. Check Cloud SQL deadlocks/locks and customer-visible request failures before changing cell admission." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_incident] -} - -resource "google_monitoring_alert_policy" "relay_cloud_sql_backends" { - project = var.project_id - display_name = "Orca Relay: Cloud SQL connection headroom" - combiner = "OR" - enabled = true - - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Cloud SQL backends above 320" - - condition_threshold { - filter = "resource.type=\"cloudsql_database\" AND resource.label.\"database_id\"=\"${var.project_id}:${local.relay_database_instance_name}\" AND metric.type=\"cloudsql.googleapis.com/database/postgresql/num_backends\"" - comparison = "COMPARISON_GT" - threshold_value = 320 - duration = "300s" - - aggregations { - alignment_period = "60s" - per_series_aligner = "ALIGN_MAX" - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "Cloud SQL connections exceeded 80% of the 400-connection ceiling. Pause Relay pool or cell growth and inspect pool waits before the modeled 385-connection operating maximum is reached." - mime_type = "text/markdown" - } -} - -resource "google_monitoring_alert_policy" "relay_cloud_sql_checkpoint_loop" { - project = var.project_id - display_name = "Orca Relay: Cloud SQL checkpoint loop" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "WAL-triggered checkpoints above 3 in 5 minutes" - - condition_threshold { - filter = "resource.type=\"cloudsql_database\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cloud_sql_wal_checkpoint\"" - comparison = "COMPARISON_GT" - threshold_value = 3 - duration = "300s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "Healthy operation is one timed checkpoint every 5 minutes. Repeated `checkpoint starting: wal` lines mean WAL is outrunning `max_wal_size` and every checkpoint fsync stalls all relay SQL for seconds. Check `checkpoint complete` sync= times and disk write throughput against the PD-SSD ceiling; the fix is disk size and `max_wal_size` in the Terraform root that owns the instance (orca-cloud `infra/terraform-foundation`)." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_incident] -} - -resource "google_monitoring_alert_policy" "relay_cloud_sql_disk" { - project = var.project_id - display_name = "Orca Relay: Cloud SQL disk utilization" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Cloud SQL disk above 70%" - - condition_threshold { - filter = "resource.type=\"cloudsql_database\" AND resource.label.\"database_id\"=\"${var.project_id}:${local.relay_database_instance_name}\" AND metric.type=\"cloudsql.googleapis.com/database/disk/utilization\"" - comparison = "COMPARISON_GT" - threshold_value = 0.7 - duration = "600s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_MAX" - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "The shared auth/relay Cloud SQL disk is filling. `refresh_tokens` is the largest table and grows without pruning; grow the disk (IOPS scale with size) before it reaches the WAL checkpoint loop, and prune revoked token rows." - mime_type = "text/markdown" - } -} - -resource "google_monitoring_alert_policy" "relay_cloud_nat_port_drops" { - count = local.relay_gce_configured ? 1 : 0 - - project = var.project_id - display_name = "Orca Relay: Cloud NAT port exhaustion" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "NAT packets dropped for lack of ports" - - condition_threshold { - filter = "resource.type=\"nat_gateway\" AND resource.label.\"gateway_name\"=monitoring.regex.full_match(\"${local.relay_gce_name}(-.*)?\") AND metric.type=\"router.googleapis.com/nat/dropped_sent_packets_count\" AND metric.label.\"reason\"=\"OUT_OF_RESOURCES\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "120s" - - aggregations { - alignment_period = "60s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["resource.label.\"gateway_name\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "Relay cells reach Cloud SQL's public IP through this NAT. Port exhaustion makes every cell's Cloud SQL Auth Proxy dial time out at once, which reads as a fleet-wide SQL stall with a healthy database. Check `nat/port_usage` per VM and raise `max_ports_per_vm` in `relay-gce-foundation.tf`, or move the database to a private IP." - mime_type = "text/markdown" - } -} - -resource "google_monitoring_alert_policy" "relay_cell_process_exit" { - project = var.project_id - display_name = "Orca Relay: cell process exits" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Cell container exits above 3 in 15 minutes" - - condition_threshold { - filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_process_exit\"" - comparison = "COMPARISON_GT" - threshold_value = 3 - duration = "0s" - - aggregations { - alignment_period = "900s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["resource.label.\"instance_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A Relay GCE cell restarted its container more than three times in 15 minutes. Each exit drops every host and phone on that cell, and 201 exits went unpaged over 48 h on 2026-09-04. The instance hostname is `relay--`; read `jsonPayload.MESSAGE` on `cos_system` for the exit code and the container's own stderr for the stack before blaming MIG autoheal or load. A same-capacity roll is the remedy when the running image is behind." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_incident] -} - -# One count per runtime sample whose interval held a 5 s or longer freeze. A count, not a distribution: -# a percentile of the max would hide a single freeze among healthy samples. Gated until staging -# has proved it (variables.tf). -resource "google_logging_metric" "relay_event_loop_stall" { - count = var.relay_event_loop_stall_alert ? 1 : 0 - - project = var.project_id - name = "orca_relay_event_loop_stalls" - description = "Runtime samples whose interval held an event-loop freeze of at least 5,000 ms." - filter = "${local.relay_runtime_log_filter} AND jsonPayload.eventLoopDelayMsMax>=5000" - label_extractors = { - role = "EXTRACT(jsonPayload.role)" - cell_id = "EXTRACT(jsonPayload.cellId)" - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "INT64" - unit = "1" - - labels { - key = "role" - value_type = "STRING" - description = "Relay process role." - } - - labels { - key = "cell_id" - value_type = "STRING" - description = "Durable relay cell identifier, or director." - } - } -} - -resource "google_monitoring_alert_policy" "relay_event_loop_stall" { - count = var.relay_event_loop_stall_alert ? 1 : 0 - - project = var.project_id - display_name = "Orca Relay: event loop stall" - combiner = "OR" - enabled = true - # Not paging until staging has shown its noise floor. - notification_channels = [] - - dynamic "conditions" { - for_each = { - "Cloud Run" = "cloud_run_revision" - "GCE cell" = "gce_instance" - } - - content { - display_name = "Event loop froze at least 5,000 ms (${conditions.key})" - - condition_threshold { - # The metric counts only samples at or over 5,000 ms, so any count is a breach. - filter = "resource.type=\"${conditions.value}\" AND metric.type=\"logging.googleapis.com/user/${google_logging_metric.relay_event_loop_stall[0].name}\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "60s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["metric.label.\"role\"", "metric.label.\"cell_id\""] - } - - trigger { - count = 1 - } - } - } - } - - documentation { - content = "A relay process froze its event loop for 5 s or more. While frozen it serves nothing, polls nothing and no timeout fires, so assigns hang and reserve cells lose their dead-man feed. On a director, look for Cloud SQL connection failures in the same window: a connect on Cloud Run's built-in /cloudsql socket blocks the loop until the connector's dial ends (staging 2026-10-11, production 2026-09-28). Otherwise look for a long synchronous task." - mime_type = "text/markdown" - } -} - -# Pages on cell holds only: director holds of 1-2.5 s recur several times a day with rehoming paused, -# and pausing rehome does not stop them, so paging on them would ask on-call to do nothing. -resource "google_monitoring_alert_policy" "relay_long_cell_inventory_hold" { - project = var.project_id - display_name = "Orca Relay: cell table lock held over 1 second" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Cell table lock held at least 1,000 ms (GCE cell)" - - condition_threshold { - # The metric filter already drops samples under 1,000 ms, so any value present is a breach. - filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/${google_logging_metric.relay_long_cell_inventory_hold.name}\" AND metric.label.\"role\"=\"cell\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "60s" - per_series_aligner = "ALIGN_PERCENTILE_99" - cross_series_reducer = "REDUCE_MAX" - group_by_fields = ["metric.label.\"cell_id\""] - } - - trigger { - count = 1 - } - } - } - - conditions { - display_name = "Cell table lock held at least 1,000 ms (Cloud Run cell)" - - condition_threshold { - filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/${google_logging_metric.relay_long_cell_inventory_hold.name}\" AND metric.label.\"role\"=\"cell\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "60s" - per_series_aligner = "ALIGN_PERCENTILE_99" - cross_series_reducer = "REDUCE_MAX" - group_by_fields = ["metric.label.\"cell_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A relay cell held the lock on every row of the relay cell table for at least one second. While it is held, host connects and reservation changes on every cell wait and then fail, so each hold is a fleet-wide stall of that length. Between 2026-09-20 and 2026-09-22 every cell hold came from a regional rehome committed on an asia-east2 cell (c27, c28 or c29), about 3.6 s each. First response: pause regional rehoming with the `Operate Relay Production Rehome` workflow, action `pause`. See `cloud/docs/relay-incident-monitor.md`, section \"Relay lock contention alert policies\"." - mime_type = "text/markdown" - } -} - -# Visibility only, no notification channel, like the non-paging relay_custom policies. -resource "google_monitoring_alert_policy" "relay_director_cell_inventory_hold" { - project = var.project_id - display_name = "Orca Relay: director cell table lock held over 1 second" - combiner = "OR" - enabled = true - notification_channels = [] - - conditions { - display_name = "Director cell table lock held at least 1,000 ms" - - condition_threshold { - filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/${google_logging_metric.relay_long_cell_inventory_hold.name}\" AND metric.label.\"role\"=\"director\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "60s" - per_series_aligner = "ALIGN_PERCENTILE_99" - cross_series_reducer = "REDUCE_MAX" - group_by_fields = ["metric.label.\"cell_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "The director held the lock on every row of the relay cell table for at least one second. These holds happen a few times a day even with regional rehoming paused, and pausing rehome does not stop them, so this policy does not page. Use it to explain a `Orca Relay: lock timeout burst` alert that has no cell hold in the same minute. Do not drain or restart cells for it." - mime_type = "text/markdown" - } -} - -# The bar lives in the filter, as for the long-hold metric: an exponential bucket's percentile -# interpolation would put any value from 128 to 255 above 200. Why 200: a healthy cell peaks at -# 2 waiters (p99), but single-cell stalls of 50-196 recur several times a day and already time -# out. 200 keeps those for the SQL-failure alerts and pages on herds. Since 2026-10-03 a bar of 50 -# was crossed in 84 episodes, 150 in 29, and 200 in 10, among them every known herd (10-03 17:50, -# 10-05 06:26 and 18:22, 10-06 01:33 and 18:34, 10-07 20:00). -resource "google_logging_metric" "relay_cell_pool_herd" { - project = var.project_id - name = "orca_relay_cell_pool_herd" - description = "Cell runtime samples in which more than 200 requests queued at once for a PostgreSQL connection." - filter = "${local.relay_runtime_log_filter} AND jsonPayload.role=\"cell\" AND jsonPayload.databasePoolWaitersMax>200" - - metric_descriptor { - metric_kind = "DELTA" - value_type = "INT64" - unit = "1" - } -} - -resource "google_monitoring_alert_policy" "relay_cell_control_close_burst" { - project = var.project_id - display_name = "Orca Relay: cell desktop disconnect burst" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Over 120 abnormal desktop control closes on one cell in a minute" - - condition_threshold { - filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_control_abnormal_closes\"" - comparison = "COMPARISON_GT" - threshold_value = 120 - duration = "0s" - - aggregations { - alignment_period = "60s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["resource.label.\"instance_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "Over 120 desktop control sockets on one relay cell ended without a close frame (code 1006) in one minute. On 2026-10-06 at 18:34 UTC the asia-east2 cells logged 213-247 each in the minute Google's load balancer ended 1,046 of their streams, against a baseline of 6-49 per minute (US cells: 17 or fewer, all codes). Smaller pulses of about 60 per cell (01:33 and 20:40 UTC that day) stay under the bar. First response: read the load balancer logs for the same minute by `receiveTimestamp`, not `timestamp` (the LB stamps a WebSocket with its start), and bucket `jsonPayload.statusDetails`. If they show `internal_error` across several cells at once, the proxy ended the streams: record the timestamps for the Google support case, and do not drain, restart or resize the cell. If `Orca Relay: cell database pool herd` fired in the same minute, the reconnecting desktops overran the cell's database pool. Otherwise look at the cell itself: a crash or restart, the event loop, or memory." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_incident] -} - -resource "google_monitoring_alert_policy" "relay_cell_pool_herd" { - project = var.project_id - display_name = "Orca Relay: cell database pool herd" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Over 200 requests queued for a PostgreSQL connection on one cell" - - condition_threshold { - # The metric filter already drops samples at or under 200, so any value present is a breach. - filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/${google_logging_metric.relay_cell_pool_herd.name}\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "60s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["resource.label.\"instance_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "More than 200 requests on one relay cell queued at once for a PostgreSQL connection, and requests that wait 2 s fail. A healthy cell peaks at 2 waiters (p99 on 2026-10-07). Single-cell stalls of 50-196 waiters recur several times a day, mostly on the asia-east2 cells, whose every query crosses the Pacific. Each is about 30 seconds long and already timing requests out (96-367 SQL failures per 30-second sample at 184-196 on 2026-10-07). This bar leaves those to the SQL-failure alerts. Reconnect herds go well past 200 on several cells at once (2026-10-06 18:34 UTC: 233-527 on all five asia-east2 cells). Since 2026-10-03 this bar was crossed in 10 episodes. Six were herds across several cells, and four were a single cell at 201-228. This uses the interval maximum (`databasePoolWaitersMax`), so a herd shorter than the 30-second sample still shows. First response: check `Orca Relay: cell desktop disconnect burst` for the same minute. If it fired too, a mass disconnect caused this, so follow that policy and leave the database alone. If it did not, check Cloud SQL health and lock contention (`Orca Relay: lock timeout burst`) before touching the cell." - mime_type = "text/markdown" - } -} - -resource "google_monitoring_alert_policy" "relay_lock_timeout_burst" { - project = var.project_id - display_name = "Orca Relay: lock timeout burst" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Relay lock timeouts at least 20 in a minute" - - condition_threshold { - filter = "resource.type=\"cloudsql_database\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cloud_sql_lock_timeouts\"" - comparison = "COMPARISON_GT" - threshold_value = 19 - duration = "0s" - - aggregations { - alignment_period = "60s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "Postgres cancelled at least 20 relay statements in one minute because they waited too long for a lock. Each cancel is a host connect, reservation or placement that failed and retried. Only the relay database is counted; auth traffic on the same instance and fail-fast lock refusals from background sweeps are excluded. Between 2026-09-20 and 2026-09-22 all 88 such minutes overlapped a lock hold from a regional rehome on an asia-east2 cell. A burst can also come from a director hold while rehoming is paused, as on 2026-09-23 at 04:23 UTC. First response: check the `Orca Relay: cell table lock held over 1 second` policy for the same minute. If an asia-east2 cell held the lock, pause regional rehoming with the `Operate Relay Production Rehome` workflow, action `pause`. If no cell hold appears in that minute, check `Orca Relay: director cell table lock held over 1 second` instead of pausing rehome, and do not drain or restart cells for it. See `cloud/docs/relay-incident-monitor.md`, section \"Relay lock contention alert policies\"." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_incident] -} - -# Why: nothing fired while US desktops sat on asia-east2 cells for weeks in 2026-08. The two -# per-cell policies below read that as distance, and the fleet-wide one reads it as a bad region -# hint. All three are MQL because each needs the sum of a DELTA DISTRIBUTION as a volume floor, -# and the only scalar aligners a `condition_threshold` can apply to a distribution are percentiles. -# `join` is an inner join and the relay omits its percentile fields on an empty interval, so an -# idle cell drops out rather than alerting on nothing. The per-cell arms fetch `gce_instance` -# only: production runs no Cloud Run cells (`relay_cells` is empty), and a future one would need -# its own arm here. None of the metrics these query exist in the project yet, so what was checked -# against production is the query shape: the same MQL run over existing metrics of the same kind -# confirmed the distribution sum, the join arity, the unit literals, and the condition clause. -resource "google_monitoring_alert_policy" "relay_far_cell_accept_latency" { - project = var.project_id - display_name = "Orca Relay: far-cell phone accept latency" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Phone accept p95 above 2 s for 15 minutes" - - condition_monitoring_query_language { - # percentile(..., 50) over the window, not max: the published value is already a p95, so the - # median of the interval p95s reads as sustained slowness instead of one bad 30-second flush. - query = <<-EOT - { - fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accept_total_ms_p95 - | align delta(15m) | every 15m - | group_by [metric.cell_id], [accept_p95_ms: percentile(value.orca_relay_client_accept_total_ms_p95, 50)] - ; - fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accepts_completed - | align delta(15m) | every 15m - | group_by [metric.cell_id], [accepts: sum(value.orca_relay_client_accepts_completed)] - } - | join - | condition accept_p95_ms > 2000 'ms' && accepts >= 20 '1' - EOT - duration = "0s" - - trigger { - count = 1 - } - } - } - - documentation { - content = "Phones on this cell are taking over two seconds to reach relay-hello. Measured separation: an in-region accept completes in 0.3-0.6 s and a cross-Pacific one in 5-10 s, so 2 s sits well outside in-region noise and well below the far-cell floor. The 20-accept floor over 15 minutes keeps a single slow accept on a quiet cell from paging. Check which regions the cell's hosts are actually in before touching capacity: the 2026-08 cause was desktops requesting the wrong region, not a slow cell. Read the per-stage `orca_relay_client_accept_*_ms_p95` metrics to separate distance from assignment, credential, or attach work." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_snapshot] -} - -resource "google_monitoring_alert_policy" "relay_cell_control_rtt" { - project = var.project_id - display_name = "Orca Relay: cell control round trip" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Control ping p50 above 150 ms for an hour" - - condition_monitoring_query_language { - # p50 only. The desktop echoes the pong on its main thread, so the published p95 and max - # track renderer stalls, not distance; the median is the only column that reads as distance. - query = <<-EOT - { - fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_ms_p50 - | align delta(1h) | every 1h - | group_by [metric.cell_id], [control_rtt_p50_ms: percentile(value.orca_relay_control_rtt_ms_p50, 50)] - ; - fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_samples - | align delta(1h) | every 1h - | group_by [metric.cell_id], [samples: sum(value.orca_relay_control_rtt_samples)] - } - | join - | condition control_rtt_p50_ms > 150 'ms' && samples >= 500 '1' - EOT - duration = "0s" - - trigger { - count = 1 - } - } - } - - documentation { - content = "The median desktop on this cell is more than 150 ms away from it, which is a mis-homed population rather than a cell fault: an in-region control ping is tens of milliseconds and a US desktop on an asia-east2 cell is 200 ms or more. This is the signal that was missing while roughly 226 of 332 hosts on the asia cells were non-APAC for weeks in 2026-08. Confirm with the assignment table which regions those hosts requested, then rehome; do not restart or drain the cell on this alert alone. The 500-sample floor is about two continuously connected hosts at the 15-second control ping, so a nearly idle cell cannot alert on one desktop. Tuning risk: EU desktops on us-central1 sit at 100-130 ms, so a cell whose population is mostly European can approach 150 ms while correctly homed. Check where the hosts are before treating a first breach as mis-homing, and raise the bar only with that evidence." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_snapshot] -} - -resource "google_monitoring_alert_policy" "relay_region_hint_skew" { - project = var.project_id - display_name = "Orca Relay: region hint skew" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "asia-east2 hint share above 2x its placement share for an hour" - - condition_monitoring_query_language { - query = local.relay_region_hint_skew_query - duration = "0s" - - trigger { - count = 1 - } - } - } - - documentation { - content = "Desktops are asking the director for asia-east2 far more often than the director actually places them there, which is what silently homed US desktops on asia cells through 2026-08. The alert compares two shares of the same hour and never an absolute share, because an absolute bar is wrong at both ends: measured over twelve hours on 2026-09-07, while the desktop region probe was still mis-picking, asia-east2 was 33.8% of the 33,800 hinted requests but only 7.9% of the 45,364 assignments, and once the probe is fixed the genuine APAC share will climb past any fixed bar that would have caught this. Divergence was 4.27x with a 25.9-point gap, so the 2x and 15-point bars sit well inside the broken state and well outside a healthy one. `unhinted` requests are excluded from the denominator: they were 27% of all requests, and a client change that always sends a hint would move this number without any behaviour changing. Expect this to stay lit until the mis-homed backlog is rehomed, because sticky assignment never re-consults the hint, so a desktop already on an asia cell keeps being placed there no matter what it now asks for. Investigate the desktop region probe first, not relay placement." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_snapshot] -} - -# Declared for a targeted apply after the step-2 wave. Cells only: a director-only fix must not -# raise the level the cells are held to. sum / count of the distribution is the exact level. -resource "google_logging_metric" "relay_cell_fix_level" { - project = var.project_id - name = "orca_relay_cell_fix_level" - description = "RELAY_FIX_LEVEL reported by each cell's runtime metrics line." - filter = "${local.relay_runtime_log_filter} AND jsonPayload.role=\"cell\" AND jsonPayload.fixLevel:*" - value_extractor = "EXTRACT(jsonPayload.fixLevel)" - label_extractors = { - cell_id = "EXTRACT(jsonPayload.cellId)" - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "DISTRIBUTION" - unit = "1" - - labels { - key = "cell_id" - value_type = "STRING" - description = "Durable relay cell identifier." - } - } - - bucket_options { - linear_buckets { - num_finite_buckets = 64 - width = 1 - offset = 0 - } - } -} - -locals { - relay_cell_fix_level_hourly = "(sum by (cell_id) (increase(logging_googleapis_com:user_orca_relay_cell_fix_level_sum{monitored_resource=\"gce_instance\"}[1h])) / sum by (cell_id) (increase(logging_googleapis_com:user_orca_relay_cell_fix_level_count{monitored_resource=\"gce_instance\"}[1h])))" - relay_cell_serving = "(sum by (cell_id) (increase(logging_googleapis_com:user_orca_relay_controls_sum{monitored_resource=\"gce_instance\",role=\"cell\"}[1h])) > 0)" -} - -# One PromQL condition per policy, and log-based metrics allow at most 25 h of lookback, so the -# floor is a fixed variable rather than a fleet maximum: raise it with a targeted apply once a -# wave has rolled every serving cell. Images from before the field report no level at all. -resource "google_monitoring_alert_policy" "relay_cell_outdated_fix_level" { - project = var.project_id - display_name = "Orca Relay: cell left on an outdated image" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Serving cell below fix level ${var.relay_cell_min_fix_level} for 6 hours" - - condition_prometheus_query_language { - query = "((${local.relay_cell_fix_level_hourly} < ${var.relay_cell_min_fix_level}) or (${local.relay_cell_serving} unless on (cell_id) ${local.relay_cell_fix_level_hourly})) and on (cell_id) ${local.relay_cell_serving}" - duration = "21600s" - } - } - - documentation { - content = "A cell holding desktops runs an image below `relay_cell_min_fix_level`, or one too old to report `fixLevel`. On 2026-09-28, 18 cells still ran images without the pg connection-error fix and crashed in a two-minute database failover, dropping ~16.3k hosts. Roll the named cell with a same-capacity roll to the current digest. Empty cells do not fire because they hold no controls. Raise the floor only after a wave has rolled every serving cell." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_cell_fix_level] -} - -# Why: the four signals that had to be assembled by hand during the 2026-09-04 incident. -resource "google_monitoring_dashboard" "relay_incident" { - project = var.project_id - - dashboard_json = jsonencode({ - displayName = "Orca Relay: incident overview" - mosaicLayout = { - columns = 12 - tiles = [ - { - xPos = 0 - yPos = 0 - width = 3 - height = 4 - widget = { - title = "Cloud SQL WAL checkpoints" - xyChart = { - dataSets = [{ - plotType = "LINE" - targetAxis = "Y1" - timeSeriesQuery = { - timeSeriesFilter = { - filter = "metric.type=\"logging.googleapis.com/user/orca_relay_cloud_sql_wal_checkpoint\" AND resource.type=\"cloudsql_database\"" - aggregation = { - alignmentPeriod = "300s" - perSeriesAligner = "ALIGN_SUM" - crossSeriesReducer = "REDUCE_SUM" - } - } - } - }] - yAxis = { - label = "checkpoints" - scale = "LINEAR" - } - } - } - }, - { - xPos = 3 - yPos = 0 - width = 3 - height = 4 - widget = { - title = "Cloud NAT dropped packets" - xyChart = { - dataSets = [{ - plotType = "LINE" - targetAxis = "Y1" - timeSeriesQuery = { - timeSeriesFilter = { - filter = "metric.type=\"router.googleapis.com/nat/dropped_sent_packets_count\" AND resource.type=\"nat_gateway\" AND resource.label.\"gateway_name\"=monitoring.regex.full_match(\"${local.relay_gce_name}(-.*)?\")" - aggregation = { - alignmentPeriod = "60s" - perSeriesAligner = "ALIGN_SUM" - crossSeriesReducer = "REDUCE_SUM" - groupByFields = ["resource.label.\"gateway_name\"", "metric.label.\"reason\""] - } - } - } - }] - yAxis = { - label = "packets" - scale = "LINEAR" - } - } - } - }, - { - xPos = 6 - yPos = 0 - width = 3 - height = 4 - widget = { - title = "Auth refresh 401s" - xyChart = { - dataSets = [{ - plotType = "LINE" - targetAxis = "Y1" - timeSeriesQuery = { - timeSeriesFilter = { - filter = "metric.type=\"logging.googleapis.com/user/orca_auth_refresh_401\"" - aggregation = { - alignmentPeriod = "300s" - perSeriesAligner = "ALIGN_SUM" - crossSeriesReducer = "REDUCE_SUM" - } - } - } - }] - yAxis = { - label = "rejections" - scale = "LINEAR" - } - } - } - }, - { - xPos = 9 - yPos = 0 - width = 3 - height = 4 - widget = { - title = "Standing desktop controls (fleet sum)" - xyChart = { - dataSets = [{ - plotType = "LINE" - targetAxis = "Y1" - timeSeriesQuery = { - timeSeriesFilter = { - # ALIGN_MEAN, not ALIGN_SUM: each process reports its standing control count once per interval. - filter = "metric.type=\"logging.googleapis.com/user/orca_relay_controls\"" - aggregation = { - alignmentPeriod = "300s" - perSeriesAligner = "ALIGN_MEAN" - crossSeriesReducer = "REDUCE_SUM" - } - } - } - }] - yAxis = { - label = "controls" - scale = "LINEAR" - } - } - } - } - ] - } - }) - - depends_on = [google_logging_metric.relay_incident, google_logging_metric.relay_snapshot] -} - -# ticketCheck=shadow (assignment-lease-shadow.ts): one line per cell per minute with traffic. -# The lease never decides an admission, so these observe only. -locals { - relay_assignment_lease_shadow_metrics = { - agree = { field = "classes.agree", description = "Host hellos whose echoed lease was valid and the database admitted." } - disagree = { field = "classes.disagree", description = "Host hellos whose echoed lease was valid but the database refused; step 5 must explain each before a lease alone admits." } - absent = { field = "classes.absent", description = "Host hellos with no echoed lease (desktops that predate the header)." } - expired = { field = "classes.expired", description = "Host hellos whose echoed lease had expired." } - bad_signature = { field = "classes.badSignature", description = "Host hellos whose echoed lease failed signature, key id, or shape checks." } - wrong_host = { field = "classes.wrongHost", description = "Host hellos whose echoed lease named another user or host." } - wrong_cell = { field = "classes.wrongCell", description = "Host hellos whose echoed lease named another cell; expected during drains." } - epoch_behind = { field = "classes.epochBehind", description = "Host hellos whose echoed lease was older than the hello's assignment epoch." } - epoch_ahead = { field = "classes.epochAhead", description = "Host hellos whose echoed lease was newer than the hello's assignment epoch." } - db_refused = { field = "dbRefused", description = "Host hellos the database refused, whatever the echoed lease said." } - db_read_ms = { field = "dbReadMs.p99", description = "p99 of the hello's assignment database read, the read a lease-admitted hello would skip." } - } -} - -resource "google_logging_metric" "relay_assignment_lease_shadow" { - for_each = local.relay_assignment_lease_shadow_metrics - - project = var.project_id - name = "orca_relay_assignment_lease_shadow_${each.key}" - description = each.value.description - filter = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_assignment_lease_shadow\"" - value_extractor = "EXTRACT(jsonPayload.${each.value.field})" - label_extractors = { - cell_id = "EXTRACT(jsonPayload.cellId)" - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "DISTRIBUTION" - unit = each.key == "db_read_ms" ? "ms" : "1" - - labels { - key = "cell_id" - value_type = "STRING" - description = "Durable relay cell identifier." - } - } - - bucket_options { - exponential_buckets { - num_finite_buckets = 24 - growth_factor = 2 - scale = 1 - } - } -} - -# Directors only: a cell's fatal rejection ends in a container exit, which -# relay_cell_process_exit already pages on, so paging here too would double every cell incident. -# The counter still includes cells, for diagnosis. -resource "google_monitoring_alert_policy" "relay_process_fatal" { - project = var.project_id - display_name = "Orca Relay: fatal unhandled rejection on a director" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Fatal unhandled rejection (Cloud Run)" - - condition_threshold { - filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_process_fatal\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["resource.label.\"service_name\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A director logged `orca_relay_process_fatal` and its instance exited: an unhandled rejection that is not a database fault (a programming error, a schema error, or a bare socket errno). Read the line's `code` and `message`, then the instance stderr stack. Cells log the same line before exiting; the `cell process exits` alert pages for them." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_incident] -} - -# Summed across every process: one database fault reaches every cell and director at once, and -# per-instance series would open ~30 incidents for one cause. -resource "google_monitoring_alert_policy" "relay_database_rejection_fenced" { - project = var.project_id - display_name = "Orca Relay: database rejection survived without a handler" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "Fenced database rejections, fleet-wide" - - # One series across directors and cells, which a single threshold filter cannot span. - condition_prometheus_query_language { - query = "sum(increase(logging_googleapis_com:user_orca_relay_database_rejection_fenced[5m])) > 0" - duration = "0s" - } - } - - documentation { - content = "A database fault reached `unhandledRejection` and the process survived it (`orca_relay_database_rejection_fenced`). Nothing crashed, but some caller started a database promise without awaiting it. Group the lines by `code`, `message` and instance to find it, and fix it. Before the fence, this was a process exit." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_incident] -} - -# Console only: a single desktop sending a damaged lease can hold this up, so it is a lead to -# check, never a page. A fleet-wide key or key-id mismatch reads as every cell at once. -resource "google_monitoring_alert_policy" "relay_assignment_lease_bad_signature" { - project = var.project_id - display_name = "Orca Relay: assignment lease shadow sees bad signatures" - combiner = "OR" - enabled = true - notification_channels = [] - - conditions { - display_name = "Bad lease signatures on a cell for 15 minutes" - - # The class totals are distributions (a value extractor needs one), so the sum is the count. - condition_prometheus_query_language { - query = "sum by (cell_id) (increase(logging_googleapis_com:user_orca_relay_assignment_lease_shadow_bad_signature_sum{monitored_resource=\"gce_instance\"}[15m])) > 0" - duration = "900s" - } - } - - documentation { - content = "A cell with `ticketCheck: shadow` kept seeing assignment leases that fail verification for 15 minutes. The database still decides every admission, so no desktop is affected. One misbehaving client can cause this on one cell; many cells at once points to a key or key-id mismatch between directors and cells, which would block `ticketCheck: enforce`. Compare the `kid` the directors issue with `k1-` on the cell." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_assignment_lease_shadow] -} - -# relay-control-flag-channel.ts logs one line per change of cause, not per failed poll, and nothing -# on recovery: a cell stuck unreadable writes exactly one line, so any count is the signal. -resource "google_logging_metric" "relay_control_flags_unreadable" { - project = var.project_id - name = "orca_relay_control_flags_unreadable" - description = "Cells that started failing to read their switch file (cells/.json in the relay-control bucket). The cell keeps its last applied switches." - filter = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_control_flags_unreadable\"" - label_extractors = { - cell_id = "REGEXP_EXTRACT(jsonPayload.object, \"cells/([^.]+)\")" - failure = "EXTRACT(jsonPayload.failure)" - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "INT64" - unit = "1" - - labels { - key = "cell_id" - value_type = "STRING" - description = "Durable relay cell identifier." - } - labels { - key = "failure" - value_type = "STRING" - description = "metadata, not-found, http, network, malformed, or void (the object failed the parser)." - } - } -} - -# Console only until the step-3/5 roll finishes: a single storage 5xx on one cell also writes a line. -resource "google_monitoring_alert_policy" "relay_control_flags_unreadable" { - project = var.project_id - display_name = "Orca Relay: cell cannot read its switch file" - combiner = "OR" - enabled = true - notification_channels = [] - - conditions { - display_name = "Switch-file read failures on a cell" - - condition_threshold { - filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_control_flags_unreadable\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["metric.label.\"cell_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A cell logged `orca_relay_control_flags_unreadable`: its read of `cells/.json` in `-relay-control` failed, and it keeps its last applied switches (defaults, all off, if it never read one). The line is written once per change of cause and nothing marks recovery, so this incident closing does not mean the cell recovered: compare the object's generation with the cell's last `orca_relay_cell_flags_applied` line. `failure` says why: `not-found` (no object for this cell), `metadata` or `http` (token, IAM, or storage), `network`, `malformed` or `void` (the object is unparseable or failed the schema; fix the object). Many cells at once points to the bucket or its IAM. A one-off `http` or `network` that the next poll cleared needs nothing." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_control_flags_unreadable] -} - -# cell-reserve-dead-man.ts logs once per switch-file generation when it trips, so any count is the signal. -resource "google_logging_metric" "relay_cell_reserve_dead_man_tripped" { - project = var.project_id - name = "orca_relay_cell_reserve_dead_man_tripped" - description = "Reserve-mode cells that heard from no placing director for their dead-man window (60-120 s) and went back to admitting through the database." - filter = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_cell_reserve_dead_man_tripped\"" - label_extractors = { - cell_id = "EXTRACT(jsonPayload.cellId)" - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "INT64" - unit = "1" - - labels { - key = "cell_id" - value_type = "STRING" - description = "Durable relay cell identifier." - } - } -} - -resource "google_monitoring_alert_policy" "relay_cell_reserve_dead_man_tripped" { - project = var.project_id - display_name = "Orca Relay: reserve cell lost its placing director" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "A reserve-mode cell tripped its dead-man" - - condition_threshold { - filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_reserve_dead_man_tripped\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["metric.label.\"cell_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A cell with `admitMode=reserve` in its switch file heard from no placing director (no booking, no feed poll marked `reserver=1`) for its dead-man window, so it went back to database admission and is re-registering its controls. Nobody is disconnected. Postgres still records the cell as reserve, so sweeps keep skipping it while it is heartbeat-live. First find why placement stopped: a director rollback, placement turned off, or every director failing its feed polls. Then, once the cell's `runtime-status` reports `admitModeEffective=db`, run `cloud-operate-relay-production-cell-flags.yml` with `--set admitMode=db` for that cell: it writes db into the switch file and records db in Postgres, so sweeps resume. **This step is required, within hours.** Until it runs, sweeps skip the cell, so its `reserved_requests` and leases only rise, until the database path treats the cell as full. The trip is latched for this switch-file generation: the cell will not return to reserve until a new reserve write, even if a director comes back. Several cells at once points to the directors, not the cells." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_cell_reserve_dead_man_tripped] -} - -# The reserve watch workflow logs this from its runner (resource global) when a watch is refused or -# a flip back did not land in write mode: either can leave a cell in reserve unwatched. -resource "google_logging_metric" "relay_reserve_watch_alarm" { - project = var.project_id - name = "orca_relay_reserve_watch_alarm" - description = "Reserve watch runs that were refused or whose flip back to db did not land, so the cell may be in reserve mode with nothing watching it." - filter = "resource.type=\"global\" AND logName=\"projects/${var.project_id}/logs/orca-relay-reserve-watch\" AND jsonPayload.event=\"orca_relay_reserve_watch_alarm\"" - label_extractors = { - cell_id = "EXTRACT(jsonPayload.cellId)" - kind = "EXTRACT(jsonPayload.kind)" - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "INT64" - unit = "1" - - labels { - key = "cell_id" - value_type = "STRING" - description = "Durable relay cell identifier." - } - - labels { - key = "kind" - value_type = "STRING" - description = "refused or flip-back-failed." - } - } -} - -resource "google_monitoring_alert_policy" "relay_reserve_watch_alarm" { - project = var.project_id - display_name = "Orca Relay: reserve cell may be unwatched" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "A reserve watch was refused or its flip back did not land" - - condition_threshold { - filter = "resource.type=\"global\" AND metric.type=\"logging.googleapis.com/user/orca_relay_reserve_watch_alarm\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["metric.label.\"cell_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A write-mode `Operate Relay Production Reserve Watch` run was refused (a rollout held the Cloud SQL rollout lease, the lease was unreadable, bad inputs, or the cell never reported reserve within 10 minutes) or its flip back to db did not land (no ID token, a failed login, or a cell that does not answer). The cell may be in reserve mode with nothing watching it. Read the cell's `runtime-status`; if its switch file says reserve, either rerun the watch or run `cloud-operate-relay-production-cell-flags.yml` with `admitMode=db` for that cell, using the break-glass inputs the failed run printed if the cell does not answer." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_reserve_watch_alarm] -} - -# host-session-registry.ts: a flip back closed a control it could not re-register (4409), after 20 refusals. -resource "google_logging_metric" "relay_cell_reregistration_gave_up" { - project = var.project_id - name = "orca_relay_cell_reregistration_gave_up" - description = "Controls a cell closed with 4409 during a flip back to db, because the database kept refusing their lease." - filter = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_cell_reregistration_gave_up\"" - label_extractors = { - cell_id = "EXTRACT(jsonPayload.cellId)" - reason = "EXTRACT(jsonPayload.reason)" - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "INT64" - unit = "1" - - labels { - key = "cell_id" - value_type = "STRING" - description = "Durable relay cell identifier." - } - labels { - key = "reason" - value_type = "STRING" - description = "The refusal: wrong_assignment (the row names another seat) or another database answer." - } - } -} - -resource "google_monitoring_alert_policy" "relay_cell_reregistration_gave_up" { - project = var.project_id - display_name = "Orca Relay: flip back closed a desktop it could not re-register" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "A cell closed a control during its flip back" - - condition_threshold { - filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_reregistration_gave_up\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["metric.label.\"cell_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "During a flip back to db, a cell asked the database for a control's lease 20 times and was refused each time, so it closed that desktop with 4409 and the desktop re-assigned. `reason` says why: `wrong_assignment` means the host's row names another seat (normally the ledger reconcile repairs that first); anything else is a database answer the lease path does not accept, which needs a code look. A handful on one flip is tolerable; a steady stream means the flip back is disconnecting desktops: stop flipping further cells." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_cell_reregistration_gave_up] -} - -# host-session-registry.ts: one line per 20 consecutive retries of a database that did not answer. -resource "google_logging_metric" "relay_cell_reregistration_stalled" { - project = var.project_id - name = "orca_relay_cell_reregistration_stalled" - description = "Controls whose flip-back lease has failed 20 times in a row because the database did not answer." - filter = "resource.type=\"gce_instance\" AND jsonPayload.event=\"orca_relay_cell_reregistration_stalled\"" - label_extractors = { - cell_id = "EXTRACT(jsonPayload.cellId)" - reason = "EXTRACT(jsonPayload.reason)" - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "INT64" - unit = "1" - - labels { - key = "cell_id" - value_type = "STRING" - description = "Durable relay cell identifier." - } - labels { - key = "reason" - value_type = "STRING" - description = "The last database error." - } - } -} - -resource "google_monitoring_alert_policy" "relay_cell_reregistration_stalled" { - project = var.project_id - display_name = "Orca Relay: flip back stalled on the database" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "A cell keeps retrying flip-back leases against an unanswering database" - - condition_threshold { - filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_reregistration_stalled\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["metric.label.\"cell_id\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A cell flipping back to db has retried a control's lease 20 times in a row because the database did not answer (a stall, dropped connections, a busy lock). Nobody is disconnected: the cell keeps every control and retries for as long as it is open, and it reports `admitModeEffective=reserve` until it is done, so Postgres keeps reserve and sweeps stay off it. Look at Cloud SQL first. The flag workflow's wait may time out meanwhile; re-run it with `--set admitMode=db` once the database recovers." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_cell_reregistration_stalled] -} - -# relay-server.ts (director): placement is off while the map shows reserve cells; logged once a minute. -resource "google_logging_metric" "relay_reserve_placement_off_with_reserve_cells" { - project = var.project_id - name = "orca_relay_reserve_placement_off_with_reserve_cells" - description = "Directors running with reserve placement off while some cells report reserve mode." - filter = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${var.relay_cloud_run_service_name}\" AND jsonPayload.event=\"orca_relay_reserve_placement_off_with_reserve_cells\"" - label_extractors = { - revision = "EXTRACT(resource.labels.revision_name)" - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "INT64" - unit = "1" - - labels { - key = "revision" - value_type = "STRING" - description = "Cloud Run revision of the director." - } - } -} - -resource "google_monitoring_alert_policy" "relay_reserve_placement_off_with_reserve_cells" { - project = var.project_id - display_name = "Orca Relay: director placement off while cells are in reserve" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "A director cannot place onto reserve cells" - - condition_threshold { - filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/orca_relay_reserve_placement_off_with_reserve_cells\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["metric.label.\"revision\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A director is not placing (`placement` in the line: off, dry-run, or configured on but never built; `configured` says what `ORCA_RELAY_RESERVE_PLACEMENT` asked for) while its map shows cells in reserve mode, so it books nothing there and leaves their hosts to the other directors. A director that predates step 5 logs nothing, so a rollback past it shows only as dead-man trips. If every director has placement off, each reserve cell's dead-man trips within 60-120 s (`Orca Relay: reserve cell lost its placing director`). If only some are (a partial rollback, or a manual traffic split), the rest keep the dead-man from tripping, and this alert is the only sign. Put every director on the step-5 revision with placement on, or flip the reserve cells to db (`cloud-operate-relay-production-cell-flags.yml --set admitMode=db`) before rolling the directors back." - mime_type = "text/markdown" - } - - depends_on = [google_logging_metric.relay_reserve_placement_off_with_reserve_cells] -} - -# cell-poll-reachability.ts: a reserve cell stopped answering polls, but the director would not call it -# dead (too few feed cells, or too many failing with it), so its hosts wait for an operator fence. -# Logged at most once a minute per cell per director. Gated until staging has proved it (variables.tf); -# apply the metric first, then the policy a few minutes later: Monitoring refuses a policy on a new -# log metric until the metric is visible, which the dependency ordering alone does not wait for. -resource "google_logging_metric" "relay_dead_cell_verdict_withheld" { - count = var.relay_dead_cell_verdict_alert ? 1 : 0 - - project = var.project_id - name = "orca_relay_dead_cell_verdict_withheld" - description = "Directors that saw a reserve cell stop answering their seat-feed polls but withheld the dead-cell verdict: too few feed cells to judge, a third of the fleet failing, or half its region." - filter = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${var.relay_cloud_run_service_name}\" AND jsonPayload.event=\"orca_relay_dead_cell_verdict_withheld\"" - label_extractors = { - cell_id = "EXTRACT(jsonPayload.cellId)" - withheld = "EXTRACT(jsonPayload.withheld)" - } - - metric_descriptor { - metric_kind = "DELTA" - value_type = "INT64" - unit = "1" - - labels { - key = "cell_id" - value_type = "STRING" - description = "Durable relay cell identifier." - } - labels { - key = "withheld" - value_type = "STRING" - description = "Which guard withheld the verdict: sample (fewer than 6 feed cells, or the cell is not among them), fleet (a third of all feed cells) or region (half the cell's region)." - } - } -} - -resource "google_monitoring_alert_policy" "relay_dead_cell_verdict_withheld" { - count = var.relay_dead_cell_verdict_alert ? 1 : 0 - - project = var.project_id - display_name = "Orca Relay: a dead-looking reserve cell was not called dead" - combiner = "OR" - enabled = true - notification_channels = var.relay_alert_notification_channels - - conditions { - display_name = "A dead-cell verdict was withheld" - - condition_threshold { - filter = "resource.type=\"cloud_run_revision\" AND metric.type=\"logging.googleapis.com/user/${google_logging_metric.relay_dead_cell_verdict_withheld[0].name}\"" - comparison = "COMPARISON_GT" - threshold_value = 0 - duration = "0s" - - aggregations { - alignment_period = "300s" - per_series_aligner = "ALIGN_SUM" - cross_series_reducer = "REDUCE_SUM" - group_by_fields = ["metric.label.\"withheld\""] - } - - trigger { - count = 1 - } - } - } - - documentation { - content = "A reserve cell's heartbeat went stale and a director's seat-feed polls to it kept failing, but the director withheld the dead-cell verdict because the poll failures were not that cell's alone: at least half of its region's feed cells were failing polls too (`withheld=region`, typically a us-central1 to asia-east2 partition), more than a third of all feed cells were (`withheld=fleet`, typically the director's own network or credential), or there were too few feed cells to judge or the cell was not among them (`withheld=sample`). Hosts that lost their control on that cell are not moved off automatically; they get `relay_home_cell_unavailable` until the cell answers again or an operator fences it. Cells only cut off from the directors keep serving the hosts still connected. Check director egress and which cells fail polls before fencing anything: fencing a cell that is still serving disconnects its hosts." - mime_type = "text/markdown" - } - -} diff --git a/cloud/infra/terraform/relay-shared.tf b/cloud/infra/terraform/relay-shared.tf deleted file mode 100644 index b1afc4d1890..00000000000 --- a/cloud/infra/terraform/relay-shared.tf +++ /dev/null @@ -1,90 +0,0 @@ -# Values the relay root shares with the foundation and apps roots, expressed as literals or -# data lookups so the relay never references another root's resources. Every literal here -# renders byte-identically to the resource attribute it replaces; the partition test pins that. -locals { - relay_shared_labels = { - app = "orca-cloud" - environment = var.environment - managed_by = "terraform" - } - relay_github_repository = "${var.github_owner}/${var.github_repo}" - relay_github_repository_claims = [ - "assertion.repository == '${local.relay_github_repository}'", - "assertion.repository_id == '${var.github_repo_id}'", - "assertion.repository_owner_id == '${var.github_owner_id}'", - ] - - # The primary repository first, then every repository var.github_accepted_repositories adds. - # Each one renders its own OR arm in every provider condition, so a repository move can trust - # both repos at once. A repository that imports these workflows may rename the files, hence the - # per-repository prefix; the primary's is var.github_workflow_file_prefix. - relay_github_accepted_repositories = concat([{ - owner = var.github_owner - repo = var.github_repo - repo_id = var.github_repo_id - owner_id = var.github_owner_id - workflow_file_prefix = var.github_workflow_file_prefix - }], var.github_accepted_repositories) - - relay_github_single_repository = length(local.relay_github_accepted_repositories) == 1 - - relay_github_accepted_repository_names = [ - for repository in local.relay_github_accepted_repositories : - "${repository.owner}/${repository.repo}" - ] - - # Everything before the workflow file name, per accepted repository. - relay_github_workflow_ref_prefixes = [ - for repository in local.relay_github_accepted_repositories : - "${repository.owner}/${repository.repo}/.github/workflows/${repository.workflow_file_prefix}" - ] - - # The three repository claims as one conjunction, per accepted repository, for the OR arms. - relay_github_accepted_repository_claims = [ - for repository in local.relay_github_accepted_repositories : - join(" && ", [ - "assertion.repository == '${repository.owner}/${repository.repo}'", - "assertion.repository_id == '${repository.repo_id}'", - "assertion.repository_owner_id == '${repository.owner_id}'" - ]) - ] - - # With one accepted repository the claims lead each condition exactly as they always have. With - # more they move inside the arms, because a leading claim would contradict the other arm. - relay_github_leading_repository_claims = ( - local.relay_github_single_repository ? local.relay_github_repository_claims : [] - ) - - relay_create_github_deploy_identity = var.github_owner != "" && var.github_repo != "" - relay_create_production_ops_identity = local.relay_create_github_deploy_identity && var.environment == "production" - - # google_service_account.github_deploy lives in the relay root in production and in the apps - # root in staging, so its email is derived rather than read, matching the runtime accounts above. - # Staging Relay workflows authenticate as the relay-owned github_staging_relay_deploy account - # instead, so every relay binding, the cell startup metadata, and the director env follow it. - relay_github_deploy_service_account_email = ( - var.environment == "production" - ? "${var.name_prefix}-gha-deploy@${var.project_id}.iam.gserviceaccount.com" - : "${var.name_prefix}-gha-relay@${var.project_id}.iam.gserviceaccount.com" - ) - relay_github_deploy_service_account_member = "serviceAccount:${local.relay_github_deploy_service_account_email}" - - relay_workload_identity_pool_id = "${var.name_prefix}-github" - relay_workload_identity_pool_name = "projects/${data.google_project.relay.number}/locations/global/workloadIdentityPools/${local.relay_workload_identity_pool_id}" - - # Cloud SQL instance is foundation-owned; cell plans already derive the connection name so - # they stay independent of database drift. - relay_database_instance_name = "${var.name_prefix}-auth-db" - relay_database_connection_name = "${var.project_id}:${var.region}:${local.relay_database_instance_name}" -} - -data "google_project" "relay" { - project_id = var.project_id -} - -# Existence checks only: nothing in a plan value depends on these reads. -data "google_artifact_registry_repository" "relay_images" { - project = var.project_id - location = var.region - repository_id = var.artifact_repository_id -} diff --git a/cloud/infra/terraform/relay-staging-deploy-iam.tf b/cloud/infra/terraform/relay-staging-deploy-iam.tf deleted file mode 100644 index 74ee413f109..00000000000 --- a/cloud/infra/terraform/relay-staging-deploy-iam.tf +++ /dev/null @@ -1,182 +0,0 @@ -# The staging Relay deploy identity. -# -# In production the shared `github_deploy` account is relay-owned; in staging it belongs to -# infra/terraform-apps and four app workflows can also mint it. These resources give the six -# staging Relay workflows their own account, provider, and bindings so the relay root owns every -# credential its own workflows use. -# -# Bindings that already name local.relay_github_deploy_service_account_member follow that local -# (relay-shared.tf) and are NOT repeated here: the staging power custom role, serviceAccountUser -# on relay_runtime and relay_director_runtime, the three regional-placement secret bindings, and -# the cell-flags workflow's relay_control_workflow_writer. -# Everything below replaces a grant the apps root still makes to `github_deploy`. - -locals { - create_staging_relay_deploy_identity = ( - local.relay_create_github_deploy_identity && var.environment == "staging" - ) - github_staging_relay_deploy_workflow_files = [ - "bootstrap-relay-staging-capacity.yml", - "deploy-relay-staging-gce-candidate.yml", - "deploy-relay-staging.yml", - "operate-relay-asia-admission.yml", - "operate-relay-staging-cell-flags.yml", - "power-relay-staging.yml" - ] - github_staging_relay_deploy_workflow_clauses = [ - for prefix in local.relay_github_workflow_ref_prefixes : - "(${join(" || ", [for workflow_file in local.github_staging_relay_deploy_workflow_files : "assertion.workflow_ref == '${prefix}${workflow_file}@refs/heads/main'"])})" - ] - # Apps-owned account the shared staging power workflow scales to zero alongside the director. - staging_auth_runtime_service_account_email = "${var.name_prefix}-auth@${var.project_id}.iam.gserviceaccount.com" -} - -resource "google_service_account" "github_staging_relay_deploy" { - count = local.create_staging_relay_deploy_identity ? 1 : 0 - - project = var.project_id - account_id = "${var.name_prefix}-gha-relay" - display_name = "Orca Relay staging deploy" - description = "Runs the exact reviewed Relay staging deploy, candidate, power, and admission workflows." -} - -resource "google_iam_workload_identity_pool_provider" "github_staging_relay_deploy" { - count = local.create_staging_relay_deploy_identity ? 1 : 0 - - project = var.project_id - workload_identity_pool_id = local.relay_workload_identity_pool_id - workload_identity_pool_provider_id = "github-relay-deploy" - display_name = "GitHub Relay staging deploy" - - attribute_mapping = { - "google.subject" = "assertion.sub" - "attribute.repository" = "assertion.repository" - "attribute.repository_id" = "assertion.repository_id" - "attribute.repository_owner_id" = "assertion.repository_owner_id" - "attribute.ref" = "assertion.ref" - "attribute.environment" = "assertion.environment" - "attribute.workflow_ref" = "assertion.workflow_ref" - "attribute.relay_ops_identity" = "'staging-deploy'" - } - - # An allowlist of exact workflow refs, never a prefix: a namespace grant would hand this account - # to any future staging workflow with no Terraform diff. - attribute_condition = join(" && ", concat(local.relay_github_leading_repository_claims, [ - "assertion.ref == 'refs/heads/main'", - "assertion.environment == 'staging'", - local.relay_github_workflow_conditions["github_staging_relay_deploy"] - ])) - - oidc { - issuer_uri = "https://token.actions.githubusercontent.com" - } -} - -resource "google_service_account_iam_member" "github_staging_relay_deploy_workload_identity_user" { - count = local.create_staging_relay_deploy_identity ? 1 : 0 - - service_account_id = google_service_account.github_staging_relay_deploy[0].name - role = "roles/iam.workloadIdentityUser" - member = "principalSet://iam.googleapis.com/${local.relay_workload_identity_pool_name}/attribute.relay_ops_identity/staging-deploy" -} - -# Every one but the cell-flags workflow runs `terraform init` (or `infra.mjs init --env staging`) first, and the -# GCS backend lists the bucket before it can open the state. Bucket metadata only, no object -# access; this mirrors relay_fence_broker_bucket_reader. -resource "google_storage_bucket_iam_member" "github_staging_relay_deploy_state_list" { - count = local.create_staging_relay_deploy_identity ? 1 : 0 - - bucket = "${var.project_id}-terraform-state" - role = "roles/storage.legacyBucketReader" - member = google_service_account.github_staging_relay_deploy[0].member -} - -# Reads reviewed topology: `terraform output -json relay_gce_cell_deployments` and the -# `terraform console` binds in Deploy Relay Staging. No step in the six applies, so this is -# objectViewer, not the capacity identity's objectAdmin, over the same two objects. -resource "google_storage_bucket_iam_member" "github_staging_relay_deploy_state" { - count = local.create_staging_relay_deploy_identity ? 1 : 0 - - bucket = "${var.project_id}-terraform-state" - role = "roles/storage.objectViewer" - member = google_service_account.github_staging_relay_deploy[0].member - - condition { - title = "relay_staging_deploy_state" - description = "Limits the staging Relay deploy workflows to the default Terraform state and lock." - expression = join(" || ", [ - "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tfstate'", - "resource.name == 'projects/_/buckets/${var.project_id}-terraform-state/objects/terraform/state/default.tflock'" - ]) - } -} - -# Deploy Relay Staging step "Require the mirrored immutable image" runs -# `gcloud artifacts docker images describe`; nothing in the six writes to the repository. -resource "google_artifact_registry_repository_iam_member" "github_staging_relay_deploy_artifact_reader" { - count = local.create_staging_relay_deploy_identity ? 1 : 0 - - project = var.project_id - location = var.region - repository = var.artifact_repository_id - role = "roles/artifactregistry.reader" - member = google_service_account.github_staging_relay_deploy[0].member -} - -# Deploy Relay Staging step "Deploy director blue/green", Operate Relay Asia Admission step -# "Deploy the registered additive director topology", and Power Relay Staging's scale-to-zero all -# drive `gcloud run services update|update-traffic` against the staging director. -resource "google_cloud_run_v2_service_iam_member" "github_staging_relay_deploy_director_developer" { - count = local.create_staging_relay_deploy_identity ? 1 : 0 - - project = var.project_id - location = var.region - name = var.relay_cloud_run_service_name - role = "roles/run.developer" - member = google_service_account.github_staging_relay_deploy[0].member -} - -# Power Relay Staging step "Inspect or change staging power state" scales both entries of -# CLOUD_RUN_SERVICES in power-staging-relay.mjs to zero, and the second one is the shared staging -# auth service. The same workflow already stops the shared staging database through -# orcaRelayStagingPower, so this stays with the power operator rather than the apps root. -resource "google_cloud_run_v2_service_iam_member" "github_staging_relay_deploy_auth_developer" { - count = local.create_staging_relay_deploy_identity && var.relay_staging_power_auth_service_name != "" ? 1 : 0 - - project = var.project_id - location = var.region - name = var.relay_staging_power_auth_service_name - role = "roles/run.developer" - member = google_service_account.github_staging_relay_deploy[0].member -} - -# That same scale-to-zero mints a revision when the latest ready one is not already at zero, and -# Cloud Run gates a revision on actAs over the service's runtime account. -resource "google_service_account_iam_member" "github_staging_relay_deploy_auth_runtime_user" { - count = local.create_staging_relay_deploy_identity && var.relay_staging_power_auth_service_name != "" ? 1 : 0 - - service_account_id = "projects/${var.project_id}/serviceAccounts/${local.staging_auth_runtime_service_account_email}" - role = "roles/iam.serviceAccountUser" - member = google_service_account.github_staging_relay_deploy[0].member -} - -# Deploy Relay Staging GCE Candidate preflight reads MIG, instance, and backend-service topology -# (deploy-relay-gce-candidate.mjs inspectCell), which the power role's instanceGroupManagers.get -# does not cover. -resource "google_project_iam_member" "github_staging_relay_deploy_compute_viewer" { - count = local.create_staging_relay_deploy_identity ? 1 : 0 - - project = var.project_id - role = "roles/compute.viewer" - member = google_service_account.github_staging_relay_deploy[0].member -} - -# Deploy Relay Staging step "Deploy director blue/green" with a director floor above one reads -# the candidate's Cloud Run request log to confirm every floor instance serves before the full move. -resource "google_project_iam_member" "github_staging_relay_deploy_logging_viewer" { - count = local.create_staging_relay_deploy_identity ? 1 : 0 - - project = var.project_id - role = "roles/logging.viewer" - member = google_service_account.github_staging_relay_deploy[0].member -} diff --git a/cloud/infra/terraform/relay.tf b/cloud/infra/terraform/relay.tf deleted file mode 100644 index ae7994c5bfc..00000000000 --- a/cloud/infra/terraform/relay.tf +++ /dev/null @@ -1,684 +0,0 @@ -resource "google_service_account" "relay_runtime" { - project = var.project_id - account_id = "${var.name_prefix}-relay" - display_name = var.environment == "staging" ? "Orca Relay" : "Orca Relay cells" - description = var.environment == "staging" ? ( - "Runtime identity for the Orca Relay director and stamped cells." - ) : "Runtime identity for stamped Orca Relay cells." -} - -resource "google_service_account" "relay_director_runtime" { - project = var.project_id - account_id = "${var.name_prefix}-relay-dir" - display_name = "Orca Relay director" - description = "Runtime and regional rehoming caller identity for the Orca Relay director." -} - -resource "google_project_iam_member" "relay_runtime_cloudsql_client" { - project = var.project_id - role = "roles/cloudsql.client" - member = google_service_account.relay_runtime.member -} - -resource "google_project_iam_member" "relay_director_runtime_cloudsql_client" { - project = var.project_id - role = "roles/cloudsql.client" - member = google_service_account.relay_director_runtime.member -} - -resource "random_password" "relay_assignment_signing_key" { - length = 48 - special = false -} - -resource "google_secret_manager_secret" "relay_assignment_signing_key" { - project = var.project_id - secret_id = "orca-cloud-relay-assignment-signing-key" - labels = local.relay_shared_labels - - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "relay_assignment_signing_key" { - secret = google_secret_manager_secret.relay_assignment_signing_key.id - secret_data = random_password.relay_assignment_signing_key.result -} - -resource "google_secret_manager_secret_iam_member" "relay_assignment_signing_key_accessor" { - project = var.project_id - secret_id = google_secret_manager_secret.relay_assignment_signing_key.secret_id - role = "roles/secretmanager.secretAccessor" - member = google_service_account.relay_runtime.member -} - -resource "google_secret_manager_secret_iam_member" "relay_assignment_signing_key_director_accessor" { - project = var.project_id - secret_id = google_secret_manager_secret.relay_assignment_signing_key.secret_id - role = "roles/secretmanager.secretAccessor" - member = google_service_account.relay_director_runtime.member -} - -resource "google_secret_manager_secret" "relay_regional_placement_enabled" { - project = var.project_id - secret_id = "orca-cloud-relay-regional-placement-enabled" - labels = local.relay_shared_labels - - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "relay_regional_placement_enabled" { - secret = google_secret_manager_secret.relay_regional_placement_enabled.id - secret_data = tostring(var.relay_regional_placement_enabled) -} - -resource "google_secret_manager_secret_iam_member" "relay_regional_placement_runtime_accessor" { - project = var.project_id - secret_id = google_secret_manager_secret.relay_regional_placement_enabled.secret_id - role = "roles/secretmanager.secretAccessor" - member = google_service_account.relay_runtime.member -} - -resource "google_secret_manager_secret_iam_member" "relay_regional_placement_director_accessor" { - project = var.project_id - secret_id = google_secret_manager_secret.relay_regional_placement_enabled.secret_id - role = "roles/secretmanager.secretAccessor" - member = google_service_account.relay_director_runtime.member -} - -resource "google_secret_manager_secret_iam_member" "relay_regional_placement_deploy_accessor" { - count = local.relay_create_github_deploy_identity ? 1 : 0 - - project = var.project_id - secret_id = google_secret_manager_secret.relay_regional_placement_enabled.secret_id - role = "roles/secretmanager.secretAccessor" - member = local.relay_github_deploy_service_account_member -} - -resource "google_secret_manager_secret_iam_member" "relay_regional_placement_deploy_adder" { - count = local.relay_create_github_deploy_identity ? 1 : 0 - - project = var.project_id - secret_id = google_secret_manager_secret.relay_regional_placement_enabled.secret_id - role = "roles/secretmanager.secretVersionAdder" - member = local.relay_github_deploy_service_account_member -} - -resource "google_secret_manager_secret_iam_member" "relay_regional_placement_deploy_viewer" { - count = local.relay_create_github_deploy_identity ? 1 : 0 - - project = var.project_id - secret_id = google_secret_manager_secret.relay_regional_placement_enabled.secret_id - role = "roles/secretmanager.viewer" - member = local.relay_github_deploy_service_account_member -} - -data "external" "relay_serving_regional_placement_version" { - program = [ - "node", - "${path.module}/../../dev/scripts/read-relay-serving-regional-placement-version.mjs" - ] - query = { - project = var.project_id - region = var.region - service = var.relay_cloud_run_service_name - bootstrap_version = google_secret_manager_secret_version.relay_regional_placement_enabled.version - } -} - -resource "google_cloud_run_v2_service" "relay" { - project = var.project_id - name = var.relay_cloud_run_service_name - location = var.region - ingress = "INGRESS_TRAFFIC_ALL" - invoker_iam_disabled = true - labels = local.relay_shared_labels - - template { - service_account = google_service_account.relay_director_runtime.email - timeout = "${var.relay_director_request_timeout_seconds}s" - max_instance_request_concurrency = var.relay_director_concurrency - - scaling { - min_instance_count = var.relay_min_instances - max_instance_count = var.relay_max_instances - } - - # Built-in socket only without the sidecar: its connect() blocks the event loop (variables.tf). - dynamic "volumes" { - for_each = var.relay_director_cloud_sql_proxy_sidecar ? [] : ["cloudsql"] - - content { - name = volumes.value - - cloud_sql_instance { - instances = [local.relay_database_connection_name] - } - } - } - - # Stays first: deploys and readers address the relay as containers[0], and gcloud flags without - # --container land on the ingress container. - containers { - name = var.relay_director_cloud_sql_proxy_sidecar ? "relay" : null - image = var.relay_cloud_run_image - depends_on = var.relay_director_cloud_sql_proxy_sidecar ? ["cloud-sql-proxy"] : null - - env { - name = "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED" - - value_source { - secret_key_ref { - secret = google_secret_manager_secret.relay_regional_placement_enabled.secret_id - version = data.external.relay_serving_regional_placement_version.result.version - } - } - } - - env { - name = "ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT" - value = data.external.relay_serving_regional_placement_version.result.cohort_percent - } - - # The director deploy owns these (shadow cells are set by hand): an apply keeps whatever the - # serving revision has, and adds nothing when it has none. - dynamic "env" { - for_each = { - for name, value in { - ORCA_RELAY_RESERVE_PLACEMENT = data.external.relay_serving_regional_placement_version.result.reserve_placement - ORCA_RELAY_SHADOW_SEAT_FEED_CELLS = data.external.relay_serving_regional_placement_version.result.shadow_seat_feed_cells - } : name => value if value != "" - } - - content { - name = env.key - value = env.value - } - } - - ports { - container_port = 8080 - } - - dynamic "volume_mounts" { - for_each = var.relay_director_cloud_sql_proxy_sidecar ? [] : ["cloudsql"] - - content { - name = volume_mounts.value - mount_path = "/cloudsql" - } - } - - env { - name = "DATABASE_URL" - - value_source { - secret_key_ref { - secret = (var.relay_director_cloud_sql_proxy_sidecar - ? google_secret_manager_secret.relay_director_database_url[0].secret_id - : google_secret_manager_secret.relay_database_url.secret_id) - version = "latest" - } - } - } - - env { - name = "ORCA_RELAY_ASSIGNMENT_SIGNING_KEY" - - value_source { - secret_key_ref { - secret = google_secret_manager_secret.relay_assignment_signing_key.secret_id - version = "latest" - } - } - } - - env { - name = "ORCA_RELAY_PUBLIC_URL" - value = var.relay_base_url - } - - env { - name = "ORCA_RELAY_CELL_URL" - value = var.relay_base_url - } - - env { - name = "ORCA_RELAY_AUTH_ISSUER" - value = var.auth_base_url - } - - env { - name = "ORCA_RELAY_AUTH_AUDIENCE" - value = "orca-relay" - } - - env { - name = "ORCA_RELAY_JWKS_URL" - value = "${var.auth_base_url}/.well-known/jwks.json" - } - - env { - name = "ORCA_RELAY_ROLE" - value = "director" - } - - env { - name = "ORCA_RELAY_ADMISSION_SELECTOR_VERSION" - value = "3" - } - - env { - name = "ORCA_RELAY_CELL_ID" - value = "director" - } - - env { - name = "ORCA_RELAY_CELLS_JSON" - value = local.relay_director_cells_json - } - - env { - name = "ORCA_RELAY_ADMIN_AUDIENCE" - value = "${var.relay_base_url}/v1/admin/drain" - } - - env { - name = "ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT" - value = local.relay_github_deploy_service_account_email - } - - env { - name = "ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT" - value = local.relay_capacity_service_account_email - } - - env { - name = "ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT" - value = local.relay_asia_proof_service_account_email - } - - env { - name = "ORCA_RELAY_MONITOR_SERVICE_ACCOUNT" - value = try(google_service_account.github_monitor[0].email, "") - } - - env { - name = "ORCA_RELAY_FENCE_SERVICE_ACCOUNT" - value = try(google_service_account.github_fence[0].email, "") - } - - env { - name = "ORCA_RELAY_FENCE_BROKER_SERVICE_ACCOUNT" - value = try(google_service_account.relay_fence_broker[0].email, "") - } - - env { - name = "ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT" - value = google_service_account.relay_runtime.email - } - - env { - name = "ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT" - value = google_service_account.relay_director_runtime.email - } - - env { - name = "ORCA_RELAY_REHOME_AUDIENCE" - value = "${var.relay_base_url}/v1/admin/host-drain" - } - - env { - name = "ORCA_RELAY_HEARTBEAT_AUDIENCE" - value = "${var.relay_base_url}/v1/admin/cell-heartbeat" - } - - env { - name = "ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED" - value = tostring(var.relay_public_assignments_enabled) - } - - env { - name = "ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY" - value = tostring(var.relay_public_assignment_concurrency) - } - - env { - name = "ORCA_RELAY_PUBLIC_ASSIGNMENT_RETRY_AFTER_SECONDS" - value = tostring(var.relay_public_assignment_retry_after_seconds) - } - - env { - name = "ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX" - value = tostring(var.relay_public_assignment_queue_max) - } - - env { - name = "ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS" - value = tostring(var.relay_public_assignment_wait_ms) - } - - env { - name = "ORCA_RELAY_DATABASE_POOL_MAX" - value = tostring(var.relay_director_database_pool_max) - } - - env { - name = "ORCA_RELAY_PUBLIC_STICKY_CONCURRENCY" - value = tostring(var.relay_public_sticky_concurrency) - } - - env { - name = "ORCA_RELAY_PUBLIC_STICKY_QUEUE_MAX" - value = tostring(var.relay_public_sticky_queue_max) - } - - env { - name = "ORCA_RELAY_PUBLIC_STICKY_WAIT_MS" - value = tostring(var.relay_public_sticky_wait_ms) - } - - env { - name = "ORCA_RELAY_PUBLIC_STICKY_RETRY_AFTER_SECONDS" - value = tostring(var.relay_public_sticky_retry_after_seconds) - } - - resources { - limits = { - cpu = var.relay_cloud_run_cpu - memory = var.relay_cloud_run_memory - } - - cpu_idle = false - } - } - - # Accepts a loopback TCP connection at once and dials Cloud SQL in the background, so the pool's - # 2 s connect timeout holds through a start-up or failover. Authenticates as the director's - # runtime account. The GCE cells run the same proxy, over a Unix socket on a full Linux kernel. - dynamic "containers" { - for_each = var.relay_director_cloud_sql_proxy_sidecar ? ["cloud-sql-proxy"] : [] - - content { - name = containers.value - image = var.relay_gce_cloud_sql_proxy_image - args = [ - "--address=127.0.0.1", - "--port=5432", - "--health-check", - # Cloud Run's probe is not documented to reach loopback; only the ingress port is routed. - "--http-address=0.0.0.0", - "--http-port=9090", - "--structured-logs", - "--exit-zero-on-sigterm", - # Lets in-flight statements finish while the relay drains after SIGTERM. - "--max-sigterm-delay=10s", - local.relay_database_connection_name - ] - - # Go sizes its scheduler to the host's cores, not the 0.25 vCPU quota; one keeps it from - # being throttled in bursts. - env { - name = "GOMAXPROCS" - value = "1" - } - - # Ready once it listens; it does not wait for the database, so the relay boots through - # an outage exactly as it does today. - startup_probe { - period_seconds = 1 - timeout_seconds = 1 - failure_threshold = 60 - - http_get { - path = "/startup" - port = 9090 - } - } - - # /liveness answers without the database; failing it means the proxy itself is wedged. - liveness_probe { - period_seconds = 10 - timeout_seconds = 1 - failure_threshold = 3 - - http_get { - path = "/liveness" - port = 9090 - } - } - - # A quarter vCPU carries the pool's few connections. Cloud Run rejects only a total under - # 1 vCPU with concurrency above 1 (validated against the API with the relay's 1 vCPU and - # concurrency 1000), so the sidecar adds 25% to director compute, not 100%. - resources { - limits = { - cpu = "0.25" - memory = "256Mi" - } - - cpu_idle = false - } - } - } - } - - # Deploys update immutable images; Terraform owns service shape and IAM. - lifecycle { - # Why: the relay refuses to boot unless both admission lanes fit the pool, so catch it - # at plan time rather than as a crash loop on the candidate revision. - precondition { - condition = (var.relay_public_assignment_concurrency + - var.relay_public_sticky_concurrency) <= var.relay_director_database_pool_max - error_message = "Placement plus reconnect admission must fit the director database pool." - } - - ignore_changes = [ - client, - client_version, - template[0].containers[0].image - ] - } - - depends_on = [ - data.google_artifact_registry_repository.relay_images, - google_project_iam_member.relay_director_runtime_cloudsql_client, - google_secret_manager_secret_iam_member.relay_assignment_signing_key_director_accessor, - google_secret_manager_secret_iam_member.relay_regional_placement_director_accessor, - google_secret_manager_secret_iam_member.relay_database_url_director_accessor, - google_secret_manager_secret_version.relay_assignment_signing_key, - google_secret_manager_secret_version.relay_regional_placement_enabled, - google_secret_manager_secret_version.relay_database_url, - google_secret_manager_secret_iam_member.relay_director_database_url_accessor, - google_secret_manager_secret_version.relay_director_database_url - ] -} - -resource "google_cloud_run_v2_service" "relay_cell" { - for_each = var.relay_cells - - project = var.project_id - name = each.value.service_name - location = var.region - ingress = "INGRESS_TRAFFIC_ALL" - invoker_iam_disabled = true - # Require an explicit configuration change before a stamped cell can be decommissioned. - deletion_protection = each.value.deletion_protection - labels = merge(local.relay_shared_labels, { - "orca-relay-role" = "cell" - "orca-relay-cell" = each.key - }) - - template { - service_account = google_service_account.relay_runtime.email - timeout = "${var.relay_request_timeout_seconds}s" - max_instance_request_concurrency = var.relay_concurrency - - scaling { - min_instance_count = each.value.min_instances - max_instance_count = each.value.max_instances - } - - volumes { - name = "cloudsql" - - cloud_sql_instance { - instances = [local.relay_database_connection_name] - } - } - - containers { - image = var.relay_cloud_run_image - - ports { - container_port = 8080 - } - - volume_mounts { - name = "cloudsql" - mount_path = "/cloudsql" - } - - env { - name = "DATABASE_URL" - - value_source { - secret_key_ref { - secret = google_secret_manager_secret.relay_database_url.secret_id - version = "latest" - } - } - } - - env { - name = "ORCA_RELAY_ASSIGNMENT_SIGNING_KEY" - - value_source { - secret_key_ref { - secret = google_secret_manager_secret.relay_assignment_signing_key.secret_id - version = "latest" - } - } - } - - env { - name = "ORCA_RELAY_PUBLIC_URL" - value = each.value.url - } - - env { - name = "ORCA_RELAY_CELL_URL" - value = each.value.url - } - - env { - name = "ORCA_RELAY_AUTH_ISSUER" - value = var.auth_base_url - } - - env { - name = "ORCA_RELAY_AUTH_AUDIENCE" - value = "orca-relay" - } - - env { - name = "ORCA_RELAY_JWKS_URL" - value = "${var.auth_base_url}/.well-known/jwks.json" - } - - env { - name = "ORCA_RELAY_ROLE" - value = "cell" - } - - env { - name = "ORCA_RELAY_CELL_ID" - value = each.key - } - - env { - name = "ORCA_RELAY_CELL_CAPACITY" - value = tostring(each.value.capacity_requests) - } - - env { - name = "ORCA_RELAY_CELLS_JSON" - value = "[]" - } - - env { - name = "ORCA_RELAY_ADMIN_AUDIENCE" - value = "${var.relay_base_url}/v1/admin/drain" - } - - env { - name = "ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT" - value = local.relay_github_deploy_service_account_email - } - - env { - name = "ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT" - value = local.relay_capacity_service_account_email - } - - env { - name = "ORCA_RELAY_ASIA_PROOF_SERVICE_ACCOUNT" - value = local.relay_asia_proof_service_account_email - } - - env { - name = "ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT" - value = google_service_account.relay_runtime.email - } - - env { - name = "ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT" - value = contains(var.relay_region_rehome_source_cell_ids, each.key) ? google_service_account.relay_director_runtime.email : "" - } - - env { - name = "ORCA_RELAY_REHOME_AUDIENCE" - value = contains(var.relay_region_rehome_source_cell_ids, each.key) ? "${var.relay_base_url}/v1/admin/host-drain" : "" - } - - env { - name = "ORCA_RELAY_DIRECTOR_URL" - value = var.relay_base_url - } - - env { - name = "ORCA_RELAY_HEARTBEAT_AUDIENCE" - value = "${var.relay_base_url}/v1/admin/cell-heartbeat" - } - - resources { - limits = { - cpu = var.relay_cloud_run_cpu - memory = var.relay_cloud_run_memory - } - - cpu_idle = false - } - } - } - - lifecycle { - ignore_changes = [ - client, - client_version, - template[0].containers[0].image - ] - } - - depends_on = [ - data.google_artifact_registry_repository.relay_images, - google_project_iam_member.relay_runtime_cloudsql_client, - google_secret_manager_secret_iam_member.relay_assignment_signing_key_accessor, - google_secret_manager_secret_iam_member.relay_database_url_accessor, - google_secret_manager_secret_version.relay_assignment_signing_key, - google_secret_manager_secret_version.relay_database_url - ] -} diff --git a/cloud/infra/terraform/variables.tf b/cloud/infra/terraform/variables.tf deleted file mode 100644 index c6b843c1694..00000000000 --- a/cloud/infra/terraform/variables.tf +++ /dev/null @@ -1,602 +0,0 @@ -variable "artifact_repository_id" { - type = string - description = "Artifact Registry Docker repository ID." -} - -variable "environment" { - type = string - description = "Deployment environment." - - validation { - condition = contains(["staging", "production"], var.environment) - error_message = "environment must be staging or production." - } -} - -variable "github_owner" { - type = string - description = "GitHub owner allowed to deploy through Workload Identity Federation." - default = "stablyai" -} - -variable "github_repo" { - type = string - description = "GitHub repo allowed to deploy through Workload Identity Federation." - default = "orca" -} - -# Numeric IDs survive a rename or transfer of the repository; every provider pins them next to the name. -variable "github_repo_id" { - type = string - description = "Numeric GitHub repository ID of github_owner/github_repo." - default = "1183888342" - - validation { - condition = can(regex("^[0-9]+$", var.github_repo_id)) - error_message = "github_repo_id must be the numeric repository ID." - } -} - -variable "github_owner_id" { - type = string - description = "Numeric GitHub owner ID of github_owner." - default = "127256420" - - validation { - condition = can(regex("^[0-9]+$", var.github_owner_id)) - error_message = "github_owner_id must be the numeric owner ID." - } -} - -# The rename the relay repository applies to the workflow files it carries. The public repo keeps -# the workflows under `cloud-` names, so every relay workflow_ref is built from this head. -variable "github_workflow_file_prefix" { - type = string - description = "Filename prefix on github_owner/github_repo's copies of the relay workflows." - default = "cloud-" - - validation { - condition = can(regex("^[a-z0-9-]*$", var.github_workflow_file_prefix)) - error_message = "github_workflow_file_prefix must be lowercase letters, digits, or hyphens." - } -} - -# Additional repositories whose identical workflows the same identities must accept during a -# repository move. Each entry renders its own OR arm in every provider condition, so both repos -# can run the same workflows through the same identities. `workflow_file_prefix` is the rename the -# importing repository applies to the workflow files it copies. Empty is the steady state, and is -# where the public extraction left it: stablyai/orca is now the primary and only repository. -variable "github_accepted_repositories" { - type = list(object({ - owner = string - repo = string - repo_id = string - owner_id = string - workflow_file_prefix = string - })) - description = "Extra repositories accepted alongside github_owner/github_repo during the public extraction." - default = [] - - validation { - condition = alltrue([ - for repository in var.github_accepted_repositories : - can(regex("^[0-9]+$", repository.repo_id)) && can(regex("^[0-9]+$", repository.owner_id)) - ]) - error_message = "github_accepted_repositories entries must carry numeric repo_id and owner_id values." - } - - validation { - condition = alltrue([ - for repository in var.github_accepted_repositories : - can(regex("^[a-z0-9-]*$", repository.workflow_file_prefix)) - ]) - error_message = "github_accepted_repositories workflow_file_prefix must be lowercase letters, digits, or hyphens." - } -} - -variable "name_prefix" { - type = string - description = "Prefix used for named resources." -} - -variable "project_id" { - type = string - description = "GCP project ID." -} - -variable "region" { - type = string - description = "GCP region for regional resources." - default = "us-central1" -} - -variable "auth_base_url" { - type = string - description = "Public base URL of the auth service; OAuth callbacks and JWT issuer derive from it." -} - -variable "manage_relay_domain_mapping" { - type = bool - description = "Manage the Google Cloud Run mapping independently of the Cloudflare record." - default = false -} - -variable "relay_base_url" { - type = string - description = "Public TLS origin of the stable relay director." -} - -variable "relay_cloud_run_service_name" { - type = string - description = "Cloud Run service name for Orca Relay." -} - -variable "relay_staging_power_auth_service_name" { - type = string - description = "Shared staging auth Cloud Run service that Power Relay Staging scales to zero; empty outside staging." - default = "" -} - -variable "relay_cloud_run_image" { - type = string - description = "Initial image for the Terraform-created relay Cloud Run service." - default = "us-docker.pkg.dev/cloudrun/container/hello" -} - -variable "relay_cloud_run_cpu" { - type = string - description = "CPU limit for the relay container." - default = "1" -} - -variable "relay_cloud_run_memory" { - type = string - description = "Memory limit for the relay container." - default = "512Mi" -} - -variable "relay_fence_broker_service_name" { - type = string - description = "Private Cloud Run service that owns reviewed Relay Terraform fences." - default = "orca-cloud-relay-fence" -} - -variable "relay_fence_broker_image" { - type = string - description = "Immutable image for the private Relay fence broker." - default = "us-docker.pkg.dev/cloudrun/container/hello" - - validation { - condition = ( - var.relay_fence_broker_image == "us-docker.pkg.dev/cloudrun/container/hello" || - can(regex("^[a-z0-9.-]+/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$", var.relay_fence_broker_image)) - ) - error_message = "relay_fence_broker_image must be the bootstrap image or an immutable digest." - } -} - -variable "relay_fence_source_cell_id" { - type = string - description = "Exact incident source cell accepted by the private fence broker." - default = "production-gce-c3" -} - -variable "relay_fence_failed_target_cell_id" { - type = string - description = "Exact failed registered target accepted by the private fence broker." - default = "production-gce-c12" -} - -variable "relay_fence_replacement_target_cell_id" { - type = string - description = "Exact replacement target accepted by the private fence broker." - default = "production-gce-c13" -} - -variable "relay_fence_unobserved_connection_bound" { - type = number - description = "Reviewed unobserved connection bound enforced during supersession." - default = 60 - - validation { - condition = ( - var.relay_fence_unobserved_connection_bound >= 0 && - var.relay_fence_unobserved_connection_bound < 500 - ) - error_message = "relay_fence_unobserved_connection_bound must be between zero and 499." - } -} - -variable "relay_director_concurrency" { - type = number - description = "Cloud Run concurrency for short-lived director HTTP requests." - default = 80 -} - -variable "relay_director_request_timeout_seconds" { - type = number - description = "Cloud Run timeout for short-lived director HTTP requests." - default = 30 -} - -variable "relay_concurrency" { - type = number - description = "Cloud Run cell concurrency; every WebSocket leg counts." - default = 1000 -} - -variable "relay_request_timeout_seconds" { - type = number - description = "Cloud Run cell request timeout for standing WebSocket legs." - default = 3600 -} - -variable "relay_public_assignments_enabled" { - type = bool - description = "Emergency switch for public assignment and resolve requests." - default = true -} - -variable "relay_regional_placement_enabled" { - type = bool - description = "Initial preferred-region placement state; audited director deploys own later changes." - default = true -} - -variable "relay_region_rehome_source_cell_ids" { - type = set(string) - description = "Reviewed Relay cells, in any configured region, allowed to advertise and accept the regional rehome source protocol." - default = [] -} - -variable "relay_public_assignment_concurrency" { - type = number - description = "Per-director public assignment operations allowed to reach shared state." - default = 2 -} - -variable "relay_public_assignment_retry_after_seconds" { - type = number - description = "Minimum retry interval enforced per relay host during assignment recovery." - default = 5 -} - -# Why: these three match the application defaults today. Pinning them keeps a code-side -# default change from silently re-tuning production on the next unrelated apply. -variable "relay_public_assignment_queue_max" { - type = number - description = "Queued public assignment operations allowed per director instance." - default = 128 -} - -variable "relay_public_assignment_wait_ms" { - type = number - description = "Milliseconds a public assignment waits for an admission slot before 503." - default = 4000 -} - -# Why: the sticky (reconnect) lane shared the assignment pool but lived only as a code -# default, so Terraform could not see it. Raising placement concurrency alone then pushed -# placement + sticky past the pool and the director refused to boot. -variable "relay_public_sticky_concurrency" { - type = number - description = "Per-director reconnect-lane operations allowed to reach shared state." - default = 1 -} - -variable "relay_public_sticky_queue_max" { - type = number - description = "Queued reconnect-lane operations allowed per director instance." - default = 64 -} - -variable "relay_public_sticky_wait_ms" { - type = number - description = "Milliseconds a reconnect waits for an admission slot before 503." - default = 2000 -} - -variable "relay_public_sticky_retry_after_seconds" { - type = number - description = "Minimum retry interval enforced per relay host during reconnect recovery." - default = 2 -} - -variable "relay_director_database_pool_max" { - type = number - description = "Director database pool size; must fit placement plus sticky admission slots." - default = 3 - - validation { - condition = var.relay_director_database_pool_max >= 3 - error_message = "The director pool must fit both placement and sticky admission slots." - } -} - -variable "relay_min_instances" { - type = number - description = "Minimum instances for the stable relay director." - default = 1 -} - -variable "relay_max_instances" { - type = number - description = "Maximum instances for the stateless stable relay director." - default = 2 - - validation { - condition = var.relay_max_instances >= 1 - error_message = "The relay director needs at least one instance." - } -} - -variable "relay_cells" { - type = map(object({ - service_name = string - url = string - capacity_requests = number - min_instances = number - max_instances = number - deletion_protection = optional(bool, true) - })) - description = "Explicit stamped max-one relay cells keyed by durable cell ID." - default = {} - - validation { - condition = alltrue([ - for cell in values(var.relay_cells) : - cell.max_instances == 1 && - cell.min_instances >= 0 && - cell.min_instances <= cell.max_instances && - cell.capacity_requests >= 1 && - cell.capacity_requests <= 1000 && - can(regex("^https://[^/]+$", cell.url)) - ]) - error_message = "Relay cells must use HTTPS origins, capacity 1..1000, and max exactly one." - } -} - -variable "relay_alert_notification_channels" { - type = list(string) - description = "Cloud Monitoring notification-channel resource names for Orca Relay alerts. Empty keeps policies visible without paging." - default = [] -} - -variable "relay_cell_min_fix_level" { - type = number - description = "Lowest RELAY_FIX_LEVEL a serving relay cell may run before the outdated-image alert fires. Raise it after a wave rolls every serving cell." - default = 1 -} - -variable "relay_gce_domain" { - type = string - description = "Parent DNS name for GCE relay cells; each cell is one exact host below it." - default = "" - - validation { - condition = var.relay_gce_domain == "" || ( - can(regex("^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)+$", var.relay_gce_domain)) && - !startswith(var.relay_gce_domain, "*.") - ) - error_message = "relay_gce_domain must be empty or a lowercase DNS name without a wildcard or scheme." - } -} - -variable "relay_gce_subnetwork_cidr" { - type = string - description = "Private IPv4 range dedicated to GCE relay cells." - default = "10.42.0.0/24" - - validation { - condition = can(cidrhost(var.relay_gce_subnetwork_cidr, 1)) - error_message = "relay_gce_subnetwork_cidr must be a valid IPv4 CIDR." - } -} - -variable "relay_gce_additional_region_subnetwork_cidrs" { - type = map(string) - description = "Private IPv4 ranges for additive Relay regions; the primary region keeps its legacy resources." - default = {} - - validation { - condition = alltrue([ - for region, cidr in var.relay_gce_additional_region_subnetwork_cidrs : - contains(["asia-east2"], region) && - can(cidrhost(cidr, 1)) - ]) - error_message = "Additional Relay regions must be allowlisted and use valid IPv4 CIDRs." - } -} - -variable "relay_gce_cells" { - type = map(object({ - hostname = string - region = optional(string, "us-central1") - zone = string - machine_type = string - boot_disk_gb = number - boot_image = string - capacity_requests = number - database_pool_max = optional(number, 10) - image = string - initially_enabled = optional(bool, true) - connection_hard_cap = optional(number) - connection_unobserved_bound = optional(number) - })) - description = "Private GCE relay cells keyed by durable cell ID; unfenced cells remain fixed-one." - default = {} - - validation { - condition = alltrue([ - for cell_id, cell in var.relay_gce_cells : - can(regex("^[a-z][a-z0-9-]{0,39}$", cell_id)) && - can(regex("^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$", cell.hostname)) && - contains(["us-central1", "asia-east2"], cell.region) && - startswith(cell.zone, "${cell.region}-") && - can(regex("^[a-z0-9-]+$", cell.machine_type)) && - can(regex("^https://www[.]googleapis[.]com/compute/v1/projects/cos-cloud/global/images/cos-(stable-)?[0-9]+-[0-9]+-[0-9]+-[0-9]+$", cell.boot_image)) && - cell.boot_disk_gb >= 20 && - cell.boot_disk_gb <= 100 && - cell.capacity_requests >= 1 && - cell.capacity_requests <= 100000 && - cell.database_pool_max >= 1 && - cell.database_pool_max <= 100 && - ( - (cell.connection_hard_cap == null && - cell.connection_unobserved_bound == null) || - try( - contains([600, 1000, 3000], cell.connection_hard_cap) && - cell.connection_unobserved_bound >= 0 && - cell.connection_unobserved_bound < cell.connection_hard_cap - 100, - false - ) - ) && - can(regex("^[a-z0-9.-]+/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$", cell.image)) - ]) && length(distinct([for cell in values(var.relay_gce_cells) : cell.hostname])) == length(var.relay_gce_cells) - error_message = "GCE cells need an allowlisted region and matching zone, unique DNS labels, a pinned COS boot image, bounded machine/disk/capacity/pool values, paired supported connection limits with rebind headroom, and digest-pinned relay images." - } -} - -variable "relay_gce_cell_log_sample_rate" { - type = number - description = "Fraction of relay cell load-balancer requests written to Cloud Logging; 1 keeps assign-to-connection joins exact." - default = 1 - - validation { - condition = var.relay_gce_cell_log_sample_rate >= 0 && var.relay_gce_cell_log_sample_rate <= 1 - error_message = "relay_gce_cell_log_sample_rate must be between 0 and 1." - } -} - -variable "relay_gce_fenced_cells" { - type = set(string) - description = "Reviewed relay GCE cell IDs whose Terraform-owned MIG target size is zero." - default = [] -} - -variable "relay_cloud_sql_private_ip" { - type = bool - description = "Dial Cloud SQL over its private IP inside this VPC instead of its public IP through Cloud NAT. Requires the foundation root's private services access peering to be applied first; a cell that cannot reach the private IP never becomes ready." - default = false -} - -variable "relay_director_cloud_sql_proxy_sidecar" { - type = bool - description = "Reach Cloud SQL from the director through a Cloud SQL Auth Proxy sidecar on 127.0.0.1 (the GCE cells run the same proxy beside the relay, over a Unix socket), instead of Cloud Run's built-in /cloudsql socket. A connect() on the built-in socket blocks the director's whole event loop until the connector's upstream dial ends, up to 10 s per attempt while the instance starts or fails over." - default = false -} - -variable "relay_dead_cell_verdict_alert" { - type = bool - description = "Count and alert on directors withholding the dead-cell verdict for a reserve cell whose seat-feed polls fail (cell-poll-reachability.ts). Off in production until staging has proved it. Turning it on: apply the metric first, then the policy a few minutes later, because Cloud Monitoring refuses a policy on a log metric it cannot see yet." - default = false -} - -variable "relay_event_loop_stall_alert" { - type = bool - description = "Count runtime samples whose event-loop delay max reached 5 s and alert on any. The p99 metric reads normal through a single long freeze." - default = false -} - -variable "relay_gce_cloud_sql_proxy_image" { - type = string - description = "Digest-pinned Cloud SQL Auth Proxy image used by private relay workers and the director sidecar." - default = "gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:e8c76356fb586300c63f0d29c8c75cfeeba33c29736f9d4afa2df2d78cd0fa19" - - validation { - condition = can(regex("^[a-z0-9.-]+/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$", var.relay_gce_cloud_sql_proxy_image)) - error_message = "relay_gce_cloud_sql_proxy_image must be pinned by sha256 digest." - } -} - -# --- Mobile push gateway --------------------------------------------------------------------- -# There is no staging push gateway by decision, so this defaults false and only -# environments/production.tfvars turns it on. Everything in push-gateway.tf is behind it. -variable "push_gateway_enabled" { - type = bool - description = "Create the Orca mobile push gateway, its database, secrets, and identity." - default = false -} - -variable "push_base_url" { - type = string - description = "Public TLS origin of the mobile push gateway." - default = "https://push.onorca.dev" - - validation { - condition = can(regex("^https://[^/]+$", var.push_base_url)) - error_message = "push_base_url must be an HTTPS origin with no path." - } -} - -variable "push_cloud_run_service_name" { - type = string - description = "Cloud Run service name for the mobile push gateway." - default = "orca-cloud-push" -} - -variable "push_cloud_run_image" { - type = string - description = "Initial image for the Terraform-created push gateway service; deploys own it after." - default = "us-docker.pkg.dev/cloudrun/container/hello" -} - -variable "push_cloud_run_cpu" { - type = string - description = "CPU limit for the push gateway container." - default = "1" -} - -variable "push_cloud_run_memory" { - type = string - description = "Memory limit for the push gateway container." - default = "512Mi" -} - -# Keep a warm instance to run durable delivery retries without incoming requests. -variable "push_min_instances" { - type = number - description = "Minimum instances for the push gateway." - default = 1 -} - -variable "push_max_instances" { - type = number - description = "Maximum instances for the push gateway." - default = 4 - - validation { - condition = var.push_max_instances >= 1 - error_message = "The push gateway needs at least one instance." - } -} - -# The dedicated database budget counts pools across all three rollout revision resources. -variable "push_database_pool_max" { - type = number - description = "Push gateway database pool size per instance; instances x pool is its Cloud SQL draw." - default = 2 - - validation { - condition = var.push_database_pool_max >= 1 && var.push_database_pool_max <= 100 - error_message = "The push gateway pool must hold at least one connection and stay under the per-service bound." - } -} - -variable "push_concurrency" { - type = number - description = "Cloud Run concurrency for short-lived push gateway HTTP requests." - default = 80 -} - -variable "push_request_timeout_seconds" { - type = number - description = "Cloud Run timeout for push gateway requests; every route is short-lived." - default = 30 -} - -variable "manage_push_domain_mapping" { - type = bool - description = "Manage the push gateway Cloud Run domain mapping; the DNS record stays in the apps root." - default = false -} diff --git a/cloud/infra/terraform/versions.tf b/cloud/infra/terraform/versions.tf deleted file mode 100644 index d8dbae3135c..00000000000 --- a/cloud/infra/terraform/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -terraform { - required_version = ">= 1.7.0" - - required_providers { - google = { - source = "hashicorp/google" - version = "~> 8.8" - } - - random = { - source = "hashicorp/random" - version = "~> 3.6" - } - - external = { - source = "hashicorp/external" - version = "~> 2.3" - } - } - - backend "gcs" {} -} - -provider "google" { - project = var.project_id - region = var.region -} diff --git a/cloud/package.json b/cloud/package.json index df50f862c89..ae6753d7d5d 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -5,19 +5,9 @@ "scripts": { "build": "pnpm -r build", "dev": "pnpm --filter @orca-cloud/relay dev", - "incident:relay": "pnpm --filter @orca-cloud/relay-ops incident:monitor", - "incident:relay-preflight": "pnpm --filter @orca-cloud/relay-ops incident:preflight", - "incident:relay-pre-drain-sample": "pnpm --filter @orca-cloud/relay-ops incident:pre-drain-sample", - "infra:apply": "node dev/scripts/infra.mjs apply", - "infra:init": "node dev/scripts/infra.mjs init", - "infra:plan": "node dev/scripts/infra.mjs plan", "lint": "pnpm -r lint", - "load:relay:controls": "node dev/scripts/load-relay-controls.mjs", - "load:relay:model": "node dev/scripts/run-relay-load-model.mjs", - "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs", - "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev", - "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-reserve-dead-man-alert.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs && pnpm test:runner", - "test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-clock-skew-monitor-workflow.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-resume-phone.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/watch-relay-reserve-cell.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", + "pretest": "pnpm test:runner", + "test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test", "typecheck": "pnpm -r typecheck", "test:runner": "node --import tsx --conditions=orca-source --test dev/scripts/run-relay-tests.test.mjs" }, diff --git a/cloud/pnpm-lock.yaml b/cloud/pnpm-lock.yaml index 1e85c443705..c51f851cdf9 100644 --- a/cloud/pnpm-lock.yaml +++ b/cloud/pnpm-lock.yaml @@ -308,31 +308,6 @@ importers: specifier: ^5.0.5 version: 5.0.5(@types/node@24.19.2)(vite@8.3.4(@types/node@24.19.2)(esbuild@0.28.2)(tsx@4.23.15)) - apps/relay-ops: - dependencies: - '@hono/node-server': - specifier: ^2.1.5 - version: 2.1.5(hono@4.13.13) - hono: - specifier: ^4.13.13 - version: 4.13.13 - zod: - specifier: ^4.6.7 - version: 4.6.7 - devDependencies: - '@types/node': - specifier: ^24.19.2 - version: 24.19.2 - tsx: - specifier: ^4.23.15 - version: 4.23.15 - typescript: - specifier: ^7.0.2 - version: 7.0.2 - vitest: - specifier: ^5.0.5 - version: 5.0.5(@types/node@24.19.2)(vite@8.3.4(@types/node@24.19.2)(esbuild@0.28.2)(tsx@4.23.15)) - packages/postgres-schema: devDependencies: '@types/node': diff --git a/config/scripts/win32-test-lane-registration.test.mjs b/config/scripts/win32-test-lane-registration.test.mjs index 27df7880378..e146062f56e 100644 --- a/config/scripts/win32-test-lane-registration.test.mjs +++ b/config/scripts/win32-test-lane-registration.test.mjs @@ -295,7 +295,7 @@ function readWindowsWorkflow() { ) } const run = String(step.run ?? '') - if (!run.includes('vitest run')) { + if (!/(?:vitest|vitest\.mjs) run/.test(run)) { throw new Error(`The "${name}" step no longer invokes vitest; this guard is stale.`) } return run